diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..d72fd520b1 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.pdf binary diff --git a/.github/ISSUE_TEMPLATE/bug_desktop.yml b/.github/ISSUE_TEMPLATE/bug_desktop.yml new file mode 100644 index 0000000000..f029a62ec4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_desktop.yml @@ -0,0 +1,93 @@ +name: Bug report (Desktop app) +description: Something does not work in the FreeToken Desktop app. +labels: ["bug", "Desktop"] +body: + - type: markdown + attributes: + value: | + Before opening an issue, please read the [FAQ](https://github.com/FlashML-org/FreeToken/issues/84) and the [Roadmap](https://github.com/FlashML-org/FreeToken/issues/79). Most install and runtime problems are answered in the FAQ. Reports missing the information below may be closed until it is provided; see [CONTRIBUTING.md](https://github.com/FlashML-org/FreeToken/blob/main/CONTRIBUTING.md#reporting-issues). + - type: checkboxes + id: checks + attributes: + label: Before you start + options: + - label: I have read the [FAQ](https://github.com/FlashML-org/FreeToken/issues/84) and my problem is not answered there. + required: true + - label: I have read the [Roadmap](https://github.com/FlashML-org/FreeToken/issues/79) and this is not already planned there. + required: true + - label: I have searched [existing issues](https://github.com/FlashML-org/FreeToken/issues?q=is%3Aissue) and found no duplicate. + required: true + - label: I have restarted the Desktop app to pick up the latest update and the problem still happens. + required: true + - type: textarea + id: description + attributes: + label: What happened + description: What you did, what you expected, and what happened instead. + validations: + required: true + - type: input + id: version + attributes: + label: Desktop app version + description: Shown in the app's settings / about page. + validations: + required: true + - type: dropdown + id: os + attributes: + label: OS + options: + - Windows 11 + - Windows 10 + - Ubuntu + - Debian + - Fedora + - Arch Linux + - Other Linux + validations: + required: true + - type: input + id: os_detail + attributes: + label: OS details + description: Distribution version (e.g. Ubuntu 24.04), kernel, desktop environment, or anything unusual about the system. + - type: input + id: gpu + attributes: + label: GPU and driver + description: "GPU model, memory, driver/runtime version, and architecture from `nvidia-smi` or `rocminfo`. Mention other GPUs in the machine and whether the run used CUDA or ROCm/HIP." + placeholder: Radeon 8060S gfx1151, ROCm 10.x; or RTX 4060 Laptop 8GB, driver 580.xx + validations: + required: true + - type: input + id: cpu + attributes: + label: CPU and system RAM + placeholder: i9-13900H, 32GB + validations: + required: true + - type: input + id: model + attributes: + label: Checkpoint + description: The exact Hugging Face or ModelScope ID, not just the model name. + placeholder: Qwen/Qwen3.6-35B-A3B-FP8 + validations: + required: true + - type: textarea + id: settings + attributes: + label: Model settings + description: The settings used when loading the model (context length, backend, any changed defaults). + - type: textarea + id: log + attributes: + label: Engine log + description: "**Logs → Server status → Copy** in the app, pasted as text, not a screenshot." + render: text + - type: textarea + id: extra + attributes: + label: Anything else + description: Proxies, unusual setups, or anything that may matter. diff --git a/.github/ISSUE_TEMPLATE/bug_engine.yml b/.github/ISSUE_TEMPLATE/bug_engine.yml new file mode 100644 index 0000000000..d47f1a9bca --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_engine.yml @@ -0,0 +1,105 @@ +name: Bug report (Engine) +description: Something does not work with the FreeToken engine (ft command line, pip wheel, or source build). +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Before opening an issue, please read the [FAQ](https://github.com/FlashML-org/FreeToken/issues/84) and the [Roadmap](https://github.com/FlashML-org/FreeToken/issues/79). Most install and runtime problems are answered in the FAQ. Reports missing the information below may be closed until it is provided; see [CONTRIBUTING.md](https://github.com/FlashML-org/FreeToken/blob/main/CONTRIBUTING.md#reporting-issues). + - type: checkboxes + id: checks + attributes: + label: Before you start + options: + - label: I have read the [FAQ](https://github.com/FlashML-org/FreeToken/issues/84) and my problem is not answered there. + required: true + - label: I have read the [Roadmap](https://github.com/FlashML-org/FreeToken/issues/79) and this is not already planned there. + required: true + - label: I have searched [existing issues](https://github.com/FlashML-org/FreeToken/issues?q=is%3Aissue) and found no duplicate. + required: true + - label: I am on the latest release, or on a freshly rebuilt `main` when building from source. + required: true + - type: textarea + id: description + attributes: + label: What happened + description: What you did, what you expected, and what happened instead. + validations: + required: true + - type: dropdown + id: install + attributes: + label: How did you install FreeToken + options: + - pip / uv wheel + - Built from source + validations: + required: true + - type: input + id: version + attributes: + label: FreeToken version + description: "`ft --version`, or `git rev-parse --short HEAD` when building from source." + validations: + required: true + - type: dropdown + id: os + attributes: + label: OS + options: + - Windows 11 + - Windows 10 + - WSL2 on Windows 11 + - WSL2 on Windows 10 + - Ubuntu + - Debian + - Fedora + - Arch Linux + - Other Linux + validations: + required: true + - type: input + id: os_detail + attributes: + label: OS details + description: Distribution version (e.g. Ubuntu 24.04), kernel, WSL distro, Python version, or anything unusual about the system. + - type: input + id: gpu + attributes: + label: GPU and driver + description: "GPU model, memory, driver/runtime version, and architecture from `nvidia-smi` or `rocminfo`. Mention other GPUs in the machine and whether the run used CUDA or ROCm/HIP." + placeholder: Radeon 8060S gfx1151, ROCm 10.x; or RTX 4060 Laptop 8GB, driver 580.xx + validations: + required: true + - type: input + id: cpu + attributes: + label: CPU and system RAM + placeholder: i9-13900H, 32GB + validations: + required: true + - type: input + id: model + attributes: + label: Checkpoint + description: The exact Hugging Face or ModelScope ID, not just the model name. + placeholder: Qwen/Qwen3.6-35B-A3B-FP8 + validations: + required: true + - type: textarea + id: command + attributes: + label: Command + description: The exact command you ran. + render: shell + - type: textarea + id: log + attributes: + label: Full log + description: The full log as text, not a screenshot of the last line. + render: text + - type: textarea + id: extra + attributes: + label: Anything else + description: Proxies, unusual setups, or anything that may matter. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..c3517a9e4f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,17 @@ +blank_issues_enabled: false +contact_links: + - name: FAQ (read first) + url: https://github.com/FlashML-org/FreeToken/issues/84 + about: Most install and runtime problems are already answered here. + - name: Roadmap (read first) + url: https://github.com/FlashML-org/FreeToken/issues/79 + about: What we are working on next. Please do not open a new issue for something already on the Roadmap. + - name: Usage questions (Discord) + url: https://discord.gg/MsA277cJzZ + about: Ask on the Community Discord instead of opening an issue. + - name: Usage questions (WeChat, CN) + url: https://github.com/FlashML-org/FreeToken/blob/main/assets/freetoken-wechatgroup.png + about: Scan the QR code to join the Community WeChat group. + - name: Development discussion + url: https://join.slack.com/t/flashml/shared_invite/zt-3zpdh5j10-9dwTXrgLiqpVxizhA9KVbA + about: Join the Developer Slack to discuss Roadmap items before starting work. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000000..131c9f4277 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,46 @@ +name: Feature request +description: Suggest a feature or improvement. For a model or checkpoint that does not load, use "Support Model Checkpoint" instead. +labels: ["feature"] +body: + - type: markdown + attributes: + value: | + Before opening an issue, please check the [Roadmap](https://github.com/FlashML-org/FreeToken/issues/79) and the [FAQ](https://github.com/FlashML-org/FreeToken/issues/84). If your request is already on the Roadmap, please do not open a new issue for it. Features not on the Roadmap should start as an issue, not a PR; see [CONTRIBUTING.md](https://github.com/FlashML-org/FreeToken/blob/main/CONTRIBUTING.md). + - type: checkboxes + id: checks + attributes: + label: Before you start + options: + - label: I have read the [Roadmap](https://github.com/FlashML-org/FreeToken/issues/79) and this is not already planned there. + required: true + - label: I have read the [FAQ](https://github.com/FlashML-org/FreeToken/issues/84). + required: true + - label: I have searched [existing issues](https://github.com/FlashML-org/FreeToken/issues?q=is%3Aissue) and found no duplicate. + required: true + - type: dropdown + id: target + attributes: + label: Applies to + options: + - Desktop app + - Engine + - Both + validations: + required: true + - type: textarea + id: problem + attributes: + label: What problem does this solve + description: The use case, and why current FreeToken does not cover it. + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed solution + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives considered diff --git a/.github/ISSUE_TEMPLATE/model_checkpoint.yml b/.github/ISSUE_TEMPLATE/model_checkpoint.yml new file mode 100644 index 0000000000..ca89db09e0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/model_checkpoint.yml @@ -0,0 +1,64 @@ +name: Support Model Checkpoint +description: A checkpoint FreeToken cannot load, whether a new model architecture or an unsupported checkpoint or quantization of a supported one. +labels: ["new-model"] +body: + - type: markdown + attributes: + value: | + Before opening an issue, please check the [supported models](https://github.com/FlashML-org/FreeToken/blob/main/docs/models.md), the [Roadmap](https://github.com/FlashML-org/FreeToken/issues/79) and the [FAQ](https://github.com/FlashML-org/FreeToken/issues/84). A checkpoint listed there that fails to load is a bug: use a Bug report instead. Other checkpoints of a supported architecture usually work without changes; try them first. If the architecture is supported but a specific unlisted checkpoint or quantization fails to load, this is the right template. + - type: checkboxes + id: checks + attributes: + label: Before you start + options: + - label: I have checked the [supported models](https://github.com/FlashML-org/FreeToken/blob/main/docs/models.md) and this checkpoint is not listed there. A listed checkpoint that fails is a bug; use a Bug report instead. + required: true + - label: I have read the [Roadmap](https://github.com/FlashML-org/FreeToken/issues/79) and this model is not already planned there. + required: true + - label: I have read the [FAQ](https://github.com/FlashML-org/FreeToken/issues/84). + required: true + - label: I have searched [existing issues](https://github.com/FlashML-org/FreeToken/issues?q=is%3Aissue) and found no duplicate. + required: true + - type: input + id: link + attributes: + label: Hugging Face link + description: The exact checkpoint you want to run, not just the model family. + placeholder: https://huggingface.co/Qwen/Qwen3.6-35B-A3B-FP8 + validations: + required: true + - type: dropdown + id: arch + attributes: + label: Is the model architecture already supported + description: Check the architecture, not the checkpoint, against the [supported models](https://github.com/FlashML-org/FreeToken/blob/main/docs/models.md) table. + options: + - Yes, but this checkpoint or quantization does not load + - No, this is a new model architecture + - Not sure + validations: + required: true + - type: dropdown + id: quant + attributes: + label: Is the quantization already supported + description: FP8, NVFP4, MXFP4, and qualified GGUF variants are supported; name the exact format that fails. + options: + - Not quantized + - Yes, but this checkpoint's weight format does not load + - No, a new quantization + - GGUF, but this architecture or quantization does not load + - Not sure + validations: + required: true + - type: textarea + id: log + attributes: + label: What happens when you load it + description: The error or log from trying to load the checkpoint, as text. Leave empty if you have not tried. + render: text + - type: textarea + id: extra + attributes: + label: Anything else + description: Links to other engines that support it, or anything that may matter. diff --git a/.github/issue-labeler.yml b/.github/issue-labeler.yml new file mode 100644 index 0000000000..8d51b626ed --- /dev/null +++ b/.github/issue-labeler.yml @@ -0,0 +1,14 @@ +windows: + - '### OS\s+Windows 1[01]' +WSL: + - '### OS\s+WSL2' +linux: + - '### OS\s+(Ubuntu|Debian|Fedora|Arch Linux|Other Linux)' +amd: + - '### GPU and driver\s+[^\n]*(AMD|Radeon|ROCm)' +Desktop: + - '### Applies to\s+(Desktop app|Both)' +gguf: + - '### Is the quantization already supported\s+GGUF' +quant: + - '### Is the quantization already supported\s+No,' diff --git a/.github/workflows/freetoken-swap-daemon.yml b/.github/workflows/freetoken-swap-daemon.yml new file mode 100644 index 0000000000..e517392edd --- /dev/null +++ b/.github/workflows/freetoken-swap-daemon.yml @@ -0,0 +1,173 @@ +# What: set name to FreeToken swap daemon; why: GitHub displays this label in checks and operators use it to identify the daemon lane or step. +name: FreeToken swap daemon + +# What: set on to its nested mapping; why: GitHub evaluates these events to decide whether the daemon lane is eligible to run. +on: + # What: set pull request to its nested mapping; why: changes targeting the integration branch must pass the daemon contract before merge. + pull_request: + # What: set branches to [main]; why: only pull requests aimed at main receive this automatic validation. + branches: [main] + # What: set workflow dispatch to its nested mapping; why: maintainers can rerun the public lane without altering the candidate commit. + workflow_dispatch: + +# What: set permissions to its nested mapping; why: the workflow token receives only the capabilities declared in this mapping. +permissions: + # What: set contents to read; why: checkout can read repository content while pull-request code receives no write token. + contents: read + +# What: configure concurrency as a nested section; why: GitHub consumes concurrency to preserve this lane's repository gate, ordered execution, or bounded result reporting. +concurrency: + # What: set group to freetoken-swap-daemon-${{ github.ref }}; why: runs for different refs do not cancel one another, while stale runs for one ref share a key. + group: freetoken-swap-daemon-${{ github.ref }} + # What: set cancel in progress to true; why: a newer commit supersedes an older run for the same workflow and ref. + cancel-in-progress: true + +# What: set jobs to its nested mapping; why: GitHub treats each nested entry as an independently gated validation job. +jobs: + # What: set daemon linux to its nested mapping; why: this job isolates the torch-free daemon contract on the hosted Linux process model. + daemon-linux: + # This is a hosted, secret-free smoke lane. Never route pull-request code to + # the repository's self-hosted engine builder or any protected runtime. + # What: set if to github.repository == 'dbourdea/FreeToken'; why: fork or renamed-repository execution is rejected before hosted work begins. + if: github.repository == 'dbourdea/FreeToken' + # What: set runs on to ubuntu-latest; why: the daemon suite exercises Linux process behavior on an isolated hosted runner. + runs-on: ubuntu-latest + # What: set timeout minutes to 10; why: a stalled install or child-process test cannot consume the runner beyond the bounded window. + timeout-minutes: 10 + # What: set steps to its nested mapping; why: the runner preserves checkout, dependency installation, testing, reporting, and final gating in this order. + steps: + # What: set uses to actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd; why: the immutable action revision prevents an upstream tag change from altering checkout behavior. + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + # What: set name to Install torch-free daemon test dependencies; why: GitHub displays this label in checks and operators use it to identify the daemon lane or step. + - name: Install torch-free daemon test dependencies + # What: set run to |; why: the runner executes this exact command or block scalar as the step's behavior. + # What: preserve the exact python m pip install disable pip version check dependency-install command; why: the hosted lane executes this byte-preserved command to install the torch-free packages required by the daemon suite. + # What: add pytest's constraint to the shared pip command; why: the hosted lane needs the supported test runner without unrelated production stacks. + # What: add FastAPI's constraint to the shared pip command; why: daemon API tests construct the control and routing application. + # What: add HTTPX's constraint to the shared pip command; why: the test client exercises in-process HTTP routes and streaming behavior. + # What: add Pydantic's constraint to the shared pip command; why: FastAPI request and response models require the supported validation layer. + # What: add Uvicorn's constraint to the shared pip command; why: daemon startup imports the ASGI server without requiring GPU packages. + run: | + python -m pip install --disable-pip-version-check \ + 'pytest>=8,<9' \ + 'fastapi>=0.115,<1' \ + 'httpx>=0.27,<1' \ + 'pydantic>=2.9,<3' \ + 'uvicorn>=0.30,<1' + # What: set name to Run daemon suite, including disposable Linux child gates; why: GitHub displays this label in checks and operators use it to identify the daemon lane or step. + - name: Run daemon suite, including disposable Linux child gates + # What: set id to daemon-tests; why: later reporting and gate expressions address this step outcome through the stable identifier. + id: daemon-tests + # What: set continue on error to true; why: the reporting step can inspect JUnit output before the final gate restores failure status. + continue-on-error: true + # What: set env to its nested mapping; why: the step receives only the report path and prior-step outcome inputs needed by its command. + env: + # What: add the repository's python directory to pytest imports; why: the daemon suite loads freetoken directly without installing torch-heavy runtime dependencies. + PYTHONPATH: python + # What: set run to python -m pytest tests/daemon -q --junitxml="$RUNNER_TEMP/daemon.xml"; why: the runner executes this exact command or block scalar as the step's behavior. + run: python -m pytest tests/daemon -q --junitxml="$RUNNER_TEMP/daemon.xml" + # What: set name to Report bounded test result; why: GitHub displays this label in checks and operators use it to identify the daemon lane or step. + - name: Report bounded test result + # What: run the reporting step regardless of earlier outcomes; why: JUnit notices and errors remain visible even when the daemon test step failed or was cancelled. + if: always() + # What: set env to its nested mapping; why: the step receives only the report path and prior-step outcome inputs needed by its command. + env: + # What: preserve the exact report runner temp daemon xml reporter fragment; why: the inline script consumes this byte-preserved fragment as part of its JUnit parse, annotation, or final failure decision. + REPORT: ${{ runner.temp }}/daemon.xml + # What: preserve the exact outcome steps daemon tests outcome reporter fragment; why: the inline script consumes this byte-preserved fragment as part of its JUnit parse, annotation, or final failure decision. + OUTCOME: ${{ steps.daemon-tests.outcome }} + # What: set run to |; why: the runner executes this exact command or block scalar as the step's behavior. + # What: start the embedded Python reporter; why: the shell step needs a bounded script to parse JUnit XML and restore the test outcome. + # What: import os inside the inline reporter; why: the reporter directly uses os to read environment state, exit status, or JUnit XML. + # What: import sys inside the inline reporter; why: the reporter directly uses sys to read environment state, exit status, or JUnit XML. + # What: import xml.etree.ElementTree inside the inline reporter; why: the reporter directly uses xml etree element tree to read environment state, exit status, or JUnit XML. + # What: compute reporter state root from et parse os environ report getroot; why: the later JUnit summary or failure gate reads root to determine its annotation and exit behavior. + # What: compute reporter state suites from root if root tag testsuite else root findall; why: the later JUnit summary or failure gate reads suites to determine its annotation and exit behavior. + # What: start the JUnit outcome-count mapping; why: the reporter aggregates tests, failures, errors, and skips across every testsuite node. + # What: sum one JUnit outcome attribute across suites; why: multi-suite reports need a single count for the workflow notice and failure diagnosis. + # What: enumerate the four JUnit outcome counters; why: the summary reports total tests and distinguishes failed, errored, and skipped cases. + # What: finish the JUnit count-comprehension; why: all four counters must be aggregated before the workflow emits its summary notice. + # What: emit the print workflow annotation; why: GitHub surfaces this notice or error to identify the bounded daemon result without exposing raw private artifacts. + # What: compute reporter state notice title from daemon linux suite; why: the later JUnit summary or failure gate reads notice title to determine its annotation and exit behavior. + # What: compute reporter state join f key from value for key value in counts items; why: the later JUnit summary or failure gate reads join f key to determine its annotation and exit behavior. + # What: finish the summary formatting expression; why: the notice must include every computed JUnit counter in one readable message. + # What: compute reporter state failures from the JUnit expression delimiter; why: the later JUnit summary or failure gate reads failures to determine its annotation and exit behavior. + # What: iterate for case in root iter testcase in the reporter; why: the inline reporter inspects every bounded suite, testcase, or escaped annotation component before deciding the result. + # What: compute reporter state node from case find failure; why: the later JUnit summary or failure gate reads node to determine its annotation and exit behavior. + # What: test whether the case lacks a failure node; why: the reporter then falls back to an error node so both JUnit failure categories are covered. + # What: compute reporter state node from case find error; why: the later JUnit summary or failure gate reads node to determine its annotation and exit behavior. + # What: test whether the case lacks both failure and error nodes; why: successful and skipped cases do not need GitHub error annotations. + # What: skip test cases without failure or error nodes; why: successful and skipped cases do not need GitHub error annotations. + # What: compute reporter state test id from f case get classname case get name strip; why: the later JUnit summary or failure gate reads test id to determine its annotation and exit behavior. + # What: compute reporter state message from node get message or test failed splitlines; why: the later JUnit summary or failure gate reads message to determine its annotation and exit behavior. + # What: compute reporter state detail from next; why: the later JUnit summary or failure gate reads detail to determine its annotation and exit behavior. + # What: start selecting the first nonempty failure-detail line; why: annotations need a concise diagnostic instead of the entire traceback payload. + # What: normalize each candidate diagnostic line before selection; why: whitespace-only lines must not become the visible failure detail. + # What: iterate for line in node text or splitlines in the reporter; why: the inline reporter inspects every bounded suite, testcase, or escaped annotation component before deciding the result. + # What: gate the reporter on if line lstrip startswith; why: the inline reporter emits errors or exits only when this parsed JUnit or step-outcome predicate requires it. + # What: preserve the exact group boundary around if line lstrip startswith and the JUnit expression delimiter reporter fragment; why: the inline script consumes this byte-preserved fragment as part of its JUnit parse, annotation, or final failure decision. + # What: preserve the exact group boundary around the JUnit expression delimiter and the JUnit expression delimiter reporter fragment; why: the inline script consumes this byte-preserved fragment as part of its JUnit parse, annotation, or final failure decision. + # What: preserve the exact group boundary around the JUnit expression delimiter and if detail reporter fragment; why: the inline script consumes this byte-preserved fragment as part of its JUnit parse, annotation, or final failure decision. + # What: gate the reporter on if detail; why: the inline reporter emits errors or exits only when this parsed JUnit or step-outcome predicate requires it. + # What: compute reporter state message from f message detail; why: the later JUnit summary or failure gate reads message to determine its annotation and exit behavior. + # What: iterate for old new in r d in the reporter; why: the inline reporter inspects every bounded suite, testcase, or escaped annotation component before deciding the result. + # What: compute reporter state message from message replace old new; why: the later JUnit summary or failure gate reads message to determine its annotation and exit behavior. + # What: preserve the exact failures append test id message reporter fragment; why: the inline script consumes this byte-preserved fragment as part of its JUnit parse, annotation, or final failure decision. + # What: gate the reporter on if os environ outcome success; why: the inline reporter emits errors or exits only when this parsed JUnit or step-outcome predicate requires it. + # What: gate the reporter on if not failures; why: the inline reporter emits errors or exits only when this parsed JUnit or step-outcome predicate requires it. + # What: emit the print workflow annotation; why: GitHub surfaces this notice or error to identify the bounded daemon result without exposing raw private artifacts. + # What: compute reporter state error title from daemon linux suite; why: the later JUnit summary or failure gate reads error title to determine its annotation and exit behavior. + # What: preserve the exact pytest failed without a junit failure reporter fragment; why: the inline script consumes this byte-preserved fragment as part of its JUnit parse, annotation, or final failure decision. + # What: preserve the exact group boundary around pytest failed without a junit failure and for test id message in failures reporter fragment; why: the inline script consumes this byte-preserved fragment as part of its JUnit parse, annotation, or final failure decision. + # What: iterate for test id message in failures in the reporter; why: the inline reporter inspects every bounded suite, testcase, or escaped annotation component before deciding the result. + # What: emit the print f error title test id message workflow annotation; why: GitHub surfaces this notice or error to identify the bounded daemon result without exposing raw private artifacts. + # What: exit the reporter with failure status; why: the workflow must remain red after reporting an unsuccessful daemon test step. + # What: preserve the exact py reporter fragment; why: the inline script consumes this byte-preserved fragment as part of its JUnit parse, annotation, or final failure decision. + run: | + python - <<'PY' + import os + import sys + import xml.etree.ElementTree as ET + + root = ET.parse(os.environ["REPORT"]).getroot() + suites = [root] if root.tag == "testsuite" else root.findall("testsuite") + counts = { + key: sum(int(suite.get(key, "0")) for suite in suites) + for key in ("tests", "failures", "errors", "skipped") + } + print( + "::notice title=daemon Linux suite::" + + ", ".join(f"{key}={value}" for key, value in counts.items()) + ) + failures = [] + for case in root.iter("testcase"): + node = case.find("failure") + if node is None: + node = case.find("error") + if node is None: + continue + test_id = f"{case.get('classname', '')}.{case.get('name', '')}".strip(".") + message = (node.get("message") or "test failed").splitlines()[0][:500] + detail = next( + ( + line.lstrip()[1:].strip() + for line in (node.text or "").splitlines() + if line.lstrip().startswith(">") + ), + "", + ) + if detail: + message = f"{message}; {detail}"[:500] + for old, new in (("%", "%25"), ("\r", "%0D"), ("\n", "%0A")): + message = message.replace(old, new) + failures.append((test_id, message)) + if os.environ["OUTCOME"] != "success": + if not failures: + print( + "::error title=daemon Linux suite::" + "pytest failed without a JUnit failure record" + ) + for test_id, message in failures[:10]: + print(f"::error title={test_id}::{message}") + sys.exit(1) + PY diff --git a/.github/workflows/issue-labels.yml b/.github/workflows/issue-labels.yml new file mode 100644 index 0000000000..35f325bf3b --- /dev/null +++ b/.github/workflows/issue-labels.yml @@ -0,0 +1,30 @@ +name: Label issues + +on: + issues: + types: [opened, edited] + +permissions: + issues: write + +jobs: + label: + runs-on: ubuntu-latest + steps: + - uses: github/issue-labeler@v3.4 + with: + configuration-path: .github/issue-labeler.yml + enable-versioned-regex: 0 + include-title: 0 + repo-token: ${{ github.token }} + - if: github.event.action == 'opened' + uses: actions/github-script@v7 + with: + script: | + const body = context.payload.issue.body || ''; + if (!/### Is the quantization already supported\s+GGUF/.test(body)) return; + await github.rest.issues.createComment({ + ...context.repo, + issue_number: context.issue.number, + body: 'Thanks for the request. General GGUF support is tracked on the [Roadmap](https://github.com/FlashML-org/FreeToken/issues/79) and is not taken as a separate issue yet. Please follow the Roadmap for progress, or comment there with the checkpoint you need. Note that FreeToken loads Hugging Face safetensors checkpoints directly, so an FP8 / NVFP4 / BF16 version of the same model may already work; see [supported models](https://github.com/FlashML-org/FreeToken/blob/main/docs/models.md).', + }); diff --git a/.github/workflows/nightly-wheels.yml b/.github/workflows/nightly-wheels.yml index 8a43ec896f..5cf55434de 100644 --- a/.github/workflows/nightly-wheels.yml +++ b/.github/workflows/nightly-wheels.yml @@ -60,13 +60,18 @@ jobs: GH_TOKEN: ${{ github.token }} FORCE: ${{ inputs.force }} run: | - head_stamp="+g${GITHUB_SHA:0:9}" + head_stamp="${GITHUB_SHA:0:9}" # The release also carries win_amd64 wheels with their own stamp; this - # workflow only builds linux, so compare the linux runtime wheel only. - published_stamp="$(gh api "repos/$WEB_REPO/releases/tags/$WEB_TAG" \ + # workflow only builds linux, so read the linux manifest that + # scripts/publish-wheels.sh writes. Before the first manifest exists, fall + # back to the stamp in the linux runtime wheel's name. + published_stamp="$(curl -fsSL \ + "https://github.com/$WEB_REPO/releases/download/$WEB_TAG/engine-linux_x86_64.json" \ + 2>/dev/null | jq -r '.commit // empty' || true)" + [ -n "$published_stamp" ] || published_stamp="$(gh api "repos/$WEB_REPO/releases/tags/$WEB_TAG" \ --jq '.assets[].name' 2>/dev/null \ | grep -E '^freetoken-.*linux_x86_64\.whl$' \ - | grep -oE '\+g[0-9a-f]{7,}' | head -1 || true)" + | grep -oE '\+g[0-9a-f]{7,}' | head -1 | sed 's/^+g//' || true)" echo "HEAD: $head_stamp published: ${published_stamp:-}" if [ "$FORCE" = "true" ] || [ "$head_stamp" != "$published_stamp" ]; then echo "build=true" >> "$GITHUB_OUTPUT" diff --git a/.gitignore b/.gitignore index bf804e075c..02543bbb21 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,12 @@ __pycache__/ # C extensions *.so +# torch.utils.cpp_extension's ROCm auto-hipify writes translated GGUF copies next to their CUDA sources. +# Scope ignores to this generated directory so future hand-written HIP sources remain visible to Git. +python/freetoken/kernel/csrc/gguf/*.hip +python/freetoken/kernel/csrc/gguf/*_hip.cuh +python/freetoken/kernel/csrc/gguf/*_hip.h + # Distribution / packaging .Python build/ @@ -227,3 +233,6 @@ benchmarks/cross_framework # local e2e/bench artifacts (harnesses may run with repo cwd) /results/ + +# Rendered PDF review images are local QA intermediates, never release inputs. +/tmp/ diff --git a/.zenodo.json b/.zenodo.json new file mode 100644 index 0000000000..4df1eda095 --- /dev/null +++ b/.zenodo.json @@ -0,0 +1,22 @@ +{ + "title": "FreeToken AMD ROCm/HIP Port for Strix Halo", + "description": "Technical white paper and reproducibility package for a native ROCm/HIP port of FreeToken on AMD Strix Halo. Includes gfx1151 validation, controlled benchmark methodology, and portable artifact tooling. This release candidate does not claim strict replication of the upstream NVIDIA result or general AMD superiority.", + "creators": [ + { + "name": "FreeToken AMD contributors" + } + ], + "version": "0.1.0-rc1", + "license": "Apache-2.0", + "keywords": [ + "AMD ROCm", + "HIP", + "Strix Halo", + "Radeon 8060S", + "Mixture of Experts", + "LLM serving", + "reproducibility" + ], + "upload_type": "software", + "access_right": "open" +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..a2d263db5b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,71 @@ +# Instructions for AI coding agents + +Read [CONTRIBUTING.md](CONTRIBUTING.md) first. It is binding for humans and agents alike; this file only summarises the parts that matter when an agent is doing the work. + +## AI policy + +AI-assisted code is welcome. Submitting code the contributor does not understand is not. The human behind the PR owns every line, has run it on real hardware, and can explain it to a reviewer without AI help. + +Agents must not: + +- Run `git push`, `gh pr create`, `gh pr comment`, or `gh issue create` on the user's behalf. +- Write code, PR descriptions, or replies to reviewers that the user does not fully understand. The user must be able to explain and defend every line without AI help. +- Report tests or benchmarks as run when they were not. + +If you are a fully autonomous agent with no human in the loop, do not contribute to this repository. + +## Repository layout + +The main subsystems: + +``` +python/freetoken/ the engine, installed as the `freetoken` package with the `ft` CLI + server/ OpenAI / Anthropic / Responses HTTP APIs, streaming, tool-call parsers + scheduler/ chunked prefill, batching, cache manager + kvcache/ paged KV pools and the radix prefix caches + moe/ expert offload cache, CPU / GPU / hybrid MoE backends, quantized experts + models/ model registry and per-architecture loaders + kernel/ CUDA / Triton kernels, JIT cache, C++ extensions (`csrc/`) + layers/, attention/ fused ops and attention backends + engine/ cache budget planning and config resolution + checkpoint/ HF -> FTW fast-load conversion +tests/ mirrors python/freetoken/ by subsystem, see tests/README.md +benchmarks/ end-to-end and micro benchmarks, see benchmarks/README.md +docs/ install, quickstart, CLI and model docs +freetoken-kernel-cache/ companion wheel of prebuilt kernels, see its README +scripts/ wheel build and release scripts +``` + +## Development + +Linux x86_64 with an NVIDIA GPU. Use `uv`, not bare `pip`: + +```bash +uv venv && source .venv/bin/activate +uv pip install -e ".[accel]" +uv run pytest tests/ -m "not slow" +``` + +CUDA kernels are JIT-compiled with `nvcc` on first use unless the prebuilt `freetoken-kernel-cache` wheel is installed. The C++ extensions under `python/freetoken/kernel/csrc/` are built by `setup.py`; after changing them run `python setup.py build_ext --inplace`. + +Put a new test in the `tests/` directory that mirrors the module it protects, and extend an existing file before creating a new one. Bug fixes come with a test that fails before and passes after. Performance changes come with A/B numbers against `main`. + +## Issues and PRs + +- Search existing issues and PRs before starting. Items on the [Roadmap](https://github.com/FlashML-org/FreeToken/issues/79) are discussed with maintainers before implementation; features not on it start as an issue. +- When helping the user draft an issue, follow the matching template in `.github/ISSUE_TEMPLATE/` (engine bug, model checkpoint, feature request) and fill in every required field: hardware, driver, FreeToken version, checkpoint ID, exact command, and the full log. +- One change per PR, linked to its issue, with the hardware, checkpoint ID and exact command it was tested with. + +## Code comments + +Comments explain a non-obvious "why", never restate the code. Write the code first, then add a comment only where a reader would otherwise be confused. Keep them to one or two lines. Configuration files get no comments. Use ASCII: `-` not em-dash, `->` not arrows. + +## Commits + +[Conventional Commits](https://www.conventionalcommits.org/), one line, imperative, lowercase, no trailing period: + +``` +fix(kvcache): size the SWA radix pool for chunked prefill +``` + +PRs are squash-merged, so the PR title follows the same format. The subject line is usually enough; add a body only when the change needs a why that the diff does not show, and keep it to a few lines. Only commit when the user asks. If the user wants attribution, use `Assisted-by: `, not `Co-authored-by`. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000..13b5409913 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,9 @@ +cff-version: 1.2.0 +message: "Release candidate citation metadata. Replace the release-candidate version with the immutable release tag and DOI before publication." +title: "FreeToken AMD ROCm/HIP Port for Strix Halo" +authors: + - name: FreeToken AMD contributors +license: Apache-2.0 +repository-code: "https://github.com/dbourdea/FreeToken" +version: "0.1.0-rc1" +abstract: "Native ROCm/HIP port of FreeToken with gfx1151 validation and reproducibility tooling for AMD Strix Halo." diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000000..01a8e315aa --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +Read [AGENTS.md](AGENTS.md) before starting any work in this repository. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e286808d7e..62461d43d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ Thanks for helping make FreeToken better. This page covers how to report issues - [FAQ](https://github.com/FlashML-org/FreeToken/issues/84): kept up to date; most install and runtime problems are answered there. - [Roadmap](https://github.com/FlashML-org/FreeToken/issues/79): what we are working on next. -- [Developer Slack](https://join.slack.com/t/flashml/shared_invite/zt-3zpdh5j10-9dwTXrgLiqpVxizhA9KVbA) for development discussion; [Community Discord](https://discord.gg/xzwSnMdsX) or [Community WeChat](https://github.com/FlashML-org/FreeToken/blob/main/assets/freetoken-wechatgroup.png) for usage questions. +- [Developer Slack](https://join.slack.com/t/flashml/shared_invite/zt-3zpdh5j10-9dwTXrgLiqpVxizhA9KVbA) for development discussion; [Community Discord](https://discord.gg/MsA277cJzZ) or [Community WeChat](https://github.com/FlashML-org/FreeToken/blob/main/assets/freetoken-wechatgroup.png) for usage questions. ## Reporting issues diff --git a/README.md b/README.md index 2a56a08653..fd6373d2ec 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@

-| Download | Paper | Developer Slack | Community Discord | Community WeChat | +| Download | Paper | Developer Slack | Community Discord | Community WeChat |

@@ -55,6 +55,17 @@ For More details: - [Quick start](https://github.com/FlashML-org/FreeToken/blob/main/docs/quickstart.md) - [Supported models](https://github.com/FlashML-org/FreeToken/blob/main/docs/models.md) - [CLI reference](https://github.com/FlashML-org/FreeToken/blob/main/docs/cli.md) +- [freetoken-swap named model switching](docs/freetoken-swap.md) + +### AMD ROCm/HIP port + +This release includes the native AMD ROCm/HIP port. The Radeon 8060S work is +qualified as `gfx1151` in [AMD ROCm on Radeon 8060S](docs/amd-rocm-gfx1151.md); +the separate native model-swap qualification records `gfx1150` evidence in +[the final swap report](docs/freetoken-swap-final-qualification-2026-09-23.md). +These architecture labels are not interchangeable. +for scope and platform-specific boundaries, and [Reproducibility and independent +extension](docs/reproducibility.md) for the portable public evidence workflow. ## Citation diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..b04e4029f4 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,7 @@ +# Reporting Security Issues + +To report a security issue, please use the GitHub Security Advisory ["Report a Vulnerability"](https://github.com/FlashML-org/FreeToken/security/advisories/new) tab. Please do not report security issues as public issues or pull requests. + +We will send a response indicating the next steps in handling your report. After the initial reply to your report, the maintainers will keep you informed of the progress towards a fix and full announcement, and may ask for additional information or guidance. + +Report security bugs in third-party dependencies to the person or team maintaining the dependency. diff --git a/_bmad-output/implementation-artifacts/spec-document-all-created-code.md b/_bmad-output/implementation-artifacts/spec-document-all-created-code.md new file mode 100644 index 0000000000..3e37474db0 --- /dev/null +++ b/_bmad-output/implementation-artifacts/spec-document-all-created-code.md @@ -0,0 +1,193 @@ +--- +title: 'Document every branch-created code line' +type: 'chore' +created: '2026-09-15' +status: 'done' +route: 'full' +review_loop_iteration: 3 +baseline_commit: 'a5846c0847cb371313b9ad7ceb93a1933a48d967' +context: [] +--- + + + +## Intent + +**Problem:** The `feat/freetoken-swap` implementation contains substantial code whose individual lines do not all explain both their operation and their purpose. The user requires all code created to date, and all future code, to carry those explanations. + +**Approach:** Use merge base `9ef3651309fe4058672f2cc92069238dea06be1b` as the ownership boundary. Add an adjacent, meaningful native-language comment for every nonblank executable or configuration line introduced after that base, explaining what the line does and why it exists, without changing runtime behavior. + +## Boundaries & Constraints + +**Always:** Cover branch-created Python runtime, benchmark, test, embedded HTML/CSS/JavaScript, workflow YAML, example YAML/TOML, and `pyproject.toml` additions. Preserve shebang placement, module docstrings, decorators, multiline grammar, exact protocol fixtures, exception text, serialized bytes, and public behavior. Explain syntax-only delimiters at their nearest valid structural boundary. Comments themselves do not require recursive comments. Retain the draft PR and private-artifact policy. + +**Never:** Modify the pinned llama-swap checkout, protected runtime/service/model/GPU state, upstream-owned pre-base logic, generated `_bmad/` files, or prose-only documentation merely to inflate coverage. Do not merge PRs, activate services, publish raw artifacts, or substitute generic comments that fail to identify both action and rationale. + +## I/O & Edge-Case Matrix + +| Scenario | Input / State | Expected Output / Behavior | Error Handling | +|----------|--------------|---------------------------|----------------| +| Python statement | Branch-added executable line | Adjacent `#` comment states action and purpose | Compilation/tests catch grammar or behavior changes | +| Embedded web code | HTML/CSS/JavaScript inside Python literal | Native embedded comment documents each safe line | Exact payload/string fixtures remain byte-identical when comments would alter semantics | +| YAML/TOML/config | Branch-added nonblank setting or command | Adjacent format-valid comment explains value and reason | Parser/workflow validation catches invalid syntax | +| Grammar-sensitive content | Docstring, multiline literal, backslash continuation, decorator, or fixture bytes | Explain at nearest valid boundary without mutating value or attachment | Preserve original content and document exception structurally | + + + +## Code Map + +- `.github/workflows/freetoken-swap-daemon.yml`, `examples/freetoken-swap.{toml,yaml}`, `pyproject.toml` -- branch-created operational configuration requiring native comments. +- `benchmarks/swap/*.py` -- three opt-in qualification harnesses; preserve fail-closed maintenance gates and private artifacts. +- `python/freetoken/daemon/{activity,app,catalog,client,inference_proxy,metrics,osproc,performance,proxy,readiness,router,serve_manager,server}.py` and `python/freetoken/server/control_api.py` -- production delta, including embedded router UI. +- `tests/daemon/*.py` -- branch-created behavioral and qualification coverage; preserve exact assertions and fixtures. +- `docs/*.md`, `README.md`, `python/freetoken/daemon/README.md` -- prose evidence, not executable code; do not mechanically annotate. + +## Comment Quality Contract + +- Every explanation must be specific to the documented line and its immediate enclosing symbol. The `what` clause names the semantic effect, not merely the token or delimiter; the `why` clause names the concrete consumer, invariant, failure path, or state transition that requires it. +- Imports must name at least one actual consumer or operation that needs the imported symbol. Runtime lines must connect to their concrete lifecycle/API/data-flow role. Reusing one module-wide rationale across unrelated lines is prohibited. +- Structural delimiters must identify the call, collection, signature, or branch they complete and why that construct must remain grouped. Do not emit generic “close expression,” “invoke with supplied arguments,” “set from,” or equivalent templates without the concrete semantic role. +- Never truncate a comment with `...`, embed historical physical line numbers, or copy secrets/private deployment values. Exact literals and block scalars are described at a stable enclosing boundary without changing their bytes. +- Tests must identify whether a line arranges a condition, performs the behavior, or asserts the outcome, plus the regression/failure mode it protects. Workflow keys must explain their individual operational choice. Example numeric/boolean values must be labeled illustrative and explain their tradeoff rather than imply universal suitability. +- Review every generated explanation against the underlying symbol/caller before accepting coverage. Form/count checks alone are insufficient. +- Never use textual-neighbor clauses such as “between X and Y,” the placeholders “declared structural boundary” or “literal fixture value,” or hedged consumers such as “parser, serializer, or API consumer.” Keep each generated explanation at or below 320 characters so imports and dense expressions remain reviewable. +- Describe `raise` as propagating a failure, `try` as establishing a handler boundary, and `except` as the actual handling behavior. Name the concrete behavior of route, middleware, property, classmethod, dataclass, and lifecycle decorators rather than collapsing them into one decorator template. +- Distinguish runtime accumulators from test fixtures, and distinguish GitHub workflow/JUnit concepts from model-router, llama-swap, and `ft serve` configuration. Test assertions must state the expected behavior or regression, not “assert the assert” or “exact condition.” +- Assertion rationales must describe failure when the asserted predicate is false. Side-effect calls such as `sleep`, `extend`, mutation, notification, and cleanup must not claim their `None` return is consumed. New exception construction is described as raising or signaling, not propagating an existing exception. +- Assignments and returns must name the concrete invariant, normalized value, state transition, or caller contract where it is available; tokenized expressions plus “later consumed” are insufficient. TOML table headers establish namespaces, and each example setting must explain its own operational tradeoff. +- Do not clip string or argument text to an unterminated fragment. Embedded reporter comments must explain aggregation, escaping, annotation, or failure-gating roles at stable scalar boundaries. + +## Tasks & Acceptance + +**Execution:** +- [x] Inventory added hunks from the pinned merge base and produce a deterministic coverage list by file and language. +- [x] Build symbol- and usage-aware explanations satisfying the Comment Quality Contract for every eligible Python and embedded web-code line while preserving grammar and exact-value fixtures. +- [x] Add key/value-specific explanations satisfying the Comment Quality Contract for every eligible workflow/configuration line using YAML or TOML syntax. +- [x] Review the complete diff for generic/repeated filler, factual errors, truncation, stale line references, accidental executable changes, secrets/private metadata, and untouched upstream code. +- [x] Measure and record source-size/import-parse impact, run deterministic verification, and prepare only the intended documentation delta for review; commit, push, and exact-head GitHub verification follow the mandatory review step. + +**Acceptance Criteria:** +- Given the merge-base diff, when every introduced executable/configuration line is inspected, then it has an adjacent meaningful explanation of both what it does and why, or is covered at the nearest valid boundary because inline insertion would change grammar or exact data. +- Given the pre-comment commit and final commit, when executable behavior and public outputs are compared through the existing suite, then all daemon tests pass with no logic regression. +- Given the final implementation state, when pre-publication review begins, then exactly the intended product files and this spec are staged, generated `_bmad/` runtime files are excluded, and commit/push/exact-head GitHub verification remain explicit post-review deliverables. + +## Implementation Notes + +- The pinned merge-base inventory found 11,714 eligible nonblank code/configuration lines across 31 tracked files; the transformation reported 11,714 covered lines. +- Python comments use tokenizer/AST context to distinguish definitions, calls, assignments, parameter defaults, keyword arguments, control flow, literals, and structural continuations. Multiline strings and explicit backslash continuations are documented at their nearest safe boundary. +- YAML block-scalar payloads remain byte-for-byte unchanged; one boundary comment per owned payload line is placed before the scalar key. TOML/YAML comments otherwise sit adjacent to their settings. +- A one-use refinement helper was created during implementation and deleted before review; it is not part of the repository diff. +- Final local evidence: Python compilation passed; stripping only the new comments reproduced the exact baseline bytes for all 31 product files; Python ASTs for 27 edited files remained identical; all four YAML/TOML files parsed; `tests/daemon` completed with 345 passed and 7 expected platform skips; `git diff --check` passed; prose-only documentation remained unchanged; and the privacy-pattern scan found no candidate secrets. +- The fail-closed coverage audit found exactly 11,714 valid what/why comments for 11,714 eligible branch-created lines. It found no missing coverage, non-comment byte differences, malformed explanations, truncated expressions, physical multiline line references, or rejected generic phrases. +- After loop-2 shortening, the 31 product files increased from 680,262 bytes to 3,137,738 bytes (2,457,476 bytes; 4.613x). A fresh five-run median parse of all 27 Python files increased from 337.129 ms to 459.453 ms (1.363x) on this host; this local microbenchmark does not claim runtime request-path overhead. +- Loop-2 final evidence: all 11,714 explanations are 320 characters or fewer; prior rejected phrases and placeholders have zero hits; the cited workflow, routing-group, API-key, readiness, malformed-path, runtime-accumulator, and test-action errors were corrected; exact non-comment bytes and Python ASTs still match baseline; compilation, TOML/YAML parsing, and `git diff --check` pass; and `tests/daemon` again reports 345 passed and 7 platform skips. +- Loop-3 final evidence: exact 11,714-line coverage and 320-character maximum remain intact; assertion truth, side-effect calls, pass handling, keyword-only syntax, exception origin, TOML namespaces/settings, clipped qualifier options, and reporter boundaries were corrected; non-comment bytes and ASTs still match baseline; compilation and `git diff --check` pass; and `tests/daemon` again reports 345 passed and 7 skips. +- Post-review patch evidence: the final cited sentinel, signal, readiness-loop, checkpoint, dependency-continuation, JUnit fallback, and `None`-assertion defects were corrected. The final invariant audit again reports 31 files, 11,714 comments, a 320-character maximum, zero non-comment/AST mismatches, valid TOML/YAML, and a clean `git diff --check`; the latest full suite remains 345 passed and 7 skipped. +- Commit, push, and exact-head hosted CI verification remain pending until the mandatory review workflow permits remote operations. + +## Spec Change Log + +- 2026-09-15 review loop 1 — Trigger: independent reviewers found the first derivation counted comments but allowed syntax restatement, repeated module-wide rationales, factual errors, truncated expressions, stale physical-line references, weak workflow/example/test explanations, and unacknowledged source bloat. Amendment: added the Comment Quality Contract, reset affected tasks, and required symbol/usage-aware generation, factual review, and cost measurement. Known-bad state avoided: 11,714 formally present but predominantly template-generated comments that reduce readability or misdescribe behavior. KEEP: pinned merge-base ownership; exact one-for-one eligible-line inventory; unchanged multiline literal/YAML scalar bytes; no protected-runtime access; AST/config semantic equivalence; privacy scan; full daemon suite; generated `_bmad/` exclusion. +- 2026-09-15 review loop 2 — Trigger: independent review found remaining systemic cross-domain templates, reversed exception-flow descriptions, vague decorator/literal/assertion explanations, unstable neighbor clauses, and individual comments up to 1,791 characters. Amendment: prohibited the observed placeholders and neighbor clauses, capped explanation length, required concrete exception/decorator/test semantics, and required strict workflow/router/fixture domain separation. Known-bad state avoided: formally complete coverage that still misleads maintainers about failure propagation, GitHub reporting, model routing, and expected test outcomes. KEEP: all loop-1 constraints; exact 11,714-line coverage; zero executable/config byte changes after comment stripping; 345-pass/7-skip suite; exact literals and block scalars; measured 5.308x source-size and 1.117x parse-time ratios. +- 2026-09-15 review loop 3 — Trigger: review found assertions with inverted truth semantics, side-effect calls documented as consumed return values, generic assignment/return rationales, TOML tables mislabeled as arguments, repeated unrelated example-setting rationales, and clipped argument text. Amendment: added explicit truth, side-effect, exception-origin, assignment/return, table/value, clipping, and embedded-reporter rules. Known-bad state avoided: comments that pass phrase scans while inventing data flow or hiding safety/configuration intent. KEEP: all prior constraints and verified coverage/byte-equivalence/test evidence; corrected workflow gate, readiness, API-key, routing-group, malformed-path, and runtime-accumulator explanations; 320-character limit. + +## Review Triage Log + +- Blind-1 — `medium`, `bad_spec`: verified syntax-only comments such as `app.py` delimiter explanations do not state the construct's semantic role; grouped into systemic comment-quality re-derivation. +- Blind-2 — `medium`, `bad_spec`: verified imports reuse a generic app-wide rationale instead of naming actual consumers; grouped into systemic comment-quality re-derivation. +- Blind-3 — `medium`, `bad_spec`: verified `/ready` comments incorrectly describe stop/accounting controls, risking maintainer misunderstanding of supervisor readiness; grouped into systemic comment-quality re-derivation. +- Blind-4 — `medium`, `bad_spec`: verified `fresh_health` comments incorrectly mention streaming cleanup rather than bypassing prior-generation cache; grouped into systemic comment-quality re-derivation. +- Blind-5 — `medium`, `bad_spec`: verified 28 comments truncate decisive expression text with `...`; grouped into systemic comment-quality re-derivation. +- Blind-6 — `medium`, `bad_spec`: verified workflow comments repeat a lane-wide rationale and omit the repository gate, timeout, pinned action, and result-reporting reasons; grouped into systemic comment-quality re-derivation. +- Blind-7 — `medium`, `bad_spec`: verified YAML scalar boundary comments embed physical line numbers that become stale after edits; grouped into systemic comment-quality re-derivation. +- Blind-8 — `medium`, `bad_spec`: verified example settings repeat “safe configuration” without explaining illustrative values/tradeoffs; grouped into systemic comment-quality re-derivation. +- Blind-9 — `medium`, `bad_spec`: verified test comments repeat “remains enforced” without arrange/act/assert role or protected failure mode; grouped into systemic comment-quality re-derivation. +- Blind-10 — `false`, `reject`: the user instructed this assistant to comment future code but did not request a repository linter or CI policy; absence of enforcement code is not a defect in this change. +- Blind-11 — `false`, `reject`: workflow parsed values were byte/structure-equivalent to baseline and exact-head hosted CI remains a publication gate, so a new `actionlint` dependency is not needed to validate comment-only edits. +- Blind-12 — `false`, `reject`: embedded HTML/CSS/JavaScript literal bytes were unchanged and Python AST constant equality proved that fact, so browser-side validation is not required for this comment-only boundary documentation. +- Blind-13 — `false`, `reject`: verification commands, baseline commit, counts, and expected results are recorded and independently reproducible; committing raw transient logs was neither requested nor safe/necessary. +- Blind-14 — `medium`, `bad_spec`: verified the 2.6 MB source expansion has developer/import/distribution costs that were not measured or acknowledged; retained as a separate measurement requirement in re-derivation. +- Blind-15 — `high`, `bad_spec`: verified predominant generated templates conflict directly with the approved semantic-comment example and falsely satisfy checked acceptance boxes; grouped into systemic comment-quality re-derivation. +- Edge-1 — `medium`, `bad_spec`: verified the workflow-name explanation repeats the global CI rationale rather than why the display name identifies this lane; duplicate root cause retained as its own verdict row, then grouped with systemic quality findings. +- Edge-2 — `false`, `reject`: `AM` is expected because Step 4 requires changing status to `in-review` without staging; the final status will be staged before commit, so reviewed and committed specs will not diverge. +- Verification-gap — no findings reported. +- Loop2-Blind-1 — `false`, `reject`: `git diff --check` passed against the worktree; CRLF bytes existed only in the temporary review-diff serialization and are not trailing whitespace in product files. +- Loop2-Blind-2 — `false`, `reject`: replacing per-line documentation with only selective comments would violate the human-owned frozen intent requiring every branch-created code/configuration line to be explained. +- Loop2-Blind-3 — `false`, `reject`: the comments adjacent to multiline docstrings describe the underlying string fragments that Python exposes through introspection; they do not claim the `#` comment itself is part of `__doc__`. The separate “declared structural boundary” wording defect is accepted below. +- Loop2-Blind-4 — `medium`, `bad_spec`: verified `.github/workflows/freetoken-swap-daemon.yml` attributes repository gating to `if: always()` instead of its real purpose of preserving result reporting after prior outcomes. +- Loop2-Blind-5 — `medium`, `bad_spec`: verified the workflow attributes `PYTHONPATH` to the JUnit reporter rather than pytest imports in the daemon-test step. +- Loop2-Blind-6 — `medium`, `bad_spec`: verified both TOML `group = "interactive"` explanations incorrectly describe GitHub ref concurrency instead of shared model routing/exclusivity policy. +- Loop2-Blind-7 — `medium`, `bad_spec`: verified example YAML model commands incorrectly use JUnit-reporter rationales for `ft serve` arguments. +- Loop2-Blind-8 — `medium`, `bad_spec`: verified `wait_for_ready` describes `while True` as candidate iteration rather than bounded readiness polling with explicit terminal conditions. +- Loop2-Blind-9 — `medium`, `bad_spec`: verified `wait_for_ready` describes `sleep()` as feeding a return rather than pacing injectable polling. +- Loop2-Blind-10 — `medium`, `bad_spec`: verified the `wait_for_ready` return annotation explanation does not identify the heterogeneous readiness-result mapping contract. +- Loop2-Blind-11 — `medium`, `bad_spec`: verified the `/ready` return explanation omits the supervisor-facing 200/503 acceptance contract. +- Loop2-Blind-12 — `medium`, `bad_spec`: verified runtime accumulator `parts = []` is mislabeled as a fixture. +- Loop2-Blind-13 — `medium`, `bad_spec`: verified the canary `model` field uses a hedged generic consumer rather than its concrete routed-model selection role. +- Loop2-Blind-14 — `medium`, `bad_spec`: verified test assertions use circular tokenized prose and omit the expected behavior or regression they protect. +- Loop2-Blind-15 — `medium`, `bad_spec`: verified the spec's checked semantic-review task and no-defect audit claim were contradicted by current examples; tasks were reset for re-derivation. +- Loop2-Blind-16 — `low`, `patch`: the verification command retained an unresolved `` placeholder; replace it with baseline commit `a5846c0847cb371313b9ad7ceb93a1933a48d967` before completion. +- Loop2-Blind-17 — `low`, `reject`: source and parse costs are measured explicitly, while editor/indexer, formatter, merge-conflict, and distribution effects lack a deterministic repository check and do not change the user-mandated per-line scope. +- Loop2-Edge-1 — `medium`, `bad_spec`: verified `app.py` documents malformed percent-escape rejection (`return None`) as a literal fixture return. +- Loop2-Edge-2 — `medium`, `bad_spec`: verified workflow JUnit aggregation lines retain placeholder structural explanations rather than tests/failures/errors summation semantics. +- Loop2-Edge-3 — `medium`, `bad_spec`: carried Loop2-Blind-4; the `if: always()` rationale is factually wrong at the same location. +- Loop2-Edge-4 — `medium`, `bad_spec`: verified example TOML `api_keys` discusses model memory/startup tradeoffs rather than router authentication and credential replacement. +- Loop2-Edge-5 — `false`, `reject`: carried Edge-2; Step 4 explicitly prohibits staging while constructing and reviewing the diff, so an empty index and untracked in-review spec are expected until review succeeds. +- Loop2-Extra-1 — `medium`, `bad_spec`: verified `test_catalog.py` misclassifies the `wait_for_ready(...)` action assignment as function-signature binding. +- Loop2-Extra-2 — `medium`, `bad_spec`: verified hundreds of “declared structural boundary” explanations fail to name the expression or collection being completed. +- Loop2-Extra-3 — `medium`, `bad_spec`: verified generic `raise`, `try`, and decorator templates reverse or obscure concrete control flow and registration behavior. +- Loop2-Extra-4 — `medium`, `bad_spec`: verified comments up to 1,791 characters and 1,009 textual-neighbor clauses are unstable and make dense imports unreadable; a 320-character limit and neighbor-clause prohibition were added. +- Loop2-Verification-1 — `medium`, `bad_spec`: carried Loop2-Blind-4; independent verification-gap review confirmed the same false `if: always()` rationale and reported no additional gaps. +- Loop3-Blind-1 — `medium`, `bad_spec`: verified assertion comments described failure when a true predicate “violates” an invariant; rationales now require the predicate to remain true and identify false as the failure state. +- Loop3-Blind-2 — `medium`, `bad_spec`: verified `time.sleep(1)` was documented as supplying a later exception; it now explicitly paces bounded health polling. +- Loop3-Blind-3 — `medium`, `bad_spec`: verified `raw.extend()` was documented as a consumed return value; it now describes mutation of the accumulated stream buffer. +- Loop3-Blind-4 — `medium`, `bad_spec`: verified `pass` was mislabeled as predicate evaluation; pass lines now document suppression of the anticipated handled exception. +- Loop3-Blind-5 — `medium`, `bad_spec`: verified the bare signature `*` was mislabeled as a predicate fragment; it now documents the keyword-only API constraint. +- Loop3-Blind-6 — `medium`, `bad_spec`: verified three function definitions used “execute def” boilerplate; definitions now describe declaration and caller reuse, with the hostname helper's safety contract retained in its adjacent docstring. +- Loop3-Blind-7 — `low`, `bad_spec`: verified newly constructed errors were called propagated exceptions; generated wording now says they are raised for the caller. +- Loop3-Blind-8 — `medium`, `bad_spec`: verified broad assignment templates often omitted the reason for normalization or state production; the strengthened contract now requires the concrete invariant where available. +- Loop3-Blind-9 — `medium`, `bad_spec`: verified broad return templates only said callers depend on results; the strengthened contract now requires the caller guarantee where available. +- Loop3-Blind-10 — `medium`, `bad_spec`: verified unrelated TOML settings shared one model/memory/port rationale; each example key now has a setting-specific operational tradeoff. +- Loop3-Blind-11 — `medium`, `bad_spec`: verified TOML table headers were mislabeled as command arguments; they now describe the namespace each table opens. +- Loop3-Blind-12 — `medium`, `bad_spec`: verified three qualifier comments clipped option/help or encoding text into unterminated fragments; those lines now document the complete option or artifact-write behavior. +- Loop3-Blind-13 — `medium`, `bad_spec`: verified embedded reporter delimiter comments did not explain their aggregation and diagnostic roles; cited boundaries now name count aggregation, summary formatting, case filtering, and detail selection. +- Loop3-Blind-14 — `false`, `reject`: carried Loop2-Blind-3; comments outside multiline strings describe the underlying docstring fragments without altering the introspected string bytes. +- Loop3-Blind-15 — `medium`, `bad_spec`: verified checked completion claims were premature while the above factual defects remained; tasks were reset during correction and require another independent review. +- Loop3-Edge-1 — `medium`, `bad_spec`: carried Loop3-Blind-2; the sleep data-flow claim was false and was corrected. +- Loop3-Edge-2 — `medium`, `bad_spec`: carried Loop3-Blind-2; the same line invented consumption of a `None` return. +- Loop3-Edge-3 — `medium`, `bad_spec`: carried Loop3-Blind-11; `[router]` opens a TOML table rather than representing a command argument. +- Loop3-Edge-4 — `medium`, `bad_spec`: carried Loop3-Blind-15; the clean-review claim was premature while the sleep defect remained. +- Loop3-Edge-5 — `false`, `reject`: carried Edge-2 and Loop2-Edge-5; the review workflow prohibits staging until independent review succeeds. +- Loop3-Verification — no verification gaps reported. +- Loop4-Blind-1 — `medium`, `patch`: verified the llama-swap validation comment remained clipped; directly replaced it with the complete pre-launch validation purpose. +- Loop4-Blind-2 — `false`, `reject`: carried Loop2-Blind-3 and Loop3-Blind-14; adjacent comments describe underlying docstring fragments without claiming comments are part of `__doc__`. +- Loop4-Blind-3 — `medium`, `patch`: verified two `None` capture assertions were mislabeled as delimiters; directly replaced them with eviction and retention semantics. +- Loop4-Blind-4 — `medium`, `patch`: verified the signal handler described a newly raised `KeyboardInterrupt` as propagated; directly documented signal-to-interruption conversion. +- Loop4-Blind-5 — `medium`, `patch`: verified SIGTERM registration was explained through the neighboring SIGHUP call; directly documented its restoration-path purpose. +- Loop4-Blind-6 — `medium`, `patch`: verified `MAX_JOBS` was tied to config validation instead of native-extension compilation; directly documented the two-job build cap. +- Loop4-Blind-7 — `medium`, `patch`: verified `proc = None` was mislabeled as fixture state; directly documented the conditional-cleanup sentinel. +- Loop4-Blind-8 — `medium`, `patch`: verified `config +=` omitted the generated model stanza role; directly documented command, readiness, proxy, and optional-TTL sequencing. +- Loop4-Blind-9 — `medium`, `patch`: verified the main `while True` omitted its readiness terminal conditions; directly documented listing success, process exit, and deadline expiry. +- Loop4-Blind-10 — `medium`, `patch`: verified `break` was mislabeled as predicate structure; directly documented successful readiness-loop exit. +- Loop4-Blind-11 — `medium`, `patch`: verified the half-second sleep was tied to a later trial loop; directly documented readiness retry backoff. +- Loop4-Blind-12 — `medium`, `patch`: verified three `save()` calls inherited unrelated neighboring expressions; directly documented trial, cancellation, and restoration checkpoints. +- Loop4-Blind-13 — `medium`, `patch`: verified pip continuation lines were called separate commands; directly documented each constraint as part of the shared install command. +- Loop4-Blind-14 — `medium`, `patch`: verified clean-review evidence was premature for the cited lines; this patch and post-patch verification supersede that claim. +- Loop4-Edge-1 — `medium`, `patch`: verified `observed = None` was mislabeled as fixture state; directly documented the not-yet-fetched statistics sentinel. +- Loop4-Edge-2 — `medium`, `patch`: verified reporter fallback was described as terminal handling; directly documented failure-node to error-node fallback. +- Loop4-Edge-3 — `medium`, `patch`: verified the hostname return contract remained generic; directly documented that the returned identity passed the exact-host safety gate. +- Loop4-Edge-4 — `false`, `reject`: carried Edge-2, Loop2-Edge-5, and Loop3-Edge-5; staging is intentionally prohibited until this review and patch verification finish. +- Loop4-Verification — no verification gaps reported. + +## Design Notes + +Comments should name concrete roles rather than restating syntax. For example, prefer “Normalize the requested alias so all lifecycle locks share one canonical identity” over “Assign the canonical variable.” A structural closing line may be explained by the comment attached to the construct it closes. + +## Verification + +**Commands:** +- `python -m compileall -q python/freetoken/daemon python/freetoken/server benchmarks/swap tests/daemon` -- expected: all edited Python parses. +- `python -m pytest tests/daemon -q` -- expected: complete local daemon suite passes with only platform-qualified skips. +- `git diff --check` -- expected: no whitespace or patch errors. +- `git diff --exit-code a5846c0847cb371313b9ad7ceb93a1933a48d967 -- docs README.md python/freetoken/daemon/README.md` -- expected: prose evidence remains unchanged. +- Public GitHub API check for the pushed exact head -- expected: `FreeToken swap daemon` completes successfully and PR #1 remains draft. diff --git a/assets/freetoken-wechatgroup.png b/assets/freetoken-wechatgroup.png index 2510c9f19d..6b6204d6b9 100644 Binary files a/assets/freetoken-wechatgroup.png and b/assets/freetoken-wechatgroup.png differ diff --git a/benchmarks/README.md b/benchmarks/README.md index 6218903f23..e0d622e434 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -11,6 +11,11 @@ include the full serving path. AIME-25 prompt, checkpoint-recommended sampling. python benchmarks/bench_decode_moe.py --model /path/to/model --backend offload,cpu,hybrid ``` +Use `--cache N` to pin the expert-cache slot count and +`--num-token-override N` to pin the server KV-token pool. Supply both when +comparing cache policies so automatic spare-VRAM allocation does not change the +tested context capacity. + **`bench_load_weight_generic.py`** — expert-bank load time: serial vs parallel O_DIRECT vs pre-repacked FTW, each mode in its own subprocess. Linux-only; stages the FTW under `/var/tmp` (`--ftw-dir` overrides; roughly checkpoint-sized). diff --git a/benchmarks/bench_decode_moe.py b/benchmarks/bench_decode_moe.py index 5662179270..313af33357 100644 --- a/benchmarks/bench_decode_moe.py +++ b/benchmarks/bench_decode_moe.py @@ -100,6 +100,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: help="hybrid: max PCIe fetches/layer; -1 = auto (benched pcie/cpu bandwidth fraction)", ) p.add_argument("--mem-ratio", type=float, default=0.9, help="target VRAM utilization") + p.add_argument( + "--num-token-override", + type=int, + default=None, + help=( + "pin the server KV-token pool capacity instead of accepting its automatic " + "allocation; use this to compare cache policies at the same context capacity" + ), + ) p.add_argument("--gpu", default=None, help="GPU for the serve: a UUID or nvidia-smi index (as ft serve --gpu)") p.add_argument("--no-graph", action="store_true", help="eager decode instead of CUDA graph") @@ -187,6 +196,13 @@ def serve_cmd(args: argparse.Namespace, backend: str, port: int) -> list[str]: ] if args.gpu: cmd += ["--gpu", args.gpu] + # An explicit token-pool size makes cache-policy comparisons fair: auto cache + # sizing otherwise consumes the remaining VRAM for KV pages, while a fixed + # expert cache leaves the server's conservative default KV allocation intact. + if args.num_token_override is not None: + # The benchmark names the value after the Engine field, while the public + # CLI intentionally exposes it as the concise ``--num-tokens`` flag. + cmd += ["--num-tokens", str(args.num_token_override)] if args.cache > 0: cmd += ["--moe-cache-size", str(args.cache)] elif args.cache_rate is not None: diff --git a/benchmarks/bench_gguf_q4_dense_kernel.py b/benchmarks/bench_gguf_q4_dense_kernel.py new file mode 100644 index 0000000000..e845b163ef --- /dev/null +++ b/benchmarks/bench_gguf_q4_dense_kernel.py @@ -0,0 +1,137 @@ +"""Measure the dense native-GGUF Q4_0 vector kernels used by Gemma 4 on GMKtek EVO-X2. + +The Gemma 4 26B A4B Q4_0 checkpoint has four recurring dense projection +geometries. They are supplied as defaults here so a HIP optimization can be +measured before it is allowed into the full OpenAI-compatible server benchmark: + +* 2816 by 4096 attention output projection; +* 8192 by 2816 full-attention QKV projection; +* 4224 by 2816 fused shared-MLP gate/up projection; and +* 10240 by 2816 sliding-window QKV projection. + +Like ``bench_gguf_q4_moe_kernel.py``, this tool creates valid packed Q4_0 +weights on the accelerator and measures only post-warm-up GPU event time. It +does not claim an end-to-end serving rate and must be paired with the API +benchmark before a kernel candidate is accepted. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import torch + +from freetoken.kernel.gguf import ggml_mul_mat_vec_a8 +from freetoken.models.gguf.dequant import GGML_Q4_0, row_bytes + + +# Output rows and input columns, recovered from the exact GMKtek EVO-X2 Gemma GGUF. +DEFAULT_SHAPES = ((2816, 4096), (8192, 2816), (4224, 2816), (10240, 2816)) + + +def _parse_shape(value: str) -> tuple[int, int]: + """Parse a ``ROWSxCOLS`` override and validate its Q4_0 block alignment.""" + try: + rows_text, cols_text = value.lower().split("x", 1) + rows, cols = int(rows_text), int(cols_text) + except ValueError as error: + raise argparse.ArgumentTypeError("shape must be ROWSxCOLS, for example 2816x4096") from error + if rows <= 0 or cols <= 0 or cols % 32: + raise argparse.ArgumentTypeError("rows must be positive and cols must be a positive multiple of 32") + return rows, cols + + +def _parse_args() -> argparse.Namespace: + """Parse reproducible dense-kernel benchmark controls.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--shape", + action="append", + type=_parse_shape, + help="repeatable ROWSxCOLS override; defaults to all production shapes", + ) + parser.add_argument("--vectors", type=int, default=1, help="input rows per kernel call") + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--repetitions", type=int, default=200) + parser.add_argument("--seed", type=int, default=20260828) + parser.add_argument("--json", type=Path, help="write one JSON artifact") + return parser.parse_args() + + +def _make_q4_weight(rows: int, cols: int, device: torch.device) -> torch.Tensor: + """Create finite, contiguous ``[rows, row_bytes(cols)]`` Q4_0 packed weights.""" + blocks = cols // 32 + weight = torch.randint(0, 256, (rows, blocks, 18), dtype=torch.uint8, device=device) + # Q4_0 starts each 18-byte block with an FP16 scale. Use 1/32 rather than + # arbitrary random bytes so the measured real kernel cannot create NaNs. + scale_bytes = torch.tensor([1.0 / 32.0], dtype=torch.float16, device=device).view(torch.uint8) + weight[..., :2] = scale_bytes.reshape(1, 1, 2) + return weight.reshape(rows, row_bytes(cols, GGML_Q4_0)).contiguous() + + +def _average_event_us(kernel, repetitions: int, device: torch.device) -> float: + """Measure an already-warmed kernel with GPU events and return microseconds/call.""" + start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + torch.cuda.synchronize(device) + start.record() + for _ in range(repetitions): + kernel() + end.record() + end.synchronize() + return start.elapsed_time(end) * 1000.0 / repetitions + + +def main() -> int: + """Run the selected dense projection shapes and write a durable JSON result.""" + args = _parse_args() + if args.vectors <= 0 or args.warmup <= 0 or args.repetitions <= 0: + raise ValueError("--vectors, --warmup, and --repetitions must be positive") + if not torch.cuda.is_available(): + raise RuntimeError("this benchmark requires a CUDA or HIP PyTorch device") + torch.manual_seed(args.seed) + device = torch.device("cuda") + shapes = args.shape or DEFAULT_SHAPES + measurements = [] + + for rows, cols in shapes: + weight = _make_q4_weight(rows, cols, device) + x = torch.randn(args.vectors, cols, dtype=torch.bfloat16, device=device) + + def call() -> torch.Tensor: + return ggml_mul_mat_vec_a8(weight, x, int(GGML_Q4_0), rows) + + for _ in range(args.warmup): + result = call() + torch.cuda.synchronize(device) + if not torch.isfinite(result).all(): + raise RuntimeError(f"non-finite result for dense Q4_0 shape {rows}x{cols}") + measurements.append( + { + "rows": rows, + "cols": cols, + "vectors": args.vectors, + "output_shape": list(result.shape), + "average_us": _average_event_us(call, args.repetitions, device), + } + ) + + output = { + "device": torch.cuda.get_device_name(device), + "hip": torch.version.hip, + "torch": torch.__version__, + "quant_type": "Q4_0", + "warmup": args.warmup, + "repetitions": args.repetitions, + "measurements": measurements, + } + print(json.dumps(output, indent=2, sort_keys=True)) + if args.json is not None: + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(json.dumps(output, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/bench_gguf_q4_moe_kernel.py b/benchmarks/bench_gguf_q4_moe_kernel.py new file mode 100644 index 0000000000..793dcf97ac --- /dev/null +++ b/benchmarks/bench_gguf_q4_moe_kernel.py @@ -0,0 +1,184 @@ +"""Measure FreeToken's native GGUF Q4_0 MoE vector kernels in isolation. + +This benchmark deliberately uses the Gemma 4 26B A4B Q4_0 expert geometry +observed on GMKtek EVO-X2: 128 routed experts, top-k 8, hidden width 2816, and MoE +intermediate width 704. It is not a replacement for the end-to-end OpenAI API +benchmark. Instead, it supplies the kernel-level evidence needed before a HIP +port changes Q4_0 launch geometry, indexing, or register use. + +The benchmark creates valid packed Q4_0 rows directly on the GPU. Every block +has a finite FP16 scale and random packed nibbles, so the real production +``ggml_moe_a8_vec`` path, including activation quantization, runs without model +loading, host-cache copying, scheduler work, or HTTP overhead. CUDA events are +used only after warm-up and synchronization; compilation and allocation are not +included in the reported microseconds. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from time import perf_counter + +import torch + +from freetoken.kernel.gguf import ggml_moe_a8_vec +from freetoken.models.gguf.dequant import GGML_Q4_0, row_bytes + + +# These defaults are the verified GMKtek EVO-X2 Gemma 4 26B A4B Q4_0 dimensions. +DEFAULT_EXPERTS = 128 +DEFAULT_TOP_K = 8 +DEFAULT_HIDDEN = 2816 +DEFAULT_INTERMEDIATE = 704 + + +def _parse_args() -> argparse.Namespace: + """Parse only parameters that preserve a reproducible kernel experiment.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--experts", type=int, default=DEFAULT_EXPERTS) + parser.add_argument("--top-k", type=int, default=DEFAULT_TOP_K) + parser.add_argument("--hidden", type=int, default=DEFAULT_HIDDEN) + parser.add_argument("--intermediate", type=int, default=DEFAULT_INTERMEDIATE) + parser.add_argument("--tokens", type=int, default=1, help="decoded token rows per call") + parser.add_argument("--warmup", type=int, default=20, help="unmeasured calls per kernel") + parser.add_argument("--repetitions", type=int, default=200, help="timed calls per kernel") + parser.add_argument("--seed", type=int, default=20260828) + parser.add_argument("--json", type=Path, help="write one reproducible JSON result") + return parser.parse_args() + + +def _require_valid_geometry(args: argparse.Namespace) -> None: + """Reject shapes that cannot be represented by the Q4_0 block format.""" + for name in ("hidden", "intermediate"): + value = getattr(args, name) + if value <= 0 or value % 32: + raise ValueError(f"--{name} must be a positive multiple of 32, got {value}") + for name in ("experts", "top_k", "tokens", "warmup", "repetitions"): + if getattr(args, name) <= 0: + raise ValueError(f"--{name.replace('_', '-')} must be positive") + if args.top_k > args.experts: + raise ValueError("--top-k cannot exceed --experts") + if not torch.cuda.is_available(): + raise RuntimeError("this benchmark requires a CUDA or HIP PyTorch device") + + +def _q4_scale_bytes(device: torch.device) -> torch.Tensor: + """Return little-endian bytes for a finite FP16 Q4_0 scale of 1/32. + + Q4_0 stores two FP16 scale bytes before every 16-byte packed-nibble payload. + A constant finite scale is sufficient for performance work and avoids random + bit patterns that could otherwise create NaNs during the warm-up kernel. + """ + scale = torch.tensor([1.0 / 32.0], dtype=torch.float16, device=device) + return scale.view(torch.uint8).reshape(2) + + +def _make_q4_bank(experts: int, rows: int, columns: int, device: torch.device) -> torch.Tensor: + """Create a contiguous GPU Q4_0 bank shaped exactly like an expert cache. + + The byte layout is ``[expert, output_row, columns//32, 18]`` before the + final view. Byte positions zero and one receive the valid scale, while the + remaining sixteen bytes contain arbitrary Q4_0 nibbles. The final shape + mirrors the packed tensors passed by the Gemma GGUF offload cache. + """ + packed_row_bytes = row_bytes(columns, GGML_Q4_0) + blocks = columns // 32 + bank = torch.randint( + 0, + 256, + (experts, rows, blocks, 18), + dtype=torch.uint8, + device=device, + ) + bank[..., :2] = _q4_scale_bytes(device) + return bank.reshape(experts, rows, packed_row_bytes).contiguous() + + +def _make_topk_ids(tokens: int, top_k: int, experts: int, device: torch.device) -> torch.Tensor: + """Create deterministic valid expert selections without invoking router code.""" + ids = torch.arange(tokens * top_k, dtype=torch.int32, device=device) + return (ids.remainder(experts)).reshape(tokens, top_k).contiguous() + + +def _event_time_us(callable_kernel, repetitions: int, device: torch.device) -> float: + """Return average GPU elapsed time per invocation after explicit synchronization.""" + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + torch.cuda.synchronize(device) + start.record() + for _ in range(repetitions): + callable_kernel() + end.record() + end.synchronize() + return start.elapsed_time(end) * 1000.0 / repetitions + + +def main() -> int: + """Build the two production-shaped calls, warm them, measure them, and emit JSON.""" + args = _parse_args() + _require_valid_geometry(args) + torch.manual_seed(args.seed) + device = torch.device("cuda") + + # Gate/up maps H to 2I and consumes one routing row for every selected expert. + hidden = torch.randn(args.tokens, args.hidden, device=device, dtype=torch.bfloat16) + gate_up = _make_q4_bank(args.experts, 2 * args.intermediate, args.hidden, device) + route_ids = _make_topk_ids(args.tokens, args.top_k, args.experts, device) + + def gate_up_call() -> torch.Tensor: + return ggml_moe_a8_vec( + hidden, gate_up, route_ids, args.top_k, int(GGML_Q4_0), 2 * args.intermediate, args.tokens + ) + + # Down maps I to H. Its input and routing layout match fused_q4_0.py exactly. + inter = torch.randn(args.tokens * args.top_k, args.intermediate, device=device, dtype=torch.bfloat16) + down = _make_q4_bank(args.experts, args.hidden, args.intermediate, device) + + def down_call() -> torch.Tensor: + return ggml_moe_a8_vec( + inter, down, route_ids, 1, int(GGML_Q4_0), args.hidden, args.tokens * args.top_k + ) + + # Materialize the extension and check that valid Q4_0 data produces finite outputs. + for _ in range(args.warmup): + gate_result = gate_up_call() + down_result = down_call() + torch.cuda.synchronize(device) + if not torch.isfinite(gate_result).all() or not torch.isfinite(down_result).all(): + raise RuntimeError("synthetic Q4_0 data produced a non-finite kernel result") + + wall_start = perf_counter() + gate_up_us = _event_time_us(gate_up_call, args.repetitions, device) + down_us = _event_time_us(down_call, args.repetitions, device) + torch.cuda.synchronize(device) + + result = { + "device": torch.cuda.get_device_name(device), + "hip": torch.version.hip, + "torch": torch.__version__, + "quant_type": "Q4_0", + "experts": args.experts, + "top_k": args.top_k, + "hidden": args.hidden, + "intermediate": args.intermediate, + "tokens": args.tokens, + "warmup": args.warmup, + "repetitions": args.repetitions, + "gate_up_us": gate_up_us, + "down_us": down_us, + "pair_us": gate_up_us + down_us, + "wall_seconds": perf_counter() - wall_start, + "gate_up_shape": list(gate_result.shape), + "down_shape": list(down_result.shape), + } + print(json.dumps(result, indent=2, sort_keys=True)) + if args.json is not None: + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/bench_offload_cache_copy.py b/benchmarks/bench_offload_cache_copy.py index 8374510ff2..43440637ec 100644 --- a/benchmarks/bench_offload_cache_copy.py +++ b/benchmarks/bench_offload_cache_copy.py @@ -35,6 +35,10 @@ class ModelProfile: MODELS = { "qwen3.5-35B": ModelProfile(40, 256, 8, "bf16", 2048, 512), + # Qwen3.6-35B-A3B-NVFP4 on GMKtek EVO-X2: 40 MoE layers, 256 experts, top-8, + # H=2048, I=512. This is the production inline-dequant six-bank layout, + # not the older BF16 Qwen3.5 profile above. + "qwen3.6-35B-nvfp4": ModelProfile(40, 256, 8, "nvfp4", 2048, 512), "qwen3-30B": ModelProfile(48, 128, 8, "bf16", 2048, 768), "gemma4-26B": ModelProfile(30, 128, 8, "bf16", 2816, 704), "minimax-m2.5-marlin": ModelProfile(62, 256, 8, "nvfp4_marlin", 3072, 1536), diff --git a/benchmarks/bench_rocm_gqa_attention.py b/benchmarks/bench_rocm_gqa_attention.py new file mode 100644 index 0000000000..503b64d3f1 --- /dev/null +++ b/benchmarks/bench_rocm_gqa_attention.py @@ -0,0 +1,114 @@ +"""Benchmark the Gemma 4 ROCm GQA decode tile without changing serving defaults. + +Gemma 4's sliding attention is 16 query heads by 8 KV heads at head dimension +256. ROCm serving pads this group-of-two GQA tile to 16 query-head lanes so +Triton can lower ``tl.dot`` to RDNA WMMA. This tool calls the same attention +function twice on identical tensors: once with the default tile and once with +an explicitly requested HIP probe tile. It checks numerical agreement before +reporting GPU-event latency, so a compilation success alone is never treated +as an optimization result. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import torch + +from freetoken.kernel.triton.attention import decode_paged_attention + + +def _parse_args() -> argparse.Namespace: + """Parse reproducible ROCm GQA tile benchmark controls.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--probe-block-h", type=int) + parser.add_argument("--probe-block-n", type=int) + parser.add_argument("--probe-num-warps", type=int) + parser.add_argument("--sequence-length", type=int, default=1024) + parser.add_argument("--kv-heads", type=int, default=8) + parser.add_argument("--head-dim", type=int, default=256) + parser.add_argument("--sliding-window", type=int, default=1024, help="zero means full attention") + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--repetitions", type=int, default=200) + parser.add_argument("--seed", type=int, default=20260829) + parser.add_argument("--json", type=Path) + return parser.parse_args() + + +def _event_us(call, repetitions: int) -> float: + """Return post-warm-up accelerator event time for an already-built kernel.""" + start, end = torch.cuda.Event(enable_timing=True), torch.cuda.Event(enable_timing=True) + torch.cuda.synchronize() + start.record() + for _ in range(repetitions): + call() + end.record() + end.synchronize() + return start.elapsed_time(end) * 1000.0 / repetitions + + +def main() -> int: + """Execute the exact Gemma sliding-GQA decode comparison on the HIP device.""" + args = _parse_args() + if not torch.cuda.is_available() or torch.version.hip is None: + raise RuntimeError("this benchmark requires a HIP PyTorch device") + if args.sequence_length <= 0 or args.warmup <= 0 or args.repetitions <= 0: + raise ValueError("sequence length, warmup, and repetitions must be positive") + torch.manual_seed(args.seed) + device = torch.device("cuda") + batch, query_heads, kv_heads, head_dim, splits = 1, 16, args.kv_heads, args.head_dim, 8 + if query_heads % kv_heads: + raise ValueError("--kv-heads must divide Gemma's 16 query heads") + q = torch.randn(batch, query_heads, head_dim, dtype=torch.bfloat16, device=device) + k = torch.randn(args.sequence_length, kv_heads, head_dim, dtype=torch.bfloat16, device=device) + v = torch.randn_like(k) + indptr = torch.tensor([0, args.sequence_length], dtype=torch.int32, device=device) + indices = torch.arange(args.sequence_length, dtype=torch.int32, device=device) + positions = torch.tensor([args.sequence_length - 1], dtype=torch.int64, device=device) + mid_o = torch.empty(batch, query_heads, splits, head_dim, dtype=torch.float32, device=device) + mid_lse = torch.empty(batch, query_heads, splits, dtype=torch.float32, device=device) + num_splits = torch.full((batch,), splits, dtype=torch.int32, device=device) + + def call(probe_h: int | None, probe_n: int | None, probe_warps: int | None) -> torch.Tensor: + return decode_paged_attention( + q, k, v, indptr, indices, positions, mid_o, mid_lse, num_splits, + splits, head_dim**-0.5, + sliding_window=args.sliding_window or None, + rocm_block_h_probe=probe_h, + rocm_block_n_probe=probe_n, rocm_num_warps_probe=probe_warps, + ) + + for _ in range(args.warmup): + default = call(None, None, None) + for _ in range(args.warmup): + candidate = call(args.probe_block_h, args.probe_block_n, args.probe_num_warps) + torch.cuda.synchronize() + torch.testing.assert_close(candidate.float(), default.float(), atol=2e-2, rtol=2e-2) + result = { + "device": torch.cuda.get_device_name(device), + "hip": torch.version.hip, + "geometry": {"q_heads": query_heads, "kv_heads": kv_heads, "head_dim": head_dim}, + "sequence_length": args.sequence_length, + "sliding_window": args.sliding_window or None, + "probe_block_h": args.probe_block_h, + "probe_block_n": args.probe_block_n, + "probe_num_warps": args.probe_num_warps, + "default_us": _event_us(lambda: call(None, None, None), args.repetitions), + "probe_us": _event_us( + lambda: call(args.probe_block_h, args.probe_block_n, args.probe_num_warps), + args.repetitions, + ), + "warmup": args.warmup, + "repetitions": args.repetitions, + } + print(json.dumps(result, indent=2, sort_keys=True)) + if args.json: + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/gmk_evo_x2/README.md b/benchmarks/gmk_evo_x2/README.md new file mode 100644 index 0000000000..e20d59f219 --- /dev/null +++ b/benchmarks/gmk_evo_x2/README.md @@ -0,0 +1,47 @@ +# GMKtek EVO-X2 Qwen API replication harness + +`run_api_benchmark.py` measures a running local FreeToken server through its +OpenAI-compatible streaming API. It does not start a service, modify model +files, change llama-swap, or contact another LAN host. The script refuses to +run unless the operating system host name is GMKtek EVO-X2 or an explicitly supplied +test host. + +Run a quality canary on GMKtek EVO-X2 from the isolated FreeToken environment after +the server is already warm: + +```bash +python benchmarks/gmk_evo_x2/run_api_benchmark.py \ + --model qwen3.6-35b-a3b-nvfp4 \ + --tokenizer /home/operator/freetoken-amd/models/Qwen3.6-35B-A3B-NVFP4 \ + --base-url http://127.0.0.1:1919/v1 \ + --samples 5 \ + --artifact-dir /home/operator/freetoken-amd/artifacts/qwen-replication-$(date -u +%Y%m%dT%H%M%SZ) +``` + +For a fixed-length decode TPS measurement, pass the exact paper or surrogate +prompt and opt into throughput mode. This sends `ignore_eos=true` so all samples +produce the same requested decode length: + +```bash +python benchmarks/gmk_evo_x2/run_api_benchmark.py \ + --model qwen3.6-35b-a3b-nvfp4 \ + --tokenizer /home/operator/freetoken-amd/models/Qwen3.6-35B-A3B-NVFP4 \ + --base-url http://127.0.0.1:1919/v1 \ + --mode throughput --expected-text '' --max-tokens 256 \ + --prompt "" --samples 5 \ + --artifact-dir /home/operator/freetoken-amd/artifacts/qwen-throughput-$(date -u +%Y%m%dT%H%M%SZ) +``` + +The harness writes one immutable JSON artifact per request plus a manifest and +summary. Decode TPS is based on tokenizer-counted generated text rather than +the count of network chunks. A server that fails to provide content, returns a +malformed SSE sequence, or emits an error is marked failed rather than silently +excluded. + +Quality and fixed-length throughput are intentionally separate modes. The +harness is not a paper replication until the exact published prompt, sampling, +cache state, and statistic are supplied in the protocol artifact. +Each passed sample records both decode TPS and `client_prefill_tps`. The latter +is prompt tokens divided by warm TTFT, so it represents the client-visible +request-to-first-text boundary. It is intentionally reported separately from +any server log's internal input-throughput line, whose timing boundary differs. diff --git a/benchmarks/gmk_evo_x2/bench_qwen_q4k_q5k_moe_kernel.py b/benchmarks/gmk_evo_x2/bench_qwen_q4k_q5k_moe_kernel.py new file mode 100644 index 0000000000..1f644020ab --- /dev/null +++ b/benchmarks/gmk_evo_x2/bench_qwen_q4k_q5k_moe_kernel.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Measure the exact Qwen3.6 Q4_K and Q5_K routed-MoE kernels on GMKtek EVO-X2. + +This screening benchmark reads real packed rows from the qualified Qwen3.6 +Q4_K_M GGUF instead of manufacturing bytes. It copies eight actual experts +from one selected MoE layer to the accelerator, uses deterministic routes, and +calls FreeToken's production ``ggml_moe_a8_vec`` binding. The gate/up call has +the model's Q4_K 512-to-2,048 shape; the down call has its Q5_K +2,048-to-512 shape. GPU event time is useful for selecting a kernel candidate +but is never a substitute for the quality-gated OpenAI API measurement. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import torch + +from freetoken.kernel.gguf import ggml_moe_a8_vec +from freetoken.models.gguf.dequant import GGML_Q4_K, GGML_Q5_K +from freetoken.models.gguf.reader import GgufTensor, iter_gguf_tensors + + +# The qualified Qwen model has 256 experts and routes eight experts per token. +DEFAULT_EXPERT_COUNT = 256 +DEFAULT_TOP_K = 8 +DEFAULT_LAYER = 0 + + +def _parse_args() -> argparse.Namespace: + """Read explicit benchmark controls and refuse implicit model selection.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True, type=Path, help="qualified Q4_K_M GGUF") + parser.add_argument("--layer", type=int, default=DEFAULT_LAYER, help="MoE layer to sample") + parser.add_argument("--warmup", type=int, default=30, help="unmeasured production-kernel calls") + parser.add_argument("--repetitions", type=int, default=300, help="timed calls per projection") + parser.add_argument("--json", type=Path, required=True, help="new JSON artifact path") + return parser.parse_args() + + +def _require_inputs(args: argparse.Namespace) -> None: + """Validate every input before mapping model data or reserving the GPU.""" + + if not args.model.is_file(): + raise FileNotFoundError(f"GGUF model is missing: {args.model}") + if args.layer < 0: + raise ValueError("--layer must be non-negative") + if args.warmup <= 0 or args.repetitions <= 0: + raise ValueError("--warmup and --repetitions must be positive") + if args.json.exists(): + raise FileExistsError(f"refusing to overwrite artifact: {args.json}") + if not torch.cuda.is_available(): + raise RuntimeError("this benchmark requires a CUDA or HIP PyTorch device") + + +def _tensor_map(model: Path) -> dict[str, GgufTensor]: + """Index GGUF tensor records once while retaining their zero-copy packed views.""" + + return {tensor.name: tensor for tensor in iter_gguf_tensors(str(model))} + + +def _expert_bank(tensor: GgufTensor, device: torch.device) -> torch.Tensor: + """Copy exactly eight real expert banks to GPU in FreeToken's packed layout. + + The qualified tensors expose torch shape ``[experts, rows, columns]`` and a + packed CPU view ``[experts * rows, row_bytes]``. Reshaping is metadata-only; + selecting the first eight experts bounds device memory while retaining the + quantization bytes used by the real model. + """ + + experts, rows, _columns = tensor.shape + if experts != DEFAULT_EXPERT_COUNT: + raise ValueError(f"expected {DEFAULT_EXPERT_COUNT} experts, got {experts} in {tensor.name}") + packed = tensor.packed().reshape(experts, rows, -1) + return packed[:DEFAULT_TOP_K].contiguous().to(device=device, non_blocking=False) + + +def _event_time_us(kernel, repetitions: int, device: torch.device) -> float: + """Return synchronized average device time in microseconds for one call.""" + + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + torch.cuda.synchronize(device) + start.record() + for _ in range(repetitions): + kernel() + end.record() + end.synchronize() + return start.elapsed_time(end) * 1000.0 / repetitions + + +def _finite(tensor: torch.Tensor, label: str) -> None: + """Fail closed if an experimental kernel creates an invalid floating result.""" + + if not torch.isfinite(tensor).all(): + raise RuntimeError(f"{label} produced non-finite output") + + +def main() -> int: + """Load true packed experts, warm both projections, and write one evidence file.""" + + args = _parse_args() + _require_inputs(args) + device = torch.device("cuda") + tensors = _tensor_map(args.model) + prefix = f"blk.{args.layer}." + gate_name = prefix + "ffn_gate_exps.weight" + up_name = prefix + "ffn_up_exps.weight" + down_name = prefix + "ffn_down_exps.weight" + missing = [name for name in (gate_name, up_name, down_name) if name not in tensors] + if missing: + raise KeyError(f"GGUF lacks required MoE tensors: {missing}") + + # Qwen stores gate and up separately, so screen each real Q4_K bank. The + # production fused path uses the same routed-vector binding for both banks. + gate = _expert_bank(tensors[gate_name], device) + up = _expert_bank(tensors[up_name], device) + down = _expert_bank(tensors[down_name], device) + if tensors[gate_name].ggml_type != GGML_Q4_K or tensors[up_name].ggml_type != GGML_Q4_K: + raise ValueError("Qwen gate/up tensors must be Q4_K for this benchmark") + if tensors[down_name].ggml_type != GGML_Q5_K: + raise ValueError("Qwen down tensor must be Q5_K for this benchmark") + + # One decoded token selects each copied expert once, matching Qwen's top-k + # cardinality while avoiding any router or scheduler work in this screen. + route_ids = torch.arange(DEFAULT_TOP_K, dtype=torch.int32, device=device).reshape(1, -1) + hidden = torch.randn(1, 512, dtype=torch.bfloat16, device=device) + intermediate = torch.randn(DEFAULT_TOP_K, 2048, dtype=torch.bfloat16, device=device) + + def gate_call() -> torch.Tensor: + return ggml_moe_a8_vec(hidden, gate, route_ids, DEFAULT_TOP_K, int(GGML_Q4_K), 2048, 1) + + def up_call() -> torch.Tensor: + return ggml_moe_a8_vec(hidden, up, route_ids, DEFAULT_TOP_K, int(GGML_Q4_K), 2048, 1) + + def down_call() -> torch.Tensor: + return ggml_moe_a8_vec(intermediate, down, route_ids, 1, int(GGML_Q5_K), 512, DEFAULT_TOP_K) + + for _ in range(args.warmup): + gate_output = gate_call() + up_output = up_call() + down_output = down_call() + torch.cuda.synchronize(device) + _finite(gate_output, "Q4_K gate") + _finite(up_output, "Q4_K up") + _finite(down_output, "Q5_K down") + + result = { + "schema_version": 1, + "model": str(args.model.resolve()), + "layer": args.layer, + "device": torch.cuda.get_device_name(device), + "hip": torch.version.hip, + "torch": torch.__version__, + "experts_copied": DEFAULT_TOP_K, + "top_k": DEFAULT_TOP_K, + "warmup": args.warmup, + "repetitions": args.repetitions, + "gate_q4k_us": _event_time_us(gate_call, args.repetitions, device), + "up_q4k_us": _event_time_us(up_call, args.repetitions, device), + "down_q5k_us": _event_time_us(down_call, args.repetitions, device), + "gate_shape": list(gate_output.shape), + "up_shape": list(up_output.shape), + "down_shape": list(down_output.shape), + } + result["three_projection_us"] = result["gate_q4k_us"] + result["up_q4k_us"] + result["down_q5k_us"] + args.json.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/gmk_evo_x2/multiturn_state_suite.json b/benchmarks/gmk_evo_x2/multiturn_state_suite.json new file mode 100644 index 0000000000..2a8c54aac9 --- /dev/null +++ b/benchmarks/gmk_evo_x2/multiturn_state_suite.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "description": "Bounded multi-turn state-retention control for GMKtek EVO-X2. It is not a replacement for the paper's coding-agent workflows.", + "turns": [ + { + "id": "remember", + "user": "Remember the codeword azure-17 for this conversation. Reply exactly ACK.", + "expected": "ACK" + }, + { + "id": "recall", + "user": "What codeword did I ask you to remember? Reply only with the codeword.", + "expected": "azure-17" + }, + { + "id": "transform", + "user": "Add 6 to the numeric part of the remembered codeword. Reply only with the decimal integer.", + "expected": "23" + } + ] +} diff --git a/benchmarks/gmk_evo_x2/quality_suite.json b/benchmarks/gmk_evo_x2/quality_suite.json new file mode 100644 index 0000000000..36a159654a --- /dev/null +++ b/benchmarks/gmk_evo_x2/quality_suite.json @@ -0,0 +1,24 @@ +{ + "schema_version": 1, + "description": "Small deterministic Qwen API quality suite for GMKtek EVO-X2. This is a local control, not the FreeToken paper workload.", + "cases": [ + { + "id": "canary_exact", + "prompt": "Return exactly the word GMK_EVO_X2 and nothing else. Do not add punctuation.", + "check": {"kind": "exact", "value": "GMK_EVO_X2"} + }, + { + "id": "arithmetic_exact", + "prompt": "What is 17 times 19? Reply with only the decimal number.", + "check": {"kind": "exact", "value": "323"} + }, + { + "id": "json_schema", + "prompt": "Reply with exactly this JSON object and no other text: {\"status\":\"ok\",\"value\":7}", + "check": { + "kind": "json_fields", + "fields": {"status": "ok", "value": 7} + } + } + ] +} diff --git a/benchmarks/gmk_evo_x2/run_api_benchmark.py b/benchmarks/gmk_evo_x2/run_api_benchmark.py new file mode 100644 index 0000000000..4ed212048e --- /dev/null +++ b/benchmarks/gmk_evo_x2/run_api_benchmark.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 +"""Measure a warm GMKtek EVO-X2 Qwen server through its streamed OpenAI-compatible API. + +This harness validates the host before opening a socket, records each SSE +content event timestamp, counts completed text with the supplied checkpoint +tokenizer, and preserves every failed sample as evidence. It does not start or +stop a server because service lifecycle belongs to the isolated test procedure. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import socket +import statistics +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Iterable + + +# This prompt tests transport and deterministic response handling. It is not +# claimed to reproduce FreeToken's paper workload or to provide a TPS result. +CANARY_PROMPT = "Return exactly the word GMK_EVO_X2 and nothing else. Do not add punctuation." + + +LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1"} + + +def require_loopback_url(value: str) -> str: + """Reject non-loopback targets before benchmark prompts can leave the selected host.""" + # Parse the complete URL so host validation cannot be bypassed with user-info or path text. + parsed = urllib.parse.urlparse(value) + # Require an absolute HTTP endpoint because relative and non-HTTP targets are never valid APIs here. + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise ValueError("--base-url must be an absolute http(s) URL") + # Restrict this host-qualified harness to loopback so prompts cannot leak through a mistyped URL. + if parsed.hostname.lower() not in LOOPBACK_HOSTS: + raise ValueError("--base-url must target a loopback host: localhost, 127.0.0.1, or ::1") + # Remove a trailing slash once so every request path is assembled deterministically. + return value.rstrip("/") + + +@dataclass(frozen=True) +class StreamObservation: + """One content-bearing SSE event and its monotonic arrival timestamp.""" + + offset_seconds: float + content: str + + +def nearest_rank_percentile(values: list[float], percentile: float) -> float | None: + """Return an auditable nearest-rank percentile from observed stream gaps.""" + + if not values: + return None + if not 0 < percentile <= 1: + raise ValueError("percentile must be in the interval (0, 1]") + ordered = sorted(values) + rank = max(1, int((len(ordered) * percentile) + 0.999999999)) + return ordered[rank - 1] + + +def numeric_summary(values: list[float]) -> dict[str, float | None]: + """Summarize a metric while retaining maximum and tail percentiles.""" + + if not values: + return {key: None for key in ("mean", "median", "minimum", "maximum", "stdev", "p50", "p95", "p99")} + return { + "mean": statistics.mean(values), + "median": statistics.median(values), + "minimum": min(values), + "maximum": max(values), + "stdev": statistics.stdev(values) if len(values) > 1 else None, + "p50": nearest_rank_percentile(values, 0.50), + "p95": nearest_rank_percentile(values, 0.95), + "p99": nearest_rank_percentile(values, 0.99), + } + + +def client_prefill_tps(prompt_tokens: int | None, warm_ttft_seconds: float | None) -> float | None: + """Return client-observed prompt tokens per second through the first text token. + + This is deliberately an end-to-end prefill metric: it includes request + transport, queueing, tokenization, prefix-cache lookup, scheduling, and + model prefill until the first visible text token. It is not interchangeable + with a server-internal input-throughput log line, which can begin and end at + different boundaries. ``None`` preserves a missing usage report or an + absent first text token rather than manufacturing a rate. + """ + + if not isinstance(prompt_tokens, int) or warm_ttft_seconds is None or warm_ttft_seconds <= 0: + return None + return prompt_tokens / warm_ttft_seconds + + +def parse_args(argv: list[str]) -> argparse.Namespace: + """Parse explicit inputs so every performance-affecting choice is recorded.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:1919/v1") + parser.add_argument("--model", required=True) + parser.add_argument("--tokenizer", required=True, type=Path) + parser.add_argument("--artifact-dir", required=True, type=Path) + parser.add_argument("--samples", type=int, default=5) + parser.add_argument("--max-tokens", type=int, default=128) + parser.add_argument("--prompt", default=CANARY_PROMPT) + parser.add_argument( + "--reasoning-effort", + choices=("none", "minimal", "low", "medium", "high", "xhigh", "max"), + default="none", + help="Qwen reasoning policy sent to FreeToken and retained in each artifact", + ) + parser.add_argument( + "--mode", + choices=("quality", "throughput"), + default="quality", + help="quality permits natural EOS; throughput requires fixed-length decode", + ) + parser.add_argument( + "--expected-text", + default="GMK_EVO_X2", + help="exact stripped response required in quality mode; empty disables the check", + ) + parser.add_argument("--expected-host", required=True, help="Expected hostname of the explicitly selected test machine") + parser.add_argument("--timeout-seconds", type=float, default=180.0) + parser.add_argument("--warmup", action="store_true") + args = parser.parse_args(argv) + # Validate the destination before any benchmark request can disclose prompt content. + try: + # Normalize one approved loopback endpoint for deterministic request assembly. + args.base_url = require_loopback_url(args.base_url) + except ValueError as error: + # Route validation through argparse so callers receive a standard configuration failure. + parser.error(str(error)) + if args.samples < 1: + parser.error("--samples must be at least one") + if args.mode == "throughput" and args.max_tokens < 2: + parser.error("throughput mode needs --max-tokens of at least two") + if args.timeout_seconds <= 0: + parser.error("--timeout-seconds must be positive") + return args + + +def require_expected_host(expected_host: str) -> str: + """Fail closed unless this process is executing on the declared GMKtek EVO-X2 host.""" + + actual_host = socket.gethostname().lower() + accepted = {expected_host.lower(), expected_host.lower().split(".", 1)[0]} + if actual_host not in accepted: + raise RuntimeError( + f"refusing benchmark on host {actual_host!r}; expected {expected_host!r}" + ) + return actual_host + + +def iter_sse_events(response: Any, started_at: float) -> Iterable[tuple[float, str]]: + """Yield timestamped SSE data fields without hiding malformed payloads.""" + + for raw_line in response: + received_at = time.perf_counter() + line = raw_line.decode("utf-8", errors="strict").rstrip("\r\n") + if line.startswith("data:"): + yield received_at - started_at, line[5:].lstrip() + + +def stream_completion( + args: argparse.Namespace, +) -> tuple[list[StreamObservation], str, float, float, list[str], dict[str, Any] | None]: + """Execute one fixed greedy request and collect content plus protocol errors.""" + + request_body = { + "model": args.model, + "messages": [{"role": "user", "content": args.prompt}], + "stream": True, + "stream_options": {"include_usage": True}, + "temperature": 0.0, + "top_p": 1.0, + "max_tokens": args.max_tokens, + # These are FreeToken request fields, not an OpenAI SDK extension wrapper. + # Sending them at the top level mirrors benchmarks/bench_decode_moe.py. + "top_k": 1, + # Qwen otherwise may emit its optional reasoning stream until the token + # cap. A fixed explicit policy keeps a final-answer quality canary and + # a decode-TPS run comparable across retries. + "reasoning_effort": args.reasoning_effort, + } + if args.mode == "throughput": + # Fixed-length generation makes the decode interval independent of EOS. + request_body["ignore_eos"] = True + request = urllib.request.Request( + args.base_url.rstrip("/") + "/chat/completions", + data=json.dumps(request_body, separators=(",", ":")).encode("utf-8"), + headers={"Content-Type": "application/json", "Accept": "text/event-stream"}, + method="POST", + ) + observations: list[StreamObservation] = [] + protocol_errors: list[str] = [] + usage: dict[str, Any] | None = None + completed = False + started_at = time.perf_counter() + try: + with urllib.request.urlopen(request, timeout=args.timeout_seconds) as response: + for offset, event_data in iter_sse_events(response, started_at): + if event_data == "[DONE]": + completed = True + continue + try: + event = json.loads(event_data) + except json.JSONDecodeError as error: + protocol_errors.append(f"invalid JSON SSE event: {error}") + continue + choices = event.get("choices", []) + if not choices: + event_usage = event.get("usage") + if isinstance(event_usage, dict): + usage = event_usage + continue + delta = choices[0].get("delta", {}) + # Reasoning models may emit their decode tokens in this field. + content = delta.get("content") # Score visible answer text only; reasoning is not output. + # OpenAI streaming commonly sends an empty role-only delta + # before the first generated text. It is not model output and + # must not become the client-observed TTFT timestamp. + if content: + observations.append(StreamObservation(offset, str(content))) + except urllib.error.HTTPError as error: + message = error.read().decode("utf-8", errors="replace") + raise RuntimeError(f"HTTP {error.code}: {message}") from error + except urllib.error.URLError as error: + raise RuntimeError(f"request transport failure: {error}") from error + finished_at = time.perf_counter() + if not completed: + protocol_errors.append("stream ended without [DONE]") + if not observations: + protocol_errors.append("stream contained no content events") + return ( + observations, + "".join(item.content for item in observations), + started_at, + finished_at, + protocol_errors, + usage, + ) + + +def load_tokenizer(path: Path) -> Any: + """Load the local checkpoint tokenizer for an actual generated-token count.""" + + from transformers import AutoTokenizer + + return AutoTokenizer.from_pretrained(path, local_files_only=True, trust_remote_code=False) + + +def make_sample_artifact(args: argparse.Namespace, tokenizer: Any, sample_index: int) -> dict[str, Any]: + """Run one request and return a self-contained, JSON-serializable evidence record.""" + + observations, text, started_at, finished_at, protocol_errors, usage = stream_completion(args) + generated_tokens = len(tokenizer.encode(text, add_special_tokens=False)) + first_offset = observations[0].offset_seconds if observations else None + last_offset = observations[-1].offset_seconds if observations else None + decode_seconds = None if first_offset is None or last_offset is None else last_offset - first_offset + decode_tps = None + if generated_tokens > 1 and decode_seconds is not None and decode_seconds > 0: + decode_tps = (generated_tokens - 1) / decode_seconds + prompt_tokens = usage.get("prompt_tokens") if isinstance(usage, dict) else None + # Compute the client-visible prefill rate from the server-reported prompt + # token count and the same first-text timestamp used for warm TTFT. + # ``input_tps`` remains as a compatibility alias for older artifact readers. + observed_prefill_tps = client_prefill_tps(prompt_tokens, first_offset) + if args.mode == "quality" and args.expected_text and text.strip() != args.expected_text: + protocol_errors.append( + f"quality canary mismatch: expected {args.expected_text!r}, got {text.strip()!r}" + ) + if args.mode == "throughput" and decode_tps is None: + protocol_errors.append("throughput run produced fewer than two generated tokens") + content_event_gaps = [ + observations[index].offset_seconds - observations[index - 1].offset_seconds + for index in range(1, len(observations)) + ] + return { + "schema_version": 2, + "sample_index": sample_index, + "status": "passed" if not protocol_errors else "failed", + "request": { + "base_url": args.base_url, + "model": args.model, + "prompt": args.prompt, + "prompt_sha256": hashlib.sha256(args.prompt.encode("utf-8")).hexdigest(), + "mode": args.mode, + "expected_text": args.expected_text, + "max_tokens": args.max_tokens, + "temperature": 0.0, + "top_p": 1.0, + "top_k": 1, + "reasoning_effort": args.reasoning_effort, + "ignore_eos": args.mode == "throughput", + }, + "timing": { + "wall_seconds": finished_at - started_at, + "warm_ttft_seconds": first_offset, + "decode_seconds": decode_seconds, + "decode_tps": decode_tps, + "client_prefill_tps": observed_prefill_tps, + "input_tps": observed_prefill_tps, + "content_event_gap_seconds": content_event_gaps, + "content_event_gap_summary_seconds": numeric_summary(content_event_gaps), + }, + "usage": usage, + "response": { + "text": text, + "generated_tokens": generated_tokens, + "content_event_count": len(observations), + "content_events": [asdict(item) for item in observations], + }, + "protocol_errors": protocol_errors, + } + + +def write_json(path: Path, value: Any) -> None: + """Write readable JSON once, leaving raw evidence inspectable without custom tools.""" + + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + """Validate scope, optionally warm the server, collect samples, and write a summary.""" + + args = parse_args(sys.argv[1:] if argv is None else argv) + actual_host = require_expected_host(args.expected_host) + args.artifact_dir.mkdir(parents=True, exist_ok=False) + tokenizer = load_tokenizer(args.tokenizer) + manifest = { + "schema_version": 2, + "host": actual_host, + "expected_host": args.expected_host, + "python": sys.version, + "cwd": os.getcwd(), + "arguments": {key: str(value) if isinstance(value, Path) else value for key, value in vars(args).items()}, + "tokenizer_path": str(args.tokenizer.resolve()), + } + write_json(args.artifact_dir / "manifest.json", manifest) + if args.warmup: + warmup = make_sample_artifact(args, tokenizer, 0) + write_json(args.artifact_dir / "warmup.json", warmup) + if warmup["status"] != "passed": + raise RuntimeError("warmup failed; inspect warmup.json before scored samples") + samples = [] + for sample_index in range(1, args.samples + 1): + sample = make_sample_artifact(args, tokenizer, sample_index) + samples.append(sample) + write_json(args.artifact_dir / f"sample-{sample_index:02d}.json", sample) + successful_tps = [ + sample["timing"]["decode_tps"] + for sample in samples + if sample["status"] == "passed" and sample["timing"]["decode_tps"] is not None + ] + successful_ttft = [ + sample["timing"]["warm_ttft_seconds"] + for sample in samples + if sample["status"] == "passed" and sample["timing"]["warm_ttft_seconds"] is not None + ] + successful_prefill_tps = [ + sample["timing"]["client_prefill_tps"] + for sample in samples + if sample["status"] == "passed" and sample["timing"]["client_prefill_tps"] is not None + ] + successful_gaps = [ + gap + for sample in samples + if sample["status"] == "passed" + for gap in sample["timing"]["content_event_gap_seconds"] + ] + summary = { + "schema_version": 2, + "successful_samples": len([sample for sample in samples if sample["status"] == "passed"]), + "requested_samples": args.samples, + "decode_tps": {"samples": successful_tps, **numeric_summary(successful_tps)}, + "client_prefill_tps": {"samples": successful_prefill_tps, **numeric_summary(successful_prefill_tps)}, + "warm_ttft_seconds": {"samples": successful_ttft, **numeric_summary(successful_ttft)}, + "content_event_gap_seconds": {"samples": successful_gaps, **numeric_summary(successful_gaps)}, + "failed_samples": [sample["sample_index"] for sample in samples if sample["status"] != "passed"], + } + write_json(args.artifact_dir / "summary.json", summary) + required_successes = args.samples if args.mode == "throughput" else len( + [sample for sample in samples if sample["status"] == "passed"] + ) + return 0 if required_successes == args.samples else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/gmk_evo_x2/run_concurrent_api_control.py b/benchmarks/gmk_evo_x2/run_concurrent_api_control.py new file mode 100644 index 0000000000..c9c909db25 --- /dev/null +++ b/benchmarks/gmk_evo_x2/run_concurrent_api_control.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +"""Measure simultaneous GMKtek EVO-X2 streamed requests without changing server state. + +The existing scheduler baseline measures one warm request at a time. This +control releases a fixed number of requests together, preserves each raw +response and timing stream, and reports both individual latency and aggregate +throughput. It is a local GMKtek EVO-X2 control, not a reproduction of an upstream +agent workload. The program never starts, stops, or reconfigures a server. +""" + +from __future__ import annotations + +import argparse +import json +import socket +import statistics +import sys +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable + + +DEFAULT_PROMPT = ( + "The scheduler manages incoming inference requests by prioritizing, batching, " + "and assigning them to available compute resources to optimize throughput and latency. " +) * 48 + + +LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1"} + + +def require_loopback_url(value: str) -> str: + """Reject non-loopback targets before benchmark prompts can leave the selected host.""" + # Parse the complete URL so host validation cannot be bypassed with user-info or path text. + parsed = urllib.parse.urlparse(value) + # Require an absolute HTTP endpoint because relative and non-HTTP targets are never valid APIs here. + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise ValueError("--base-url must be an absolute http(s) URL") + # Restrict this host-qualified harness to loopback so prompts cannot leak through a mistyped URL. + if parsed.hostname.lower() not in LOOPBACK_HOSTS: + raise ValueError("--base-url must target a loopback host: localhost, 127.0.0.1, or ::1") + # Remove a trailing slash once so every request path is assembled deterministically. + return value.rstrip("/") + + +@dataclass(frozen=True) +class StreamObservation: + """One visible SSE fragment and the monotonic time at which it arrived.""" + + offset_seconds: float + content: str + + +def nearest_rank_percentile(values: list[float], percentile: float) -> float | None: + """Return an observed tail value without interpolating an unmeasured result.""" + + if not values: + return None + ordered = sorted(values) + return ordered[max(0, int(len(ordered) * percentile + 0.999999999) - 1)] + + +def numeric_summary(values: list[float]) -> dict[str, float | None]: + """Report central and tail values while retaining the measured maximum.""" + + if not values: + return {key: None for key in ("mean", "median", "minimum", "maximum", "stdev", "p50", "p95", "p99")} + return { + "mean": statistics.mean(values), + "median": statistics.median(values), + "minimum": min(values), + "maximum": max(values), + "stdev": statistics.stdev(values) if len(values) > 1 else None, + "p50": nearest_rank_percentile(values, 0.50), + "p95": nearest_rank_percentile(values, 0.95), + "p99": nearest_rank_percentile(values, 0.99), + } + + +def load_tokenizer(path: Path) -> Any: + """Load the checkpoint tokenizer locally so generated-token counts are real.""" + + from transformers import AutoTokenizer + + return AutoTokenizer.from_pretrained(path, local_files_only=True, trust_remote_code=False) + + +def iter_sse_events(response: Any, started_at: float) -> Iterable[tuple[float, str]]: + """Yield every server-sent data payload with its receive timestamp.""" + + for raw_line in response: + offset = time.perf_counter() - started_at + line = raw_line.decode("utf-8", errors="strict").rstrip("\r\n") + if line.startswith("data:"): + yield offset, line[5:].lstrip() + + +def stream_completion(args: argparse.Namespace) -> tuple[list[StreamObservation], str, float, float, list[str], dict[str, Any] | None]: + """Issue one greedy fixed-length request without relying on remote source files.""" + + body = { + "model": args.model, + "messages": [{"role": "user", "content": args.prompt}], + "stream": True, + "stream_options": {"include_usage": True}, + "temperature": 0.0, + "top_p": 1.0, + "top_k": 1, + "max_tokens": args.max_tokens, + "reasoning_effort": "none", + "ignore_eos": True, + } + request = urllib.request.Request( + args.base_url.rstrip("/") + "/chat/completions", + data=json.dumps(body, separators=(",", ":")).encode("utf-8"), + headers={"Content-Type": "application/json", "Accept": "text/event-stream"}, + method="POST", + ) + started = time.perf_counter() + observations: list[StreamObservation] = [] + errors: list[str] = [] + usage: dict[str, Any] | None = None + completed = False + try: + with urllib.request.urlopen(request, timeout=args.timeout_seconds) as response: + for offset, event_data in iter_sse_events(response, started): + if event_data == "[DONE]": + completed = True + continue + try: + event = json.loads(event_data) + except json.JSONDecodeError as error: + errors.append(f"invalid JSON SSE event: {error}") + continue + if isinstance(event.get("error"), dict): + errors.append(f"server error event: {event['error']}") + if isinstance(event.get("usage"), dict): + usage = event["usage"] + for choice in event.get("choices", []): + delta = choice.get("delta", {}) + content = delta.get("content") # Score visible answer text only; reasoning is not output. + if content: + observations.append(StreamObservation(offset, str(content))) + except urllib.error.HTTPError as error: + errors.append(f"HTTP {error.code}: {error.read().decode('utf-8', errors='replace')}") + except urllib.error.URLError as error: + errors.append(f"transport failure: {error}") + finished = time.perf_counter() + if not completed: + errors.append("stream ended without [DONE]") + return observations, "".join(item.content for item in observations), started, finished, errors, usage + + +def parse_args(argv: list[str]) -> argparse.Namespace: + """Parse fixed workload, concurrency, and immutable artifact inputs.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:1919/v1") + parser.add_argument("--model", required=True) + parser.add_argument("--tokenizer", required=True, type=Path) + parser.add_argument("--artifact", required=True, type=Path) + parser.add_argument("--expected-host", required=True, help="Expected hostname of the explicitly selected test machine") + parser.add_argument("--concurrency", required=True, type=int) + parser.add_argument("--rounds", type=int, default=3) + parser.add_argument("--max-tokens", type=int, default=256) + parser.add_argument("--prompt", default=DEFAULT_PROMPT) + parser.add_argument("--timeout-seconds", type=float, default=300.0) + args = parser.parse_args(argv) + # Validate the destination before any benchmark request can disclose prompt content. + try: + # Normalize one approved loopback endpoint for deterministic request assembly. + args.base_url = require_loopback_url(args.base_url) + except ValueError as error: + # Route validation through argparse so callers receive a standard configuration failure. + parser.error(str(error)) + if args.concurrency < 1: + parser.error("--concurrency must be positive") + if args.rounds < 1: + parser.error("--rounds must be positive") + if args.max_tokens < 2: + parser.error("--max-tokens must be at least two for TPS") + if args.timeout_seconds <= 0: + parser.error("--timeout-seconds must be positive") + return args + + +def require_expected_host(expected_host: str) -> str: + """Fail closed to keep concurrency traffic on the declared GMKtek EVO-X2 host.""" + + actual_host = socket.gethostname().lower() + expected_short = expected_host.lower().split(".", 1)[0] + if actual_host not in {expected_host.lower(), expected_short}: + raise RuntimeError(f"refusing concurrent control on {actual_host!r}; expected {expected_host!r}") + return actual_host + + +def request_args(args: argparse.Namespace) -> argparse.Namespace: + """Build the compatible greedy throughput request consumed by shared code.""" + + return argparse.Namespace( + base_url=args.base_url, + model=args.model, + prompt=args.prompt, + max_tokens=args.max_tokens, + timeout_seconds=args.timeout_seconds, + reasoning_effort="none", + mode="throughput", + expected_text="", + ) + + +def run_round(args: argparse.Namespace, tokenizer: Any, round_index: int) -> dict[str, Any]: + """Release one synchronized request group and retain every request result.""" + + barrier = threading.Barrier(args.concurrency) + workload_args = request_args(args) + suite_started = time.perf_counter() + + def one_request(request_index: int) -> dict[str, Any]: + """Wait for the group, then record one independent streamed completion.""" + + barrier.wait(timeout=30.0) + observations, text, started, finished, errors, usage = stream_completion(workload_args) + generated_tokens = len(tokenizer.encode(text, add_special_tokens=False)) + ttft = observations[0].offset_seconds if observations else None + last = observations[-1].offset_seconds if observations else None + decode_seconds = last - ttft if ttft is not None and last is not None else None + decode_tps = ( + (generated_tokens - 1) / decode_seconds + if generated_tokens > 1 and decode_seconds is not None and decode_seconds > 0 + else None + ) + gaps = [ + observations[index].offset_seconds - observations[index - 1].offset_seconds + for index in range(1, len(observations)) + ] + if not observations: + errors.append("stream contained no content events") + if decode_tps is None: + errors.append("fewer than two generated tokens or no positive decode interval") + return { + "request_index": request_index, + "started_offset_seconds": started - suite_started, + "finished_offset_seconds": finished - suite_started, + "wall_seconds": finished - started, + "ttft_seconds": ttft, + "decode_seconds": decode_seconds, + "decode_tps": decode_tps, + "generated_tokens": generated_tokens, + "usage": usage, + "response_text": text, + "content_events": [ + {"offset_seconds": item.offset_seconds, "content": item.content} + for item in observations + ], + "content_event_gap_seconds": gaps, + "content_event_gap_summary_seconds": numeric_summary(gaps), + "errors": errors, + "status": "passed" if not errors else "failed", + } + + with ThreadPoolExecutor(max_workers=args.concurrency, thread_name_prefix="gmk_evo_x2-load") as executor: + requests = list(executor.map(one_request, range(1, args.concurrency + 1))) + suite_finished = time.perf_counter() + successful = [request for request in requests if request["status"] == "passed"] + first_start = min((request["started_offset_seconds"] for request in requests), default=None) + last_finish = max((request["finished_offset_seconds"] for request in requests), default=None) + span = last_finish - first_start if first_start is not None and last_finish is not None else None + aggregate_tokens = sum(request["generated_tokens"] for request in successful) + return { + "round_index": round_index, + "started_epoch_seconds": suite_started, + "wall_seconds": suite_finished - suite_started, + "requests": requests, + "summary": { + "successful_requests": len(successful), + "requested_requests": args.concurrency, + "aggregate_generated_tokens": aggregate_tokens, + "aggregate_tps": aggregate_tokens / span if span and span > 0 else None, + "decode_tps": numeric_summary([request["decode_tps"] for request in successful if request["decode_tps"] is not None]), + "ttft_seconds": numeric_summary([request["ttft_seconds"] for request in successful if request["ttft_seconds"] is not None]), + "content_event_gap_seconds": numeric_summary([gap for request in successful for gap in request["content_event_gap_seconds"]]), + }, + "status": "passed" if len(successful) == args.concurrency else "failed", + } + + +def main(argv: list[str] | None = None) -> int: + """Write the complete concurrent evidence package and propagate failures.""" + + args = parse_args(sys.argv[1:] if argv is None else argv) + host = require_expected_host(args.expected_host) + if args.artifact.exists(): + raise FileExistsError(f"refusing to overwrite existing artifact: {args.artifact}") + tokenizer = load_tokenizer(args.tokenizer) + rounds = [run_round(args, tokenizer, index) for index in range(1, args.rounds + 1)] + aggregate_tps = [item["summary"]["aggregate_tps"] for item in rounds if item["summary"]["aggregate_tps"] is not None] + all_ttft = [request["ttft_seconds"] for item in rounds for request in item["requests"] if request["ttft_seconds"] is not None] + all_gaps = [gap for item in rounds for request in item["requests"] for gap in request["content_event_gap_seconds"]] + artifact = { + "schema_version": 2, + "classification": "GMKtek EVO-X2 concurrent API control, not paper replication", + "host": host, + "request": { + "base_url": args.base_url, + "model": args.model, + "concurrency": args.concurrency, + "rounds": args.rounds, + "max_tokens": args.max_tokens, + "prompt": args.prompt, + "reasoning_effort": "none", + "temperature": 0.0, + "top_p": 1.0, + "top_k": 1, + "ignore_eos": True, + }, + "rounds": rounds, + "summary": { + "successful_rounds": sum(item["status"] == "passed" for item in rounds), + "requested_rounds": args.rounds, + "aggregate_tps": numeric_summary(aggregate_tps), + "ttft_seconds": numeric_summary(all_ttft), + "content_event_gap_seconds": numeric_summary(all_gaps), + "p99_ttft_seconds": nearest_rank_percentile(all_ttft, 0.99), + "p99_content_event_gap_seconds": nearest_rank_percentile(all_gaps, 0.99), + }, + "status": "passed" if all(item["status"] == "passed" for item in rounds) else "failed", + } + args.artifact.parent.mkdir(parents=True, exist_ok=True) + args.artifact.write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"status": artifact["status"], "summary": artifact["summary"]}, sort_keys=True)) + return 0 if artifact["status"] == "passed" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/gmk_evo_x2/run_long_context_control.py b/benchmarks/gmk_evo_x2/run_long_context_control.py new file mode 100644 index 0000000000..ccb1dda76c --- /dev/null +++ b/benchmarks/gmk_evo_x2/run_long_context_control.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Measure deterministic long-context retrieval on the isolated GMKtek EVO-X2 API. + +This tool deliberately covers the context range exposed by the running Qwen +server. It is a GMKtek EVO-X2 control, not a replication of the FreeToken paper's +much longer agent sessions. It places an exact marker at the start of a +deterministic prompt, asks the model to retrieve only that marker, records +every visible SSE event and refuses to overwrite an existing artifact. +""" + +from __future__ import annotations + +import argparse +import json +import socket +import statistics +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + + +LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1"} + + +def require_loopback_url(value: str) -> str: + """Reject non-loopback targets before benchmark prompts can leave the selected host.""" + # Parse the complete URL so host validation cannot be bypassed with user-info or path text. + parsed = urllib.parse.urlparse(value) + # Require an absolute HTTP endpoint because relative and non-HTTP targets are never valid APIs here. + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise ValueError("--base-url must be an absolute http(s) URL") + # Restrict this host-qualified harness to loopback so prompts cannot leak through a mistyped URL. + if parsed.hostname.lower() not in LOOPBACK_HOSTS: + raise ValueError("--base-url must target a loopback host: localhost, 127.0.0.1, or ::1") + # Remove a trailing slash once so every request path is assembled deterministically. + return value.rstrip("/") + + +MARKER = "azure-17" +FILLER = ( + "This is deterministic filler for a context-retrieval control. " + "Read it without changing the protected marker. " +) + + +def nearest_rank(values: list[float], percentile: float) -> float | None: + """Return an observed percentile so short streams do not invent values.""" + + if not values: + return None + ordered = sorted(values) + return ordered[max(0, int(len(ordered) * percentile + 0.999999999) - 1)] + + +def build_prompt( + filler_repetitions: int, marker: str = MARKER, prefix_nonce: str | None = None +) -> str: + """Build a retrieval prompt whose optional early nonce defeats prefix reuse. + + A nonce placed before the long filler means a radix or prefix cache cannot + reuse the expensive common prefix from an earlier sample. The protected + answer remains at the prompt beginning and therefore still tests retrieval. + """ + + if filler_repetitions < 1: + raise ValueError("filler repetitions must be positive") + nonce_line = f"Per-sample prefix nonce: {prefix_nonce}\n" if prefix_nonce else "" + return ( + "Protected marker: " + marker + "\n" + "Do not repeat or transform the marker while reading this material.\n\n" + + nonce_line + + FILLER * filler_repetitions + + "\n\nReply with only the protected marker and no other text." + ) + + +def parse_args(argv: list[str]) -> argparse.Namespace: + """Parse all material workload controls explicitly for a reproducible run.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:1919/v1") + parser.add_argument("--model", required=True) + parser.add_argument("--artifact", required=True, type=Path) + parser.add_argument("--expected-host", required=True, help="Expected hostname of the explicitly selected test machine") + parser.add_argument("--filler-repetitions", type=int, required=True) + parser.add_argument( + "--sample-variation", + choices=("none", "prefix_nonce"), + default="none", + help="Use prefix_nonce to prevent later samples from reusing the full prompt cache.", + ) + parser.add_argument("--samples", type=int, default=5) + parser.add_argument("--max-tokens", type=int, default=16) + parser.add_argument("--timeout-seconds", type=float, default=300.0) + args = parser.parse_args(argv) + # Validate the destination before any benchmark request can disclose prompt content. + try: + # Normalize one approved loopback endpoint for deterministic request assembly. + args.base_url = require_loopback_url(args.base_url) + except ValueError as error: + # Route validation through argparse so callers receive a standard configuration failure. + parser.error(str(error)) + if args.samples < 1: + parser.error("--samples must be positive") + if args.max_tokens < 1: + parser.error("--max-tokens must be positive") + if args.filler_repetitions < 1: + parser.error("--filler-repetitions must be positive") + return args + + +def require_expected_host(expected_host: str) -> str: + """Fail closed so this load never accidentally reaches a different host.""" + + actual_host = socket.gethostname().lower() + expected_short = expected_host.lower().split(".", 1)[0] + if actual_host not in {expected_host.lower(), expected_short}: + raise RuntimeError(f"refusing long-context control on {actual_host!r}; expected {expected_host!r}") + return actual_host + + +def stream_sample(args: argparse.Namespace, prompt: str) -> dict[str, Any]: + """Send one greedy streaming request and retain visible output timing.""" + + body = { + "model": args.model, + "messages": [{"role": "user", "content": prompt}], + "stream": True, + "stream_options": {"include_usage": True}, + "temperature": 0.0, + "top_p": 1.0, + "top_k": 1, + "max_tokens": args.max_tokens, + "reasoning_effort": "none", + } + request = urllib.request.Request( + args.base_url.rstrip("/") + "/chat/completions", + data=json.dumps(body, separators=(",", ":")).encode("utf-8"), + headers={"Content-Type": "application/json", "Accept": "text/event-stream"}, + method="POST", + ) + started = time.perf_counter() + events: list[dict[str, Any]] = [] + raw_sse_events: list[dict[str, Any]] = [] + errors: list[str] = [] + usage: dict[str, Any] | None = None + done = False + try: + with urllib.request.urlopen(request, timeout=args.timeout_seconds) as response: + for raw_line in response: + offset = time.perf_counter() - started + line = raw_line.decode("utf-8", errors="strict").rstrip("\r\n") + if not line.startswith("data:"): + continue + payload = line[5:].lstrip() + if payload == "[DONE]": + done = True + continue + try: + event = json.loads(payload) + except json.JSONDecodeError as error: + errors.append(f"invalid JSON SSE event: {error}") + continue + raw_sse_events.append({"offset_seconds": offset, "event": event}) + if isinstance(event.get("error"), dict): + errors.append(f"server error event: {event['error']}") + if isinstance(event.get("usage"), dict): + usage = event["usage"] + for choice in event.get("choices", []): + content = choice.get("delta", {}).get("content") + if content: + events.append({"offset_seconds": offset, "content": str(content)}) + except urllib.error.HTTPError as error: + errors.append(f"HTTP {error.code}: {error.read().decode('utf-8', errors='replace')}") + except urllib.error.URLError as error: + errors.append(f"transport failure: {error}") + if not done: + errors.append("stream ended without [DONE]") + if not events: + errors.append("stream contained no visible content events") + gaps = [events[index]["offset_seconds"] - events[index - 1]["offset_seconds"] for index in range(1, len(events))] + text = "".join(event["content"] for event in events) + return { + "text": text, + "events": events, + "raw_sse_events": raw_sse_events, + "usage": usage, + "errors": errors, + "ttft_seconds": events[0]["offset_seconds"] if events else None, + "content_event_gap_seconds": gaps, + "quality_passed": text.strip() == MARKER and not errors, + } + + +def main(argv: list[str] | None = None) -> int: + """Run immutable samples, summarize tails, and exit nonzero on any failure.""" + + args = parse_args(sys.argv[1:] if argv is None else argv) + host = require_expected_host(args.expected_host) + if args.artifact.exists(): + raise FileExistsError(f"refusing to overwrite existing artifact: {args.artifact}") + prompts = [ + build_prompt( + args.filler_repetitions, + prefix_nonce=(f"long-context-sample-{index + 1}" if args.sample_variation == "prefix_nonce" else None), + ) + for index in range(args.samples) + ] + samples = [] + for prompt in prompts: + sample = stream_sample(args, prompt) + sample["prompt"] = prompt + sample["prompt_character_count"] = len(prompt) + samples.append(sample) + ttft = [sample["ttft_seconds"] for sample in samples if sample["ttft_seconds"] is not None] + gaps = [gap for sample in samples for gap in sample["content_event_gap_seconds"]] + prompt_token_counts = [sample["usage"].get("prompt_tokens") for sample in samples if sample["usage"]] + artifact = { + "schema_version": 2, + "host": host, + "classification": "GMKtek EVO-X2 long-context control, not paper replication", + "request": { + "base_url": args.base_url, + "model": args.model, + "filler_repetitions": args.filler_repetitions, + "max_tokens": args.max_tokens, + "samples": args.samples, + "sample_variation": args.sample_variation, + "temperature": 0.0, + "reasoning_effort": "none", + }, + "prompt": { + "marker": MARKER, + "variation": args.sample_variation, + "representative_character_count": len(prompts[0]), + "representative_text": prompts[0], + }, + "samples": samples, + "summary": { + "sample_count": len(samples), + "passed_samples": sum(sample["quality_passed"] for sample in samples), + "prompt_tokens_reported": prompt_token_counts, + "ttft_seconds": { + "mean": statistics.mean(ttft) if ttft else None, + "p50": nearest_rank(ttft, 0.50), + "p95": nearest_rank(ttft, 0.95), + "p99": nearest_rank(ttft, 0.99), + "max": max(ttft) if ttft else None, + }, + "content_event_gap_seconds": { + "p50": nearest_rank(gaps, 0.50), + "p95": nearest_rank(gaps, 0.95), + "p99": nearest_rank(gaps, 0.99), + "max": max(gaps) if gaps else None, + }, + }, + "status": "passed" if all(sample["quality_passed"] for sample in samples) else "failed", + } + args.artifact.parent.mkdir(parents=True, exist_ok=True) + args.artifact.write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"status": artifact["status"], "summary": artifact["summary"]}, sort_keys=True)) + return 0 if artifact["status"] == "passed" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/gmk_evo_x2/run_multiturn_state_suite.py b/benchmarks/gmk_evo_x2/run_multiturn_state_suite.py new file mode 100644 index 0000000000..75cf24e8ce --- /dev/null +++ b/benchmarks/gmk_evo_x2/run_multiturn_state_suite.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Measure a deterministic GMKtek EVO-X2 multi-turn state-retention control. + +This is a bounded intermediate workload between single prompts and the +FreeToken paper's tool-using agents. Each turn receives the full prior visible +conversation, records raw SSE timing, and must produce its exact expected +visible answer. It never starts or stops the server. +""" + +from __future__ import annotations + +import argparse +import json +import socket +import statistics +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + + +def parse_args(argv: list[str]) -> argparse.Namespace: + """Parse explicit workload and server inputs for one immutable artifact.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:1919/v1") + parser.add_argument("--model", required=True) + parser.add_argument("--artifact", required=True, type=Path) + parser.add_argument( + "--suite", + default=Path(__file__).with_name("multiturn_state_suite.json"), + type=Path, + ) + parser.add_argument("--expected-host", required=True, help="Expected hostname of the explicitly selected test machine") + parser.add_argument("--max-tokens", type=int, default=64) + parser.add_argument("--timeout-seconds", type=float, default=180.0) + args = parser.parse_args(argv) + if args.max_tokens < 1: + parser.error("--max-tokens must be positive") + return args + + +def require_expected_host(expected_host: str) -> str: + """Fail closed so a local control cannot accidentally target another host.""" + + actual_host = socket.gethostname().lower() + if actual_host not in {expected_host.lower(), expected_host.lower().split(".", 1)[0]}: + raise RuntimeError(f"refusing multi-turn suite on {actual_host!r}; expected {expected_host!r}") + return actual_host + + +def nearest_rank(values: list[float], percentile: float) -> float | None: + """Return an observed nearest-rank tail statistic for a short stream.""" + + if not values: + return None + ordered = sorted(values) + return ordered[max(0, int(len(ordered) * percentile + 0.999999999) - 1)] + + +def stream_turn(args: argparse.Namespace, messages: list[dict[str, str]]) -> dict[str, Any]: + """Send one deterministic chat turn and preserve only visible output events.""" + + body = { + "model": args.model, + "messages": messages, + "stream": True, + "stream_options": {"include_usage": True}, + "temperature": 0.0, + "top_p": 1.0, + "top_k": 1, + "max_tokens": args.max_tokens, + "reasoning_effort": "none", + } + request = urllib.request.Request( + args.base_url.rstrip("/") + "/chat/completions", + data=json.dumps(body, separators=(",", ":")).encode("utf-8"), + headers={"Content-Type": "application/json", "Accept": "text/event-stream"}, + method="POST", + ) + started = time.perf_counter() + events: list[dict[str, Any]] = [] + errors: list[str] = [] + usage: dict[str, Any] | None = None + done = False + try: + with urllib.request.urlopen(request, timeout=args.timeout_seconds) as response: + for raw_line in response: + offset = time.perf_counter() - started + line = raw_line.decode("utf-8", errors="strict").rstrip("\r\n") + if not line.startswith("data:"): + continue + payload = line[5:].lstrip() + if payload == "[DONE]": + done = True + continue + try: + event = json.loads(payload) + except json.JSONDecodeError as error: + errors.append(f"invalid JSON SSE event: {error}") + continue + if isinstance(event.get("usage"), dict): + usage = event["usage"] + for choice in event.get("choices", []): + content = choice.get("delta", {}).get("content") + if content: + events.append({"offset_seconds": offset, "content": str(content)}) + except urllib.error.HTTPError as error: + errors.append(f"HTTP {error.code}: {error.read().decode('utf-8', errors='replace')}") + except urllib.error.URLError as error: + errors.append(f"transport failure: {error}") + if not done: + errors.append("stream ended without [DONE]") + if not events: + errors.append("stream contained no visible content events") + gaps = [ + events[index]["offset_seconds"] - events[index - 1]["offset_seconds"] + for index in range(1, len(events)) + ] + return { + "text": "".join(event["content"] for event in events), + "events": events, + "usage": usage, + "errors": errors, + "ttft_seconds": events[0]["offset_seconds"] if events else None, + "token_gap_seconds": gaps, + } + + +def main(argv: list[str] | None = None) -> int: + """Execute all turns, retain the complete conversation, and score exact output.""" + + args = parse_args(sys.argv[1:] if argv is None else argv) + host = require_expected_host(args.expected_host) + if args.artifact.exists(): + raise FileExistsError(f"refusing to overwrite existing artifact: {args.artifact}") + suite = json.loads(args.suite.read_text(encoding="utf-8")) + turns = suite.get("turns") + if not isinstance(turns, list) or not turns: + raise ValueError("suite must contain a non-empty turns list") + messages: list[dict[str, str]] = [] + results: list[dict[str, Any]] = [] + for turn in turns: + if not isinstance(turn, dict) or not isinstance(turn.get("user"), str) or not isinstance(turn.get("expected"), str): + raise ValueError("every turn requires string user and expected values") + messages.append({"role": "user", "content": turn["user"]}) + response = stream_turn(args, messages) + passed = response["text"].strip() == turn["expected"] and not response["errors"] + results.append({ + "id": turn.get("id"), + "input_messages": list(messages), + "expected": turn["expected"], + "response": response, + "status": "passed" if passed else "failed", + }) + messages.append({"role": "assistant", "content": response["text"]}) + ttft = [item["response"]["ttft_seconds"] for item in results if item["response"]["ttft_seconds"] is not None] + gaps = [gap for item in results for gap in item["response"]["token_gap_seconds"]] + artifact = { + "schema_version": 1, + "host": host, + "suite": str(args.suite.resolve()), + "request": {"base_url": args.base_url, "model": args.model, "max_tokens": args.max_tokens}, + "results": results, + "tail_metrics": { + "turn_count": len(results), + "max_ttft_seconds": max(ttft) if ttft else None, + "mean_ttft_seconds": statistics.mean(ttft) if ttft else None, + "p95_ttft_seconds": nearest_rank(ttft, 0.95), + "p99_token_gap_seconds": nearest_rank(gaps, 0.99), + "max_token_gap_seconds": max(gaps) if gaps else None, + }, + "status": "passed" if all(item["status"] == "passed" for item in results) else "failed", + } + args.artifact.parent.mkdir(parents=True, exist_ok=True) + args.artifact.write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"status": artifact["status"], "turns": len(results), "tail_metrics": artifact["tail_metrics"]}, sort_keys=True)) + return 0 if artifact["status"] == "passed" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/gmk_evo_x2/run_quality_suite.py b/benchmarks/gmk_evo_x2/run_quality_suite.py new file mode 100644 index 0000000000..31b4428a41 --- /dev/null +++ b/benchmarks/gmk_evo_x2/run_quality_suite.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Run a small, versioned quality suite against the GMKtek EVO-X2 Qwen API. + +The suite is intentionally separate from the paper's agent workloads. It +provides a repeatable precondition for local performance changes: every +candidate must preserve basic exact answers, structured JSON, and the visible +OpenAI response contract before its TPS is considered. +""" + +from __future__ import annotations + +import argparse +import json +import socket +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + + +def parse_args(argv: list[str]) -> argparse.Namespace: + """Read every external input explicitly for reproducible quality evidence.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:1919/v1") + parser.add_argument("--model", required=True) + parser.add_argument("--artifact", required=True, type=Path) + parser.add_argument( + "--suite", + default=Path(__file__).with_name("quality_suite.json"), + type=Path, + help="versioned JSON fixture defining prompts and deterministic checks", + ) + parser.add_argument("--expected-host", required=True, help="Expected hostname of the explicitly selected test machine") + parser.add_argument("--max-tokens", type=int, default=64) + parser.add_argument("--timeout-seconds", type=float, default=180.0) + args = parser.parse_args(argv) + if args.max_tokens < 1: + parser.error("--max-tokens must be positive") + if args.timeout_seconds <= 0: + parser.error("--timeout-seconds must be positive") + return args + + +def require_expected_host(expected_host: str) -> str: + """Refuse any accidental quality traffic directed from another LAN host.""" + + actual_host = socket.gethostname().lower() + accepted = {expected_host.lower(), expected_host.lower().split(".", 1)[0]} + if actual_host not in accepted: + raise RuntimeError( + f"refusing quality suite on host {actual_host!r}; expected {expected_host!r}" + ) + return actual_host + + +def request_visible_text(args: argparse.Namespace, prompt: str) -> dict[str, Any]: + """Stream one greedy chat response and preserve content-bearing SSE events.""" + + request_body = { + "model": args.model, + "messages": [{"role": "user", "content": prompt}], + "stream": True, + "stream_options": {"include_usage": True}, + "temperature": 0.0, + "top_p": 1.0, + "top_k": 1, + "max_tokens": args.max_tokens, + "reasoning_effort": "none", + } + request = urllib.request.Request( + args.base_url.rstrip("/") + "/chat/completions", + data=json.dumps(request_body, separators=(",", ":")).encode("utf-8"), + headers={"Content-Type": "application/json", "Accept": "text/event-stream"}, + method="POST", + ) + started = time.perf_counter() + events: list[dict[str, Any]] = [] + errors: list[str] = [] + usage: dict[str, Any] | None = None + completed = False + try: + with urllib.request.urlopen(request, timeout=args.timeout_seconds) as response: + for raw_line in response: + offset = time.perf_counter() - started + line = raw_line.decode("utf-8", errors="strict").rstrip("\r\n") + if not line.startswith("data:"): + continue + payload = line[5:].lstrip() + if payload == "[DONE]": + completed = True + continue + try: + event = json.loads(payload) + except json.JSONDecodeError as error: + errors.append(f"invalid JSON SSE event: {error}") + continue + if isinstance(event.get("usage"), dict): + usage = event["usage"] + for choice in event.get("choices", []): + delta = choice.get("delta", {}) + content = delta.get("content") + if content: + events.append({"offset_seconds": offset, "content": str(content)}) + except urllib.error.HTTPError as error: + errors.append(f"HTTP {error.code}: {error.read().decode('utf-8', errors='replace')}") + except urllib.error.URLError as error: + errors.append(f"transport failure: {error}") + if not completed: + errors.append("stream ended without [DONE]") + if not events: + errors.append("stream contained no visible content events") + return { + "text": "".join(event["content"] for event in events), + "events": events, + "usage": usage, + "errors": errors, + } + + +def evaluate_check(text: str, check: dict[str, Any]) -> tuple[bool, str | None]: + """Evaluate one deterministic fixture rule without model-specific heuristics.""" + + kind = check.get("kind") + if kind == "exact": + expected = check.get("value") + passed = text.strip() == expected + return passed, None if passed else f"expected exactly {expected!r}, got {text.strip()!r}" + if kind == "json_fields": + try: + parsed = json.loads(text) + except json.JSONDecodeError as error: + return False, f"visible output is not valid JSON: {error}" + expected_fields = check.get("fields") + if not isinstance(parsed, dict) or not isinstance(expected_fields, dict): + return False, "fixture requires an object and an object field map" + mismatches = { + key: {"expected": value, "actual": parsed.get(key)} + for key, value in expected_fields.items() + if parsed.get(key) != value + } + return not mismatches, None if not mismatches else f"JSON field mismatch: {mismatches}" + return False, f"unsupported check kind: {kind!r}" + + +def main(argv: list[str] | None = None) -> int: + """Run every fixture, write one immutable artifact, and return its pass state.""" + + args = parse_args(sys.argv[1:] if argv is None else argv) + host = require_expected_host(args.expected_host) + if args.artifact.exists(): + raise FileExistsError(f"refusing to overwrite existing artifact: {args.artifact}") + suite = json.loads(args.suite.read_text(encoding="utf-8")) + cases = suite.get("cases") + if not isinstance(cases, list) or not cases: + raise ValueError("suite must contain at least one case") + results = [] + for case in cases: + if not isinstance(case, dict) or not isinstance(case.get("prompt"), str): + raise ValueError("every suite case requires a string prompt") + response = request_visible_text(args, case["prompt"]) + check = case.get("check") + if not isinstance(check, dict): + raise ValueError(f"case {case.get('id')!r} requires a check object") + passed, check_error = evaluate_check(response["text"], check) + results.append({ + "id": case.get("id"), + "prompt": case["prompt"], + "check": check, + "response": response, + "check_error": check_error, + "status": "passed" if passed and not response["errors"] else "failed", + }) + artifact = { + "schema_version": 1, + "host": host, + "request": { + "base_url": args.base_url, + "model": args.model, + "max_tokens": args.max_tokens, + "temperature": 0.0, + "top_p": 1.0, + "top_k": 1, + "reasoning_effort": "none", + }, + "suite": str(args.suite.resolve()), + "results": results, + "status": "passed" if all(item["status"] == "passed" for item in results) else "failed", + } + args.artifact.parent.mkdir(parents=True, exist_ok=True) + args.artifact.write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"status": artifact["status"], "cases": len(results)}, sort_keys=True)) + return 0 if artifact["status"] == "passed" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/gmk_evo_x2/summarize_qwen_gguf_endurance.py b/benchmarks/gmk_evo_x2/summarize_qwen_gguf_endurance.py new file mode 100644 index 0000000000..36556dd8ce --- /dev/null +++ b/benchmarks/gmk_evo_x2/summarize_qwen_gguf_endurance.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Validate and summarize a retained GMKtek EVO-X2 Qwen GGUF endurance artifact. + +The endurance wrapper stores one JSON result and one process-scoped memory +sample for each deterministic multi-turn conversation. This program turns +those raw files into a single machine-readable conclusion without treating +unrelated whole-host swap as evidence that the model process was swapped. +""" + +from __future__ import annotations + +import argparse +import json +import math +import re +from pathlib import Path +from statistics import mean +from typing import Any, Iterable + + +# Accept only the session naming contract emitted by the endurance shell driver. +SESSION_NAME = re.compile(r"session-(\d+)\.json$") +# Read the two swap fields deliberately rather than parsing unrelated telemetry. +SWAP_FIELD = re.compile(r"^(runner_swap_kib|whole_host_swap_kib)=(\d+)$", re.M) + + +def percentile(values: Iterable[float], fraction: float) -> float: + """Return a nearest-rank percentile for a non-empty numeric collection.""" + + ordered = sorted(values) + if not ordered: + raise ValueError("cannot calculate a percentile of an empty collection") + rank = max(1, math.ceil(fraction * len(ordered))) + return ordered[rank - 1] + + +def read_swap_fields(path: Path) -> dict[str, int]: + """Extract the explicitly recorded per-runner and whole-host swap values.""" + + values = {name: int(value) for name, value in SWAP_FIELD.findall(path.read_text())} + if set(values) != {"runner_swap_kib", "whole_host_swap_kib"}: + raise ValueError(f"missing swap fields in {path}") + return values + + +def session_number(path: Path) -> int: + """Return the numeric session label, rejecting unrelated JSON files.""" + + match = SESSION_NAME.search(path.name) + if not match: + raise ValueError(f"unexpected session filename: {path.name}") + return int(match.group(1)) + + +def summarize(artifact_root: Path, expected_sessions: int) -> dict[str, Any]: + """Validate every session and return portable summary metrics and failures.""" + + sessions_dir = artifact_root / "sessions" + session_paths = sorted(sessions_dir.glob("session-*.json"), key=session_number) + failures: list[str] = [] + ttfts: list[float] = [] + gaps: list[float] = [] + runner_swaps: list[int] = [] + host_swaps: list[int] = [] + + if len(session_paths) != expected_sessions: + failures.append(f"expected {expected_sessions} sessions, found {len(session_paths)}") + + for session_path in session_paths: + session = session_number(session_path) + payload = json.loads(session_path.read_text()) + if payload.get("status") != "passed": + failures.append(f"session {session:02d} status={payload.get('status')!r}") + for turn in payload.get("results", []): + if turn.get("status") != "passed": + failures.append( + f"session {session:02d} turn {turn.get('id', '')} " + f"status={turn.get('status')!r}" + ) + tail = payload.get("tail_metrics", {}) + try: + ttfts.append(float(tail["max_ttft_seconds"])) + gaps.append(float(tail["max_token_gap_seconds"])) + except (KeyError, TypeError, ValueError) as error: + failures.append(f"session {session:02d} missing tail metric: {error}") + + telemetry = sessions_dir / f"session-{session:02d}-telemetry.txt" + if not telemetry.is_file(): + failures.append(f"session {session:02d} missing telemetry") + continue + try: + swap = read_swap_fields(telemetry) + except ValueError as error: + failures.append(str(error)) + continue + runner_swaps.append(swap["runner_swap_kib"]) + host_swaps.append(swap["whole_host_swap_kib"]) + if swap["runner_swap_kib"] != 0: + failures.append( + f"session {session:02d} runner swap={swap['runner_swap_kib']} KiB" + ) + + return { + "schema_version": 1, + "artifact_root": str(artifact_root), + "expected_sessions": expected_sessions, + "observed_sessions": len(session_paths), + "passed": not failures, + "failures": failures, + "max_turn_ttft_seconds": { + "mean": mean(ttfts) if ttfts else None, + "p95": percentile(ttfts, 0.95) if ttfts else None, + "p99": percentile(ttfts, 0.99) if ttfts else None, + "max": max(ttfts) if ttfts else None, + }, + "max_visible_token_gap_seconds": { + "mean": mean(gaps) if gaps else None, + "p95": percentile(gaps, 0.95) if gaps else None, + "p99": percentile(gaps, 0.99) if gaps else None, + "max": max(gaps) if gaps else None, + }, + "runner_swap_kib": { + "min": min(runner_swaps) if runner_swaps else None, + "max": max(runner_swaps) if runner_swaps else None, + }, + "whole_host_swap_kib": { + "min": min(host_swaps) if host_swaps else None, + "max": max(host_swaps) if host_swaps else None, + }, + } + + +def main() -> int: + """Parse arguments, write the summary, and use exit status as the gate.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifact_root", type=Path) + parser.add_argument("--expected-sessions", type=int, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + summary = summarize(args.artifact_root, args.expected_sessions) + args.output.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 if summary["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/reproduce/run_local_api_benchmark.py b/benchmarks/reproduce/run_local_api_benchmark.py new file mode 100644 index 0000000000..9813b7d2c8 --- /dev/null +++ b/benchmarks/reproduce/run_local_api_benchmark.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Benchmark one already-running local OpenAI-compatible server reproducibly. + +The client is loopback-only by design. It never starts, stops, or reconfigures a +server. It writes immutable per-request JSON evidence, counts generated text +with the supplied checkpoint tokenizer, and requires an explicit visible-text +quality expectation in quality mode. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + +from benchmarks.gmk_evo_x2.run_api_benchmark import ( + StreamObservation, + iter_sse_events, + load_tokenizer, + numeric_summary, + write_json, +) + + +LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1"} + + +def require_loopback_url(value: str) -> str: + """Reject non-loopback targets before this client can send a request.""" + + parsed = urllib.parse.urlparse(value) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + raise ValueError("--base-url must be an absolute http(s) URL") + if parsed.hostname.lower() not in LOOPBACK_HOSTS: + raise ValueError("--base-url must target a loopback host: localhost, 127.0.0.1, or ::1") + return value.rstrip("/") + + +def parse_args(argv: list[str]) -> argparse.Namespace: + """Require every request and measurement choice to be explicit.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:8000/v1") + parser.add_argument("--model", required=True) + parser.add_argument("--tokenizer", required=True, type=Path) + parser.add_argument("--artifact-dir", required=True, type=Path) + prompt = parser.add_mutually_exclusive_group(required=True) + prompt.add_argument("--prompt") + prompt.add_argument("--prompt-file", type=Path) + parser.add_argument("--expected-text", default="") + parser.add_argument("--mode", choices=("quality", "throughput"), default="quality") + parser.add_argument("--samples", type=int, default=5) + parser.add_argument("--max-tokens", type=int, default=128) + parser.add_argument("--reasoning-effort", default="none") + parser.add_argument("--timeout-seconds", type=float, default=180.0) + parser.add_argument("--warmup", action="store_true") + args = parser.parse_args(argv) + try: + args.base_url = require_loopback_url(args.base_url) + except ValueError as error: + parser.error(str(error)) + if args.prompt_file: + try: + args.prompt = args.prompt_file.read_text(encoding="utf-8") + except OSError as error: + parser.error(f"cannot read --prompt-file: {error}") + if not args.prompt: + parser.error("prompt text must not be empty") + if args.samples < 1: + parser.error("--samples must be at least one") + if args.max_tokens < 1: + parser.error("--max-tokens must be positive") + if args.timeout_seconds <= 0: + parser.error("--timeout-seconds must be positive") + if args.mode == "quality" and not args.expected_text: + parser.error("quality mode requires --expected-text") + if args.mode == "throughput" and args.max_tokens < 2: + parser.error("throughput mode needs --max-tokens of at least two") + return args + + +def stream_request(args: argparse.Namespace) -> tuple[list[StreamObservation], str, float, list[str], dict[str, Any] | None]: + """Send one fixed greedy request and preserve protocol failures.""" + + body: dict[str, Any] = { + "model": args.model, + "messages": [{"role": "user", "content": args.prompt}], + "stream": True, + "stream_options": {"include_usage": True}, + "temperature": 0.0, + "top_p": 1.0, + "top_k": 1, + "max_tokens": args.max_tokens, + "reasoning_effort": args.reasoning_effort, + } + if args.mode == "throughput": + body["ignore_eos"] = True + request = urllib.request.Request( + args.base_url + "/chat/completions", + data=json.dumps(body, separators=(",", ":")).encode("utf-8"), + headers={"Content-Type": "application/json", "Accept": "text/event-stream"}, + method="POST", + ) + started_at = time.perf_counter() + observations: list[StreamObservation] = [] + errors: list[str] = [] + usage: dict[str, Any] | None = None + completed = False + try: + with urllib.request.urlopen(request, timeout=args.timeout_seconds) as response: + for offset, event_data in iter_sse_events(response, started_at): + if event_data == "[DONE]": + completed = True + continue + try: + event = json.loads(event_data) + except json.JSONDecodeError as error: + errors.append(f"invalid JSON SSE event: {error}") + continue + if not event.get("choices"): + if isinstance(event.get("usage"), dict): + usage = event["usage"] + continue + delta = event["choices"][0].get("delta", {}) + content = delta.get("content") # Score visible answer text only; reasoning is not output. + if content: + observations.append(StreamObservation(offset, str(content))) + except urllib.error.HTTPError as error: + raise RuntimeError(f"HTTP {error.code}: {error.read().decode('utf-8', errors='replace')}") from error + except urllib.error.URLError as error: + raise RuntimeError(f"request transport failure: {error}") from error + if not completed: + errors.append("stream ended without [DONE]") + if not observations: + errors.append("stream contained no content events") + return observations, "".join(item.content for item in observations), started_at, errors, usage + + +def run_sample(args: argparse.Namespace, tokenizer: Any, sample_index: int) -> dict[str, Any]: + """Produce one self-contained artifact with client timing and quality state.""" + + observations, text, started_at, errors, usage = stream_request(args) + finished_at = time.perf_counter() + generated_tokens = len(tokenizer.encode(text, add_special_tokens=False)) + first = observations[0].offset_seconds if observations else None + last = observations[-1].offset_seconds if observations else None + decode_seconds = last - first if first is not None and last is not None else None + decode_tps = None + if generated_tokens > 1 and decode_seconds and decode_seconds > 0: + decode_tps = (generated_tokens - 1) / decode_seconds + if args.mode == "quality" and text.strip() != args.expected_text: + errors.append(f"visible-text mismatch: expected {args.expected_text!r}, got {text.strip()!r}") + if args.mode == "throughput" and decode_tps is None: + errors.append("throughput run produced fewer than two generated tokens") + gaps = [observations[index].offset_seconds - observations[index - 1].offset_seconds for index in range(1, len(observations))] + return { + "schema_version": 2, + "sample_index": sample_index, + "status": "passed" if not errors else "failed", + "request": { + "base_url": args.base_url, + "model": args.model, + "prompt": args.prompt, + "prompt_sha256": hashlib.sha256(args.prompt.encode("utf-8")).hexdigest(), + "mode": args.mode, + "expected_text": args.expected_text, + "max_tokens": args.max_tokens, + "temperature": 0.0, + "top_p": 1.0, + "top_k": 1, + "reasoning_effort": args.reasoning_effort, + "ignore_eos": args.mode == "throughput", + }, + "timing": { + "wall_seconds": finished_at - started_at, + "warm_ttft_seconds": first, + "decode_seconds": decode_seconds, + "decode_tps": decode_tps, + "content_event_gap_seconds": gaps, + "content_event_gap_summary_seconds": numeric_summary(gaps), + }, + "usage": usage, + "response": { + "text": text, + "generated_tokens": generated_tokens, + "content_events": [item.__dict__ for item in observations], + }, + "protocol_errors": errors, + } + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(sys.argv[1:] if argv is None else argv) + if args.artifact_dir.exists(): + raise RuntimeError(f"refusing to overwrite existing artifact directory: {args.artifact_dir}") + args.artifact_dir.mkdir(parents=True) + tokenizer = load_tokenizer(args.tokenizer) + write_json(args.artifact_dir / "manifest.json", { + "schema_version": 2, + "arguments": {name: str(value) if isinstance(value, Path) else value for name, value in vars(args).items()}, + "collection": "loopback-only client; does not start, stop, or configure a server", + }) + if args.warmup: + warmup = run_sample(args, tokenizer, 0) + write_json(args.artifact_dir / "warmup.json", warmup) + if warmup["status"] != "passed": + raise RuntimeError("warmup failed; inspect warmup.json before scored samples") + samples = [run_sample(args, tokenizer, index) for index in range(1, args.samples + 1)] + for sample in samples: + write_json(args.artifact_dir / f"sample-{sample['sample_index']:02d}.json", sample) + tps = [sample["timing"]["decode_tps"] for sample in samples if sample["status"] == "passed" and sample["timing"]["decode_tps"] is not None] + ttft = [sample["timing"]["warm_ttft_seconds"] for sample in samples if sample["status"] == "passed" and sample["timing"]["warm_ttft_seconds"] is not None] + write_json(args.artifact_dir / "summary.json", { + "schema_version": 2, + "requested_samples": args.samples, + "successful_samples": len([sample for sample in samples if sample["status"] == "passed"]), + "decode_tps": {"samples": tps, **numeric_summary(tps)}, + "warm_ttft_seconds": {"samples": ttft, **numeric_summary(ttft)}, + "failed_samples": [sample["sample_index"] for sample in samples if sample["status"] != "passed"], + }) + return 0 if all(sample["status"] == "passed" for sample in samples) else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/swap/direct_model_canary.py b/benchmarks/swap/direct_model_canary.py new file mode 100644 index 0000000000..23d996a407 --- /dev/null +++ b/benchmarks/swap/direct_model_canary.py @@ -0,0 +1,43 @@ +# What: import argument parsing; why: private run inputs must be explicit and validated. +import argparse + +# What: import dynamic module support; why: the exact reviewed router canary should be reused. +import importlib.util + +# What: import path handling; why: source and artifact locations need safe joins. +from pathlib import Path + +# What: create the parser; why: malformed operator input must fail before inference. +parser = argparse.ArgumentParser() +# What: require exact source; why: no installed copy may shadow the detached revision. +parser.add_argument("--source", required=True) +# What: require numeric loopback port; why: the canary must target the owned engine. +parser.add_argument("--port", required=True, type=int) +# What: require public model alias; why: requests must not leak private paths. +parser.add_argument("--model", required=True) +# What: require private artifact directory; why: raw responses stay local. +parser.add_argument("--artifacts", required=True) +# What: parse inputs; why: validated values drive all later operations. +args = parser.parse_args() +# What: resolve the reviewed qualifier; why: protocol parsing must match the router campaign. +module_path = Path(args.source) / "benchmarks/swap/qualify_native_router.py" +# What: create an import specification; why: exact source can be loaded without package ambiguity. +spec = importlib.util.spec_from_file_location("amd_direct_qualifier", module_path) +# What: create an isolated module; why: global qualifier state must not leak between runs. +module = importlib.util.module_from_spec(spec) +# What: execute the module; why: its reviewed canary must be callable below. +spec.loader.exec_module(module) +# What: construct loopback origin; why: requests must never leave the host. +base = f"http://127.0.0.1:{args.port}" +# What: resolve private evidence storage; why: outputs need safe path operations. +artifacts = Path(args.artifacts) +# What: repeat twice; why: qualification requires one cold-post-load and one warm completion. +for index in range(2): + # What: execute the reviewed direct canary; why: malformed streaming or wrong arithmetic fails closed. + raw, row = module.canary(base, args.model, direct=True) + # What: save raw SSE privately; why: review must distinguish protocol evidence from summaries. + (artifacts / f"completion-{index + 1}.sse").write_bytes(raw) + # What: save parsed evidence privately; why: the deterministic result must remain auditable. + (artifacts / f"completion-{index + 1}.txt").write_text(repr(row), encoding="utf-8") +# What: report success; why: the shell gate needs an explicit two-completion marker. +print("TWO_COMPLETIONS_OK") diff --git a/benchmarks/swap/qualify.py b/benchmarks/swap/qualify.py new file mode 100644 index 0000000000..5946690af8 --- /dev/null +++ b/benchmarks/swap/qualify.py @@ -0,0 +1,579 @@ +"""Opt-in Linux maintenance-window qualification against a real llama-swap binary. + +Artifacts contain local operational paths and raw model output. Keep them private. +This script never changes the protected service's configuration or enablement. +""" +# What: document opt in linux maintenance window qualification against a in the qualify docstring; why: introspection and maintainers read this exact docstring fragment to understand qualify behavior without executing it. +# What: document artifacts contain local operational paths and in the qualify docstring; why: introspection and maintainers read this exact docstring fragment to understand qualify behavior without executing it. +# What: document this script never changes the protected in the qualify docstring; why: introspection and maintainers read this exact docstring fragment to understand qualify behavior without executing it. +# What: preserve the paragraph boundary in the the qualify docstring; why: introspection and maintainers read this paragraph break to understand qualify behavior without executing it. + +# What: import argparse for main using argparse; why: main uses argparse argument parser, making that imported dependency available to its named operation. +import argparse +# What: import json for cancellation canary using json; why: cancellation_canary uses json loads, making that imported dependency available to its named operation. +import json +# What: import os for main using os; why: main uses os environ copy, making that imported dependency available to its named operation. +import os +# What: import path for main using pathlib and path; why: main uses path, making that imported dependency available to its named operation. +from pathlib import Path +# What: import signal for main using signal; why: main uses signal signal, making that imported dependency available to its named operation. +import signal +# What: import socket for require expected hostname using socket; why: require_expected_hostname uses socket gethostname, making that imported dependency available to its named operation. +import socket +# What: import subprocess for main using subprocess; why: main uses subprocess run, making that imported dependency available to its named operation. +import subprocess +# What: import sys for module initialization using sys; why: module initialization uses sys exit, making that imported dependency available to its named operation. +import sys +# What: import time for cancellation canary using time; why: cancellation_canary uses time monotonic, making that imported dependency available to its named operation. +import time +# What: import urllib error for http using urllib and error; why: http uses urllib request request, making that imported dependency available to its named operation. +import urllib.error +# What: import urllib request for http using urllib and request; why: http uses urllib request request, making that imported dependency available to its named operation. +import urllib.request +# What: import thread pool executor for main using concurrent and futures and thread pool executor; why: main uses thread pool executor, making that imported dependency available to its named operation. +from concurrent.futures import ThreadPoolExecutor + + +# What: define require_expected_hostname and its declared inputs; why: callers use require_expected_hostname to perform the behavior named by this helper without duplicating its boundary checks. +def require_expected_hostname(expected: str, *, actual: str | None = None) -> str: + """Fail closed unless the operator names this exact maintenance host. + + The mismatch deliberately omits both values so a copied error cannot publish + a private machine name. The approved public hardware label is documented + separately and is not assumed to equal the operating-system hostname. + """ + # What: document fail closed unless the operator names in the require_expected_hostname docstring; why: introspection and maintainers read this exact docstring fragment to understand require expected hostname behavior without executing it. + # What: document the mismatch deliberately omits both values in the require_expected_hostname docstring; why: introspection and maintainers read this exact docstring fragment to understand require expected hostname behavior without executing it. + # What: document a private machine name the approved in the require_expected_hostname docstring; why: introspection and maintainers read this exact docstring fragment to understand require expected hostname behavior without executing it. + # What: document separately and is not assumed to in the require_expected_hostname docstring; why: introspection and maintainers read this exact docstring fragment to understand require expected hostname behavior without executing it. + # What: preserve the paragraph boundary in the the require_expected_hostname docstring; why: introspection and maintainers read this paragraph break to understand require expected hostname behavior without executing it. + # What: compute actual from actual and gethostname and socket; why: if not expected or x00 in later reads actual, so require_expected_hostname must retain the computed value under that name. + actual = socket.gethostname() if actual is None else actual + # What: gate on expected and actual before runtime error; why: require_expected_hostname admits runtime error only for this predicate and excludes the opposite state. + if not expected or "\x00" in expected or actual != expected: + # What: raise RuntimeError for the caller; why: require_expected_hostname stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError( + # What: execute qualification host does not match the operator supplied expected hostname; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + "qualification host does not match the operator-supplied expected hostname" + # What: complete the RuntimeError call with ordered positional inputs; why: require_expected_hostname groups the supplied clauses as one RuntimeError call before its value is consumed. + ) + # What: return the hostname that passed exact-host validation; why: callers use this confirmed identity before any maintenance side effect is allowed. + return actual + + +# What: define an explicit qualification gate; why: validation must remain active under Python optimization. +def require(condition: bool, message: str) -> None: + # What: reject a failed gate with a runtime error; why: qualification evidence must never continue from an invalid state. + if not condition: + # What: raise the supplied diagnostic; why: operators need the exact failed gate in the private artifact. + raise RuntimeError(message) + + +# What: define http around url and body and timeout; why: its direct callers call http for http and rely on this exact input and result contract. +def http(url, body=None, timeout=30): + # What: compute data from body and encode and dumps and json; why: request urllib request request url data data headers later reads data, so http must retain the computed value under that name. + data = None if body is None else json.dumps(body).encode() + # What: map the content type field as application and json; why: http carries content type through request into with urllib request urlopen request timeout timeout as response. + request = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}) + # What: enter the urllib.request.urlopen managed context before return response read; why: http releases this resource or lock after return response read on both success and failure paths. + with urllib.request.urlopen(request, timeout=timeout) as response: + # What: return read and response from http; why: http exposes read and response so its caller can continue with the function\'s computed outcome. + return response.read() + + +# What: define wait_health around url and seconds; why: its direct callers call wait_health for wait health and rely on this exact input and result contract. +def wait_health(url, seconds): + # What: compute deadline from seconds and monotonic and time; why: while time monotonic deadline later reads deadline, so wait_health must retain the computed value under that name. + deadline = time.monotonic() + seconds + # What: iterate across deadline and monotonic and time to perform doc and loads and oserror and value error and json; why: wait_health repeats the body only while or for the loop header admits an iteration. + while time.monotonic() < deadline: + # What: establish the handler boundary for the protected operation; why: wait_health routes failures to oserror and value error while preserving cleanup and success flow. + try: + # What: compute doc from loads and json and http and url and 3; why: if doc get status ok later reads doc, so wait_health must retain the computed value under that name. + doc = json.loads(http(url, timeout=3)) + # What: gate on get and doc before doc; why: wait_health admits doc only for this predicate and excludes the opposite state. + if doc.get("status") == "ok": + # What: return doc from wait_health; why: wait_health exposes doc so its caller can continue with the function\'s computed outcome. + return doc + # What: handle oserror and value error by pass; why: wait_health converts that failure into this concrete recovery, response, or cleanup behavior. + except (OSError, ValueError): + # What: ignore the anticipated exception handled by this branch; why: wait_health continues its retry or cleanup path instead of re-raising that transient failure. + pass + # What: pause one second between health probes; why: wait_health avoids a busy retry loop while retaining a bounded readiness deadline. + time.sleep(1) + # What: raise TimeoutError for the caller; why: wait_health stops this rejected path before it can mutate state, dispatch work, or report success. + raise TimeoutError("health did not become ready") + + +# What: define canary around url and model and stream; why: its direct callers call canary for canary and rely on this exact input and result contract. +def canary(url, model, stream=False): + # What: compute body from model and stream and model and messages and temperature; why: body stream options include usage later reads body, so canary must retain the computed value under that name. + body = { + # What: map the model field as model; why: canary sends this field through body so the router selects the canonical model or alias for upstream dispatch. + "model": model, + # What: map the role field as user; why: canary carries role through body into body stream options include usage true. + "messages": [{"role": "user", "content": "What is 2 + 2? Reply with only the single digit."}], + # What: map the temperature field as 0; why: canary carries temperature through body into body stream options include usage true. + "temperature": 0, "max_tokens": 32, "stream": stream, + # What: map the enable thinking field as false; why: canary carries enable thinking through body into body stream options include usage true. + "chat_template_kwargs": {"enable_thinking": False}, + # What: complete the body mapping with model and messages and temperature and max tokens and stream; why: canary groups the supplied clauses as one body mapping before its value is consumed. + } + # What: gate on stream before body; why: canary admits body only for this predicate and excludes the opposite state. + if stream: + # What: map the include usage field as true; why: canary carries include usage through body entry into raw http url v1 chat completions body. + body["stream_options"] = {"include_usage": True} + # What: compute raw from http and body and url and v1 and chat; why: assert b data done in raw later reads raw, so canary must retain the computed value under that name. + raw = http(url + "/v1/chat/completions", body, timeout=660) + # What: gate on stream before parts; why: canary admits parts only for this predicate and excludes the opposite state. + if stream: + # What: initialize parts as an empty runtime accumulator; why: canary appends or maps entries into it during parts append choice get delta get content or before consuming the aggregate. + parts = [] + # What: assert that b data done is present in raw; why: canary requires b data done is present in raw to be true, so a false result stops the invalid state. + require(b"data: [DONE]" in raw, "SSE completion marker missing") + # What: iterate across splitlines and decode and raw to perform doc and choice and startswith and line and loads; why: canary repeats the body only while or for the loop header admits an iteration. + for line in raw.decode().splitlines(): + # What: gate on startswith and line before doc and loads and json and line; why: canary admits doc and loads and json and line only for this predicate and excludes the opposite state. + if line.startswith("data: ") and line != "data: [DONE]": + # What: compute doc from loads and json and line and 6; why: for choice in doc get choices later reads doc, so canary must retain the computed value under that name. + doc = json.loads(line[6:]) + # What: iterate across get and doc to perform append and parts and get and choice; why: canary repeats the body only while or for the loop header admits an iteration. + for choice in doc.get("choices", []): + # What: preserve the exact parts append choice get delta get content or literal fragment; why: canary passes this fragment verbatim through parts.append(choice.get("delta", {}).get("content") or ""), because changing it would alter a protocol payload, serialized fixture, or public message. + parts.append(choice.get("delta", {}).get("content") or "") + # What: compute content from join and parts and value; why: content doc choices message get content later reads content, so canary must retain the computed value under that name. + content = "".join(parts) + # What: select the remaining branch that performs doc json loads raw; why: canary covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: compute doc from loads and raw and json; why: content doc choices message get content later reads doc, so canary must retain the computed value under that name. + doc = json.loads(raw) + # What: compute content from get and doc and value and content and message; why: return raw content strip later reads content, so canary must retain the computed value under that name. + content = doc["choices"][0]["message"].get("content") or "" + # What: return raw and strip and content from canary; why: canary exposes raw and strip and content so its caller can continue with the function\'s computed outcome. + return raw, content.strip() + + +# What: define cancellation_canary around url and model and seconds; why: its direct callers call cancellation_canary for cancellation canary and rely on this exact input and result contract. +def cancellation_canary(url, model, *, seconds=30): + """Close a live SSE response, then require same-process terminal abort evidence. + + Active reaching zero alone is insufficient: TTL restart and normal completion + can also produce that observation. Check instance identity and completed count. + """ + # What: document close a live sse response then in the cancellation_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand cancellation canary behavior without executing it. + # What: document active reaching zero alone is insufficient in the cancellation_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand cancellation canary behavior without executing it. + # What: document can also produce that observation check in the cancellation_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand cancellation canary behavior without executing it. + # What: preserve the paragraph boundary in the the cancellation_canary docstring; why: introspection and maintainers read this paragraph break to understand cancellation canary behavior without executing it. + # What: compute stats url from model and url and v1 and stats and upstream; why: before json loads http stats url later reads stats url, so cancellation_canary must retain the computed value under that name. + stats_url = url + "/upstream/" + model + "/v1/stats" + # What: compute before from loads and json and http and stats url; why: instance before get instance id later reads before, so cancellation_canary must retain the computed value under that name. + before = json.loads(http(stats_url)) + # What: compute instance from get and before and instance id; why: assert instance backend instance identity missing later reads instance, so cancellation_canary must retain the computed value under that name. + instance = before.get("instance_id") + # What: assert that instance; why: cancellation_canary requires instance to be true, so a false result stops the invalid state. + require(bool(instance), "backend instance identity missing") + # What: assert that before requests active equals 0; why: cancellation_canary requires before requests active equals 0 to be true, so a false result stops the invalid state. + require(before["requests"]["active"] == 0, "cancellation test requires an idle backend") + # What: map the model field as model; why: cancellation_canary sends this field through body so the router selects the canonical model or alias for upstream dispatch. + body = {"model": model, "stream": True, "max_tokens": 1024, "temperature": 0, + # What: map the role field as user; why: cancellation_canary carries role through body into data json dumps body encode. + "messages": [{"role": "user", "content": + # What: apply the count from to writing every number portion of body; why: cancellation_canary uses this clause to evaluate body as one grouped value. + "Count from 1 to 1000, writing every number on a separate line. Do not summarize."}], + # What: map the enable thinking field as false; why: cancellation_canary carries enable thinking through body into data json dumps body encode. + "chat_template_kwargs": {"enable_thinking": False}} + # What: compute request from request and request and url and urllib; why: with urllib request urlopen request timeout as response later reads request, so cancellation_canary must retain the computed value under that name. + request = urllib.request.Request(url + "/v1/chat/completions", + # What: supply data to operation.encode; why: cancellation_canary binds this encode and dumps and body and json value to operation.encode's data input. + data=json.dumps(body).encode(), + # What: map the content type field as application and json; why: cancellation_canary carries content type through request into with urllib request urlopen request timeout 660 as response. + headers={"Content-Type": "application/json"}) + # What: compute raw from bytearray; why: raw extend line later reads raw, so cancellation_canary must retain the computed value under that name. + raw = bytearray() + # What: compute started from monotonic and time; why: after after first content seconds first content started later reads started, so cancellation_canary must retain the computed value under that name. + started = time.monotonic() + # What: initialize the observed-statistics sentinel to no result; why: cancellation_canary can distinguish not-yet-fetched state from a completed statistics response. + observed = None + # What: enter the urllib.request.urlopen managed context before for line in response; why: cancellation_canary releases this resource or lock after for line in response on both success and failure paths. + with urllib.request.urlopen(request, timeout=660) as response: + # Read incrementally. Reading the entire body would only test completion. + # What: iterate across response to perform extend and line and raw; why: cancellation_canary repeats the body only while or for the loop header admits an iteration. + for line in response: + # What: append the received stream line to the raw response buffer; why: cancellation_canary tracks accumulated bytes before triggering its disconnect threshold. + raw.extend(line) + # What: gate on len and raw before runtime error; why: cancellation_canary admits runtime error only for this predicate and excludes the opposite state. + if len(raw) > 1024 * 1024: + # What: raise RuntimeError for the caller; why: cancellation_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("stream exceeded cancellation capture limit") + # What: gate on strip and line before runtime error; why: cancellation_canary admits runtime error only for this predicate and excludes the opposite state. + if line.strip() == b"data: [DONE]": + # What: raise RuntimeError for the caller; why: cancellation_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("stream completed before cancellation") + # What: gate on startswith and line before the computed value; why: cancellation_canary admits the computed value only for this predicate and excludes the opposite state. + if not line.startswith(b"data: "): + # What: apply the continue portion of the enclosing predicate; why: this clause remains in cancellation_canary\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: compute doc from loads and json and line and 6; why: if any choice get delta get content later reads doc, so cancellation_canary must retain the computed value under that name. + doc = json.loads(line[6:]) + # What: gate on any and get and choice and doc before first content and monotonic and time; why: cancellation_canary admits first content and monotonic and time only for this predicate and excludes the opposite state. + if any(choice.get("delta", {}).get("content") for choice in doc.get("choices", [])): + # What: compute first content from monotonic and time; why: after after first content seconds first content started later reads first content, so cancellation_canary must retain the computed value under that name. + first_content = time.monotonic() + # What: compute observed from loads and json and http and stats url; why: assert observed instance id instance backend restarted later reads observed, so cancellation_canary must retain the computed value under that name. + observed = json.loads(http(stats_url)) + # What: assert that observed instance id equals instance; why: cancellation_canary requires observed instance id equals instance to be true, so a false result stops the invalid state. + require(observed["instance_id"] == instance, "backend restarted before disconnect") + # What: assert that observed requests active exceeds 0; why: cancellation_canary requires observed requests active exceeds 0 to be true, so a false result stops the invalid state. + require(observed["requests"]["active"] > 0, "generation already finished before disconnect") + # What: leave the stream loop after enough response bytes arrive; why: cancellation can now be triggered against a live partial response. + break + # What: select the remaining branch that performs raise runtime error stream ended without a; why: cancellation_canary covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: raise RuntimeError for the caller; why: cancellation_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("stream ended without a content delta") + # What: compute disconnected from monotonic and time; why: deadline disconnected seconds later reads disconnected, so cancellation_canary must retain the computed value under that name. + disconnected = time.monotonic() + # What: compute deadline from disconnected and seconds; why: if time monotonic deadline later reads deadline, so cancellation_canary must retain the computed value under that name. + deadline = disconnected + seconds + # What: poll cancellation statistics until a terminal result; why: the loop ends after cancellation evidence or its explicit deadline. + while True: + # What: compute after from loads and json and http and stats url; why: assert after instance id instance backend restart later reads after, so cancellation_canary must retain the computed value under that name. + after = json.loads(http(stats_url)) + # What: assert that after instance id equals instance; why: cancellation_canary requires after instance id equals instance to be true, so a false result stops the invalid state. + require(after["instance_id"] == instance, "backend restart cannot count as cancellation") + # What: gate on after before after and before; why: cancellation_canary admits after and before only for this predicate and excludes the opposite state. + if after["requests"]["active"] == 0: + # What: require after requests completed == before requests completed; why: the qualifier stops immediately when this protected invariant is false. + # What: require after requests completed == before requests completed; why: the qualifier stops immediately when this protected invariant is false. + require( + after["requests"]["completed"] == before["requests"]["completed"], + "normal completion cannot count as cancellation", + ) + # What: map the passed field as true; why: cancellation_canary carries passed into return bytes(raw), {"passed": True, "before": before, "during": observed. + return bytes(raw), {"passed": True, "before": before, "during": observed, + # What: map the after field as after; why: cancellation_canary carries after into "after": after, "firstContentSeconds": first_content - started. + "after": after, "firstContentSeconds": first_content - started, + # What: map the abort seconds field as disconnected and monotonic and time; why: cancellation_canary carries abort seconds into "abortSeconds": time.monotonic() - disconnected}. + "abortSeconds": time.monotonic() - disconnected} + # What: gate on deadline and monotonic and time before timeout error; why: cancellation_canary admits timeout error only for this predicate and excludes the opposite state. + if time.monotonic() >= deadline: + # What: raise TimeoutError for the caller; why: cancellation_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise TimeoutError("disconnected request did not reach terminal abort") + # What: call time.sleep with 0 25; why: cancellation_canary invokes time.sleep while performing the enclosing return; the call advances that operation through its result or side effect. + time.sleep(0.25) + + +# What: define main around the current object state; why: its direct callers call main for main and rely on this exact input and result contract. +def main(): + # What: compute parser from argument parser and argparse and doc; why: parser add argument name required later reads parser, so main must retain the computed value under that name. + parser = argparse.ArgumentParser(description=__doc__) + # What: iterate across the computed value to perform add argument and parser and name; why: main repeats the body only while or for the loop header admits an iteration. + for name in ("source", "python", "llama-swap", "model-a", "model-b", "artifacts", "protected-service", "protected-url", "expected-hostname"): + # What: preserve the exact parser add argument name required literal fragment; why: main passes this fragment verbatim through parser.add_argument("--" + name, required=True), because changing it would alter a protocol payload, serialized fixture, or public message. + parser.add_argument("--" + name, required=True) + # What: register the parser add argument allow maintenance action store true required True command-line option; why: main validates this operator input before starting the qualification sequence. + parser.add_argument("--allow-maintenance", action="store_true", required=True) + # What: register the parser add argument port type int default 1960 command-line option; why: main validates this operator input before starting the qualification sequence. + parser.add_argument("--port", type=int, default=1960) + # What: preserve the exact parser add argument start port type int default literal fragment; why: main passes this fragment verbatim through parser.add_argument("--start-port", type=int, default=1961), because changing it would alter a protocol payload, serialized fixture, or public message. + parser.add_argument("--start-port", type=int, default=1961) + # What: add the --extended switch for concurrency and idle-eviction checks; why: operators opt into the longer qualification cases instead of running them by default. + parser.add_argument("--extended", action="store_true", help="Also test concurrent requests and idle eviction") + # What: add the --cancellation switch for live SSE disconnect checks; why: operators explicitly request the disruptive cancellation-and-recovery qualification path. + parser.add_argument("--cancellation", action="store_true", help="Also qualify live SSE disconnect and recovery") + # What: compute args from parse args and parser; why: require expected hostname args expected hostname later reads args, so main must retain the computed value under that name. + args = parser.parse_args() + # What: call require_expected_hostname with expected hostname and args; why: main invokes require_expected_hostname while performing artifacts path args artifacts; the call advances that operation through its result or side effect. + require_expected_hostname(args.expected_hostname) + # What: compute artifacts from path and artifacts and args; why: artifacts mkdir parents exist ok later reads artifacts, so main must retain the computed value under that name. + artifacts = Path(args.artifacts) + # What: supply parents to artifacts.mkdir; why: main binds this true value to artifacts.mkdir's parents input. + artifacts.mkdir(parents=True, exist_ok=False) + # What: map the trials field as the fixture input; why: main carries trials through status into artifacts result json write text json dumps status indent 2. + status = {"trials": [], "restored": False} + + # What: define save around the current object state; why: its direct callers call save for save and rely on this exact input and result contract. + def save(): + # What: write the current qualification status as indented UTF-8 JSON; why: operators need a durable result artifact even when a later qualification phase fails. + (artifacts / "result.json").write_text(json.dumps(status, indent=2), encoding="utf-8") + + # What: compute service from sudo and n and systemctl; why: subprocess run service is active quiet args protected service check later reads service, so main must retain the computed value under that name. + service = ["sudo", "-n", "systemctl"] + # What: execute subprocess run service is active quiet args protected service check True; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + subprocess.run(service + ["is-active", "--quiet", args.protected_service], check=True) + # What: compute status entry from wait health and protected url and args and 10 and health; why: status trials append row later reads status entry, so main must retain the computed value under that name. + status["baselineHealth"] = wait_health(args.protected_url + "/health", 10) + # What: compute baseline models from loads and json and http and protected url; why: protected model baseline models data id later reads baseline models, so main must retain the computed value under that name. + baseline_models = json.loads(http(args.protected_url + "/v1/models")) + # What: compute protected model from baseline models and id and 0 and data; why: raw content canary args protected url protected model later reads protected model, so main must retain the computed value under that name. + protected_model = baseline_models["data"][0]["id"] + # What: evaluate and capture raw content canary args protected url protected model; why: the enclosing qualifier uses the captured result in its next validation or artifact step. + raw, content = canary(args.protected_url, protected_model) + # What: execute artifacts baseline json write bytes raw; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + (artifacts / "baseline.json").write_bytes(raw) + # What: gate on content before runtime error; why: main admits runtime error only for this predicate and excludes the opposite state. + if content != "4": + # What: raise RuntimeError for the caller; why: main stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("protected-service baseline canary did not return 4; no maintenance performed") + + # What: compute env from copy and environ and os; why: env pythonpath str path args source python later reads env, so main must retain the computed value under that name. + env = os.environ.copy() + # What: compute env entry from str and path and source and args and python; why: env path str path home local bin later reads env entry, so main must retain the computed value under that name. + env["PYTHONPATH"] = str(Path(args.source) / "python") + # What: compute env entry from pathsep and get and str and os; why: env torch extensions dir str artifacts torch extensions later reads env entry, so main must retain the computed value under that name. + env["PATH"] = str(Path.home() / ".local/bin") + os.pathsep + env.get("PATH", "") + # Avoid sharing extension binaries or abandoned build locks across revisions. + # What: compute env entry from str and artifacts and torch extensions; why: env max jobs later reads env entry, so main must retain the computed value under that name. + env["TORCH_EXTENSIONS_DIR"] = str(artifacts / "torch-extensions") + # What: cap native-extension compilation at two parallel jobs; why: the later kernel preflight must not exhaust the qualification host while building extensions. + env["MAX_JOBS"] = "2" + # What: compute config from start port and args and health check timeout and global ttl and unload timeout; why: config f alias cmd json dumps command later reads config, so main must retain the computed value under that name. + config = ["healthCheckTimeout: 600", "globalTTL: 0", "unloadTimeout: 45", "logToStdout: both", f"startPort: {args.start_port}", "models:"] + # What: import shlex for main using shlex; why: main uses shlex join, making that imported dependency available to its named operation. + import shlex + + # What: iterate across model a and model b and args to perform command and join and shlex and python and model; why: main repeats the body only while or for the loop header admits an iteration. + for alias, model in (("model-a", args.model_a), ("model-b", args.model_b)): + # What: compute command from join and shlex and python and model; why: config f alias cmd json dumps command later reads command, so main must retain the computed value under that name. + command = shlex.join([ + # What: apply the args python m freetoken cli serve model model portion of command; why: main uses this clause to evaluate command as one grouped value. + args.python, "-m", "freetoken.cli", "serve", "--model", model, + # What: apply the host port port served model name model id portion of command; why: main uses this clause to evaluate command as one grouped value. + "--host", "127.0.0.1", "--port", "${PORT}", "--served-model-name", "${MODEL_ID}", + # What: apply the max seq len override num tokens max prefill length portion of command; why: main uses this clause to evaluate command as one grouped value. + "--max-seq-len-override", "4096", "--num-tokens", "4096", "--max-prefill-length", "512", + # What: apply the max running requests graph memory ratio portion of command; why: main uses this clause to evaluate command as one grouped value. + "--max-running-requests", "1", "--graph", "1", "--memory-ratio", "0.75", + # What: apply the attention backend triton moe backend fused disable pynccl portion of command; why: main uses this clause to evaluate command as one grouped value. + "--attention-backend", "triton", "--moe-backend", "fused", "--disable-pynccl", + # What: complete the shlex.join call with python; why: main groups the supplied clauses as one shlex.join call before its value is consumed. + ]) + # What: append this model command, readiness endpoint, and proxy stanza; why: the generated llama-swap configuration needs a complete entry before optional TTL settings. + config += [f" {alias}:", " cmd: " + json.dumps(command), " checkEndpoint: /ready", " proxy: http://127.0.0.1:${PORT}"] + # What: gate on extended and args before append and config; why: main admits append and config only for this predicate and excludes the opposite state. + if args.extended: + # What: preserve the exact config append ttl literal fragment; why: main passes this fragment verbatim through config.append(" ttl: 5"), because changing it would alter a protocol payload, serialized fixture, or public message. + config.append(" ttl: 5") + # What: compute config path from artifacts and models and yaml; why: config path write text n join config n encoding later reads config path, so main must retain the computed value under that name. + config_path = artifacts / "models.yaml" + # What: preserve the exact config path write text n join config n encoding literal fragment; why: main passes this fragment verbatim through config_path.write_text("\n".join(config) + "\n", encoding="utf-8"), because changing it would alter a protocol payload, serialized fixture, or public message. + config_path.write_text("\n".join(config) + "\n", encoding="utf-8") + # What: run llama-swap configuration validation against the generated file; why: qualification fails before launch when the temporary routing configuration is invalid. + subprocess.run([args.llama_swap, "-config", str(config_path), "-validate"], env=env, check=True) + # What: preserve the exact print native kernel preflight started flush literal fragment; why: main passes this fragment verbatim through print("NATIVE_KERNEL_PREFLIGHT_STARTED", flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print("NATIVE_KERNEL_PREFLIGHT_STARTED", flush=True) + # What: open the with artifacts kernel build log open wb as build log resource scope; why: the qualification operation releases this resource when the guarded block exits. + with (artifacts / "kernel-build.log").open("wb") as build_log: + # What: execute subprocess run args python c from freetoken kernel gguf import module module print NATIVE KERNEL READY; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + subprocess.run([args.python, "-c", "from freetoken.kernel.gguf import _module; _module(); print('NATIVE_KERNEL_READY')"], + # What: supply env to subprocess.run; why: main binds this env value to subprocess.run's env input. + env=env, cwd=args.source, stdout=build_log, stderr=subprocess.STDOUT, check=True, timeout=600) + # What: initialize the child-process sentinel to no process; why: cleanup can test whether llama-swap started before attempting termination. + proc = None + # What: compute maintenance from false; why: maintenance later reads maintenance, so main must retain the computed value under that name. + maintenance = False + + # What: define interrupted around the current object state; why: its direct callers call interrupted for interrupted and rely on this exact input and result contract. + def interrupted(*_): + # What: convert a termination signal into KeyboardInterrupt; why: the normal interruption path then performs restoration and child cleanup. + raise KeyboardInterrupt + + # What: register the interruption handler for SIGTERM; why: service-manager termination must enter the qualifier restoration path. + signal.signal(signal.SIGTERM, interrupted) + # What: register the interruption handler for SIGHUP; why: session loss must enter the same restoration path. + signal.signal(signal.SIGHUP, interrupted) + # What: establish the handler boundary for the protected operation; why: main routes failures to base exception while preserving cleanup and success flow. + try: + # Set the restore obligation before the stop, including partial failures. + # What: compute maintenance from true; why: if maintenance later reads maintenance, so main must retain the computed value under that name. + maintenance = True + # What: execute subprocess run service stop args protected service check True timeout 90; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + subprocess.run(service + ["stop", args.protected_service], check=True, timeout=90) + # What: preserve the exact print maintenance started flush literal fragment; why: main passes this fragment verbatim through print("MAINTENANCE_STARTED", flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print("MAINTENANCE_STARTED", flush=True) + # What: enter the operation.open managed context before proc subprocess popen args llama swap config str config path; why: main releases this resource or lock after proc subprocess popen args llama swap config str config path on both success and failure paths. + with (artifacts / "swap.log").open("wb") as log: + # What: compute proc from popen and subprocess and llama swap and env; why: if time monotonic deadline or proc poll is later reads proc, so main must retain the computed value under that name. + proc = subprocess.Popen([args.llama_swap, "-config", str(config_path), "-listen", f"127.0.0.1:{args.port}"], + # What: supply env to subprocess.Popen; why: main binds this env value to subprocess.Popen's env input. + env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + # What: compute base from port and args and http; why: listing json loads http base v1 models later reads base, so main must retain the computed value under that name. + base = f"http://127.0.0.1:{args.port}" + # What: compute deadline from monotonic and time and 20; why: if time monotonic deadline or proc poll is later reads deadline, so main must retain the computed value under that name. + deadline = time.monotonic() + 20 + # What: retry model-list readiness until a terminal condition; why: the loop exits on a valid listing and raises on process exit or deadline expiry. + while True: + # What: establish the handler boundary for the protected operation; why: main routes failures to oserror and value error while preserving cleanup and success flow. + try: + # What: compute listing from loads and json and http and base and v1; why: assert item id for item in later reads listing, so main must retain the computed value under that name. + listing = json.loads(http(base + "/v1/models", timeout=2)) + # What: assert that item id for item in listing equals model a model b; why: main requires item id for item in listing equals model a model b to be true, so a false result stops the invalid state. + require({item["id"] for item in listing["data"]} == {"model-a", "model-b"}, "model listing did not contain both qualification targets") + # What: leave the readiness loop after a valid listing; why: both expected models are visible and qualification can begin. + break + # What: handle oserror and value error by if time monotonic at least deadline or proc poll; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except (OSError, ValueError): + # What: gate on deadline and monotonic and poll and time and proc before the computed value; why: main admits the computed value only for this predicate and excludes the opposite state. + if time.monotonic() >= deadline or proc.poll() is not None: + # What: re-propagate the active failure to the caller; why: main stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: wait half a second before retrying the model-list request; why: readiness polling needs backoff instead of a busy loop. + time.sleep(0.5) + # What: iterate across enumerate to perform started and monotonic and time; why: main repeats the body only while or for the loop header admits an iteration. + for index, (alias, streaming) in enumerate((("model-a", False), ("model-b", True), ("model-a", True))): + # What: compute started from monotonic and time; why: row model alias stream streaming seconds later reads started, so main must retain the computed value under that name. + started = time.monotonic() + # What: preserve the exact print f trial started index alias flush literal fragment; why: main passes this fragment verbatim through print(f"TRIAL_STARTED {index} {alias}", flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print(f"TRIAL_STARTED {index} {alias}", flush=True) + # What: compute raw and content from canary and base and alias and streaming; why: artifacts f trial index response write bytes later reads raw and content, so main must retain the computed value under that name. + raw, content = canary(base, alias, streaming) + # What: preserve the exact artifacts f trial index response write bytes literal fragment; why: main passes this fragment verbatim through (artifacts / f"trial-{index}.response").write_bytes(raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / f"trial-{index}.response").write_bytes(raw) + # What: map the model field as alias; why: main sends this field through row so the router selects the canonical model or alias for upstream dispatch. + row = {"model": alias, "stream": streaming, "seconds": time.monotonic() - started, "content": content, "passed": content == "4"} + # What: preserve the exact status trials append row literal fragment; why: main passes this fragment verbatim through status["trials"].append(row), because changing it would alter a protocol payload, serialized fixture, or public message. + status["trials"].append(row) + # What: checkpoint the completed trial in the result artifact; why: evidence survives if a later trial or cleanup step fails. + save() + # What: preserve the exact print trial result json dumps row flush literal fragment; why: main passes this fragment verbatim through print("TRIAL_RESULT " + json.dumps(row), flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print("TRIAL_RESULT " + json.dumps(row), flush=True) + # What: gate on row before runtime error; why: main admits runtime error only for this predicate and excludes the opposite state. + if not row["passed"]: + # What: raise RuntimeError for the caller; why: main stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("deterministic quality gate failed") + # What: gate on cancellation and args before raw and cancellation and cancellation canary and base; why: main admits raw and cancellation and cancellation canary and base only for this predicate and excludes the opposite state. + if args.cancellation: + # What: compute raw and cancellation from cancellation canary and base and model a; why: artifacts cancelled prefix sse write bytes raw later reads raw and cancellation, so main must retain the computed value under that name. + raw, cancellation = cancellation_canary(base, "model-a") + # What: preserve the exact artifacts cancelled prefix sse write bytes raw literal fragment; why: main passes this fragment verbatim through (artifacts / "cancelled-prefix.sse").write_bytes(raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / "cancelled-prefix.sse").write_bytes(raw) + # What: compute status entry from cancellation; why: status cancellation recovery passed later reads status entry, so main must retain the computed value under that name. + status["cancellation"] = cancellation + # What: checkpoint the cancellation result before recovery trials; why: disconnect evidence survives if post-cancellation validation fails. + save() + # What: iterate across enumerate to perform raw and content and canary and base and alias; why: main repeats the body only while or for the loop header admits an iteration. + for index, alias in enumerate(("model-a", "model-b", "model-a")): + # What: compute raw and content from canary and base and alias and true; why: artifacts f after cancel index alias sse later reads raw and content, so main must retain the computed value under that name. + raw, content = canary(base, alias, True) + # What: preserve the exact artifacts f after cancel index alias sse literal fragment; why: main passes this fragment verbatim through (artifacts / f"after-cancel-{index}-{alias}.sse").write_bytes(raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / f"after-cancel-{index}-{alias}.sse").write_bytes(raw) + # What: assert that content equals 4; why: main requires content equals 4 to be true, so a false result stops the invalid state. + require(content == "4", "post-cancellation routing failed") + # What: compute status entry from true; why: status concurrent passed later reads status entry, so main must retain the computed value under that name. + status["cancellationRecoveryPassed"] = True + # What: preserve the exact print cancellation recovery ok flush literal fragment; why: main passes this fragment verbatim through print("CANCELLATION_RECOVERY_OK", flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print("CANCELLATION_RECOVERY_OK", flush=True) + # What: gate on extended and args before names and clients and futures and print and thread pool executor; why: main admits names and clients and futures and print and thread pool executor only for this predicate and excludes the opposite state. + if args.extended: + # What: iterate across the computed value to perform clients and futures and thread pool executor and index and future; why: main repeats the body only while or for the loop header admits an iteration. + for names in (("model-a", "model-a"), ("model-a", "model-b")): + # What: enter the ThreadPoolExecutor managed context before futures clients submit canary base name for; why: main releases this resource or lock after futures clients submit canary base name for on both success and failure paths. + with ThreadPoolExecutor(2) as clients: + # What: compute futures from submit and canary and base and name; why: for index future in enumerate futures later reads futures, so main must retain the computed value under that name. + futures = [clients.submit(canary, base, name, True) for name in names] + # What: iterate across enumerate and futures to perform raw and content and result and future; why: main repeats the body only while or for the loop header admits an iteration. + for index, future in enumerate(futures): + # What: compute raw and content from result and future; why: artifacts f concurrent join names index later reads raw and content, so main must retain the computed value under that name. + raw, content = future.result() + # What: preserve the exact artifacts f concurrent join names index literal fragment; why: main passes this fragment verbatim through (artifacts / f"concurrent-{'-'.join(names)}-{index}.sse").write_bytes(ra, because changing it would alter a protocol payload, serialized fixture, or public me. + (artifacts / f"concurrent-{'-'.join(names)}-{index}.sse").write_bytes(raw) + # What: assert that content equals 4; why: main requires content equals 4 to be true, so a false result stops the invalid state. + require(content == "4", "concurrent quality gate failed") + # What: assert that b usage is present in raw; why: main requires b usage is present in raw to be true, so a false result stops the invalid state. + require(b'"usage"' in raw, "streamed usage block missing") + # What: preserve the exact print concurrent ok join names flush literal fragment; why: main passes this fragment verbatim through print("CONCURRENT_OK " + ",".join(names), flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print("CONCURRENT_OK " + ",".join(names), flush=True) + # What: compute status entry from true; why: status idle eviction passed later reads status entry, so main must retain the computed value under that name. + status["concurrentPassed"] = True + # What: compute deadline from monotonic and time and 30; why: while time monotonic deadline later reads deadline, so main must retain the computed value under that name. + deadline = time.monotonic() + 30 + # What: iterate across deadline and monotonic and time to perform running and loads and json and http and base; why: main repeats the body only while or for the loop header admits an iteration. + while time.monotonic() < deadline: + # What: compute running from loads and json and http and base and running; why: if running get running later reads running, so main must retain the computed value under that name. + running = json.loads(http(base + "/running")) + # What: gate on get and running before status; why: main admits status only for this predicate and excludes the opposite state. + if running.get("running") == []: + # What: compute status entry from true; why: assert status get idle eviction passed idle ttl did later reads status entry, so main must retain the computed value under that name. + status["idleEvictionPassed"] = True + # What: leave the idle-eviction loop after unload; why: the engine is no longer running and the eviction gate is satisfied. + break + # What: pause one second before checking idle eviction again; why: the qualifier gives asynchronous unload work time to complete without busy-waiting. + time.sleep(1) + # What: assert that status get idle eviction passed; why: main requires status get idle eviction passed to be true, so a false result stops the invalid state. + require(bool(status.get("idleEvictionPassed")), "idle TTL did not unload the models") + # What: preserve the exact print idle eviction ok flush literal fragment; why: main passes this fragment verbatim through print("IDLE_EVICTION_OK", flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print("IDLE_EVICTION_OK", flush=True) + # What: handle base exception by status error repr exc; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException as exc: + # What: compute status entry from repr and exc; why: status cleanup error repr exc later reads status entry, so main must retain the computed value under that name. + status["error"] = repr(exc) + # What: preserve the exact print qualification failed repr exc flush literal fragment; why: main passes this fragment verbatim through print("QUALIFICATION_FAILED " + repr(exc), flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print("QUALIFICATION_FAILED " + repr(exc), flush=True) + # What: run if proc is not on every exit path; why: main performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: gate on proc before oserror and timeout expired and poll and killpg and pid; why: main admits oserror and timeout expired and poll and killpg and pid only for this predicate and excludes the opposite state. + if proc is not None: + # What: establish the handler boundary for the protected operation; why: main routes failures to oserror and timeout expired and subprocess while preserving cleanup and success flow. + try: + # What: gate on poll and proc before killpg and pid and sigterm and os and proc; why: main admits killpg and pid and sigterm and os and proc only for this predicate and excludes the opposite state. + if proc.poll() is None: + # What: call os.killpg with pid and proc and sigterm and signal; why: main invokes os.killpg while performing try; the call advances that operation through its result or side effect. + os.killpg(proc.pid, signal.SIGTERM) + # What: establish the handler boundary for the protected operation; why: main routes failures to timeout expired and subprocess while preserving cleanup and success flow. + try: + # What: supply timeout to proc.wait; why: main binds this 60 value to proc.wait's timeout input. + proc.wait(timeout=60) + # What: handle timeout expired and subprocess by os killpg proc pid signal sigkill; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except subprocess.TimeoutExpired: + # What: call os.killpg with pid and proc and sigkill and signal; why: main invokes os.killpg while performing proc wait timeout; the call advances that operation through its result or side effect. + os.killpg(proc.pid, signal.SIGKILL) + # What: supply timeout to proc.wait; why: main binds this 10 value to proc.wait's timeout input. + proc.wait(timeout=10) + # What: handle oserror and timeout expired and subprocess by status cleanup error repr exc; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except (OSError, subprocess.TimeoutExpired) as exc: + # What: compute status entry from repr and exc; why: status restored health wait health args protected url health later reads status entry, so main must retain the computed value under that name. + status["cleanupError"] = repr(exc) + # What: gate on maintenance before exception and run and status and wait health and raw; why: main admits exception and run and status and wait health and raw only for this predicate and excludes the opposite state. + if maintenance: + # What: establish the handler boundary for the protected operation; why: main routes failures to exception while preserving cleanup and success flow. + try: + # What: execute subprocess run service start args protected service check True timeout 180; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + subprocess.run(service + ["start", args.protected_service], check=True, timeout=180) + # What: compute status entry from wait health and protected url and args and 300 and health; why: status restored content later reads status entry, so main must retain the computed value under that name. + status["restoredHealth"] = wait_health(args.protected_url + "/health", 300) + # What: evaluate and capture raw content canary args protected url protected model; why: the enclosing qualifier uses the captured result in its next validation or artifact step. + raw, content = canary(args.protected_url, protected_model) + # What: execute artifacts restored json write bytes raw; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + (artifacts / "restored.json").write_bytes(raw) + # What: compute status entry from content and 4; why: print restored str status restored flush later reads status entry, so main must retain the computed value under that name. + status["restored"] = content == "4" + # What: execute print RESTORED str status restored flush True; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + print("RESTORED " + str(status["restored"]), flush=True) + # What: handle exception by status restore error repr exc; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: + # What: compute status entry from repr and exc; why: passed status restored and error not later reads status entry, so main must retain the computed value under that name. + status["restoreError"] = repr(exc) + # What: preserve the exact print restore failed repr exc flush literal fragment; why: main passes this fragment verbatim through print("RESTORE_FAILED " + repr(exc), flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print("RESTORE_FAILED " + repr(exc), flush=True) + # What: checkpoint the final restoration state; why: the artifact records cleanup success or failure before exit status is computed. + save() + # What: compute passed from status and all and len and x and restored; why: and len status trials and all later reads passed, so main must retain the computed value under that name. + passed = (status["restored"] and "error" not in status and "cleanupError" not in status + # What: call all with x and status and passed and trials; why: main invokes all while performing if args extended; the call advances that operation through its result or side effect. + and len(status["trials"]) == 3 and all(x["passed"] for x in status["trials"])) + # What: gate on extended and args before passed and get and status; why: main admits passed and get and status only for this predicate and excludes the opposite state. + if args.extended: + # What: compute passed from passed and get and status and concurrent passed and idle eviction passed; why: passed passed and status get cancellation get later reads passed, so main must retain the computed value under that name. + passed = passed and status.get("concurrentPassed") and status.get("idleEvictionPassed") + # What: gate on cancellation and args before passed and get and status; why: main admits passed and get and status only for this predicate and excludes the opposite state. + if args.cancellation: + # What: compute passed from passed and get and status and passed and cancellation recovery passed; why: return if passed else later reads passed, so main must retain the computed value under that name. + passed = passed and status.get("cancellation", {}).get("passed") and status.get("cancellationRecoveryPassed") + # What: return passed and 0 and 1 from main; why: main exposes passed and 0 and 1 so its caller can continue with the function\'s computed outcome. + return 0 if passed else 1 + + +# What: gate on name before exit and sys and main; why: qualify admits exit and sys and main only for this predicate and excludes the opposite state. +if __name__ == "__main__": + # What: call sys.exit with main; why: qualify invokes sys.exit while performing the enclosing return; the call advances that operation through its result or side effect. + sys.exit(main()) diff --git a/benchmarks/swap/qualify_native_recovery.py b/benchmarks/swap/qualify_native_recovery.py new file mode 100644 index 0000000000..c10ee5c256 --- /dev/null +++ b/benchmarks/swap/qualify_native_recovery.py @@ -0,0 +1,304 @@ +"""Opt-in real-model daemon recovery test. Raw artifacts must remain private.""" +# What: document opt in real model daemon recovery test raw in the qualify_native_recovery docstring; why: introspection and maintainers read this exact docstring fragment to understand qualify native recovery behavior without executing it. + +# What: import argparse for main using argparse; why: main uses argparse argument parser, making that imported dependency available to its named operation. +import argparse +# What: import from concurrent futures import ThreadPoolExecutor; why: this module calls or annotates these symbols in the branch-created operations below. +from concurrent.futures import ThreadPoolExecutor +# What: import json for main using json; why: main uses json dumps, making that imported dependency available to its named operation. +import json +# What: import os for main using os; why: main uses os environ copy, making that imported dependency available to its named operation. +import os +# What: import path for main using pathlib and path; why: main uses path, making that imported dependency available to its named operation. +from pathlib import Path +# What: import signal for main using signal; why: main uses signal signal, making that imported dependency available to its named operation. +import signal +# What: import subprocess for main using subprocess; why: main uses subprocess run, making that imported dependency available to its named operation. +import subprocess +# What: import sys for module initialization using sys; why: module initialization uses sys exit, making that imported dependency available to its named operation. +import sys + +# What: import canary and http and require expected hostname and wait health for main using qualify and canary and http and require expected hostname and wait health; why: main uses canary and http and require expected hostname and wait health, making that imported dependency available to its named operation. +from qualify import canary, http, require_expected_hostname, wait_health + + +# What: define main around the current object state; why: its direct callers call main for main and rely on this exact input and result contract. +def main(): + # What: compute parser from argument parser and argparse and doc; why: parser add argument name required later reads parser, so main must retain the computed value under that name. + parser = argparse.ArgumentParser(description=__doc__) + # What: iterate across the computed value to perform add argument and parser and name; why: main repeats the body only while or for the loop header admits an iteration. + for name in ("source", "daemon-source", "python", "model", "extensions-dir", + # What: apply the protected service protected url artifacts expected hostname portion of the enclosing predicate; why: this clause remains in main\'s enclosing expression so its grouping and evaluation order stay intact. + "protected-service", "protected-url", "artifacts", "expected-hostname"): + # What: register the parser add argument name required True command-line option; why: main validates this operator input before starting the qualification sequence. + parser.add_argument("--" + name, required=True) + # What: register the parser add argument allow maintenance action store true required True command-line option; why: main validates this operator input before starting the qualification sequence. + parser.add_argument("--allow-maintenance", action="store_true", required=True) + # What: register the parser add argument port type int default 1963 command-line option; why: main validates this operator input before starting the qualification sequence. + parser.add_argument("--port", type=int, default=1963) + # What: compute args from parse args and parser; why: require expected hostname args expected hostname later reads args, so main must retain the computed value under that name. + args = parser.parse_args() + # What: call require_expected_hostname with expected hostname and args; why: main invokes require_expected_hostname while performing sys path insert str path args daemon source python; the call advances that operation through its result or side effect. + require_expected_hostname(args.expected_hostname) + # What: preserve the exact sys path insert str path args daemon source python literal fragment; why: main passes this fragment verbatim through sys.path.insert(0, str(Path(args.daemon_source) / "python")), because changing it would alter a protocol payload, serialized fixture, or public message. + sys.path.insert(0, str(Path(args.daemon_source) / "python")) + # What: import test client for main using fastapi and testclient and test client; why: main uses test client, making that imported dependency available to its named operation. + from fastapi.testclient import TestClient + # What: import build app for main using freetoken and daemon and app and build app; why: main uses build app, making that imported dependency available to its named operation. + from freetoken.daemon.app import build_app + # What: import model catalog for main using freetoken and daemon and catalog and model catalog; why: main uses model catalog load, making that imported dependency available to its named operation. + from freetoken.daemon.catalog import ModelCatalog + # What: import log ring for main using freetoken and daemon and logring and log ring; why: main uses log ring, making that imported dependency available to its named operation. + from freetoken.daemon.logring import LogRing + # What: import serve state store for main using freetoken and daemon and pidfile and serve state store; why: main uses serve state store, making that imported dependency available to its named operation. + from freetoken.daemon.pidfile import ServeStateStore + # What: import serve probe for main using freetoken and daemon and proxy and serve probe; why: main uses serve probe, making that imported dependency available to its named operation. + from freetoken.daemon.proxy import ServeProbe + # What: import popen child and serve manager for spawn and main using freetoken and daemon and serve manager and popen child and serve manager; why: spawn and main uses popen child and serve manager, making that imported dependency available to its named operation. + from freetoken.daemon.serve_manager import PopenChild, ServeManager + + # What: compute artifacts from path and artifacts and args; why: artifacts mkdir parents exist ok later reads artifacts, so main must retain the computed value under that name. + artifacts = Path(args.artifacts) + # What: supply parents to artifacts.mkdir; why: main binds this true value to artifacts.mkdir's parents input. + artifacts.mkdir(parents=True, exist_ok=False) + # What: map the passed field as false; why: main carries passed through status into status baseline health wait health args protected url health 10. + status = {"passed": False, "restored": False} + # What: compute service from sudo and n and systemctl; why: subprocess run service is active quiet args protected service check later reads service, so main must retain the computed value under that name. + service = ["sudo", "-n", "systemctl"] + # What: execute subprocess run service is active quiet args protected service check True; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + subprocess.run(service + ["is-active", "--quiet", args.protected_service], check=True) + # What: compute status entry from wait health and protected url and args and 10 and health; why: status initial response json later reads status entry, so main must retain the computed value under that name. + status["baselineHealth"] = wait_health(args.protected_url + "/health", 10) + # What: compute protected model from loads and json and http and protected url; why: raw content canary args protected url protected model later reads protected model, so main must retain the computed value under that name. + protected_model = json.loads(http(args.protected_url + "/v1/models"))["data"][0]["id"] + # What: evaluate and capture raw content canary args protected url protected model; why: the enclosing qualifier uses the captured result in its next validation or artifact step. + raw, content = canary(args.protected_url, protected_model) + # What: execute artifacts baseline json write bytes raw; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + (artifacts / "baseline.json").write_bytes(raw) + # What: assert that content equals 4; why: main requires content equals 4 to be true, so a false result stops the invalid state. + assert content == "4", "baseline failed; no maintenance performed" + # What: compute env from copy and environ and os; why: env pythonpath str path args source python later reads env, so main must retain the computed value under that name. + env = os.environ.copy() + # What: compute env entry from str and path and source and args and python; why: env torch extensions dir args extensions dir later reads env entry, so main must retain the computed value under that name. + env["PYTHONPATH"] = str(Path(args.source) / "python") + # What: compute env entry from extensions dir and args; why: env max jobs later reads env entry, so main must retain the computed value under that name. + env["TORCH_EXTENSIONS_DIR"] = args.extensions_dir + # What: compute env entry from 2; why: cwd args source env env stdout log later reads env entry, so main must retain the computed value under that name. + env["MAX_JOBS"] = "2" + # What: open the with artifacts kernel preflight log open wb as log resource scope; why: the qualification operation releases this resource when the guarded block exits. + with (artifacts / "kernel-preflight.log").open("wb") as log: + # What: execute subprocess run args python c from freetoken kernel gguf import module module; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + subprocess.run([args.python, "-c", "from freetoken.kernel.gguf import _module; _module()"], + # What: supply cwd to subprocess.run; why: main binds this source and args value to subprocess.run's cwd input. + cwd=args.source, env=env, stdout=log, stderr=subprocess.STDOUT, + # What: supply check to subprocess.run; why: main binds this true value to subprocess.run's check input. + check=True, timeout=600) + # Deliberately corrupt test artifact, never an existing model file. + # What: compute bad model from artifacts and invalid test model and gguf; why: bad model write bytes b invalid gguf test fixture later reads bad model, so main must retain the computed value under that name. + bad_model = artifacts / "invalid-test-model.gguf" + # What: call bad_model.write_bytes with the named fixture input; why: main invokes bad_model.write_bytes while performing common host served model name native recovery; the call advances that operation through its result or side effect. + bad_model.write_bytes(b"INVALID_GGUF_TEST_FIXTURE") + # What: compute common from host and 127 0 0 1 and served model name and native recovery and max seq len override; why: f ready timeout s nargs json dumps common n later reads common, so main must retain the computed value under that name. + common = ["--host", "127.0.0.1", "--served-model-name", "native-recovery", + # What: apply the max seq len override num tokens portion of common; why: main uses this clause to evaluate common as one grouped value. + "--max-seq-len-override", "4096", "--num-tokens", "4096", + # What: apply the max prefill length max running requests portion of common; why: main uses this clause to evaluate common as one grouped value. + "--max-prefill-length", "512", "--max-running-requests", "1", + # What: apply the graph memory ratio attention backend triton portion of common; why: main uses this clause to evaluate common as one grouped value. + "--graph", "1", "--memory-ratio", "0.75", "--attention-backend", "triton", + # What: apply the moe backend fused disable pynccl portion of common; why: main uses this clause to evaluate common as one grouped value. + "--moe-backend", "fused", "--disable-pynccl"] + # What: compute catalog path from artifacts and models and toml; why: catalog path write text n join later reads catalog path, so main must retain the computed value under that name. + catalog_path = artifacts / "models.toml" + # What: preserve the exact catalog path write text n join literal fragment; why: main passes this fragment verbatim through catalog_path.write_text("\n".join(, because changing it would alter a protocol payload, serialized fixture, or public message. + catalog_path.write_text("\n".join( + # What: preserve the exact f models name nmodel json dumps str literal fragment; why: main passes this fragment verbatim through f"[models.{name}]\nmodel = {json.dumps(str(model))}\nport = {args.port}\, because changing it would alter a protocol payload, serialized fixture, or public message. + # What: preserve the exact f ready timeout s nargs json dumps common n literal fragment; why: main passes this fragment verbatim through f"[models.{name}]\nmodel = {json.dumps(str(model))}\nport = {args.port}\, because changing it would alter a protocol payload, serialized fixture, or public message. + f"[models.{name}]\nmodel = {json.dumps(str(model))}\nport = {args.port}\n" + f"ready_timeout_s = 600\nargs = {json.dumps(common)}\n" + # What: preserve the exact for name model in good args model literal fragment; why: main passes this fragment verbatim through for name, model in (("good", args.model), ("bad", bad_model))), encoding, because changing it would alter a protocol payload, serialized fixture, or public message. + for name, model in (("good", args.model), ("bad", bad_model))), encoding="utf-8") + # What: initialize children as an empty runtime accumulator; why: main appends or maps entries into it during log path artifacts f engine len children log before consuming the aggregate. + children = [] + + # What: define spawn around model and port and launch args; why: its direct callers call spawn for spawn and rely on this exact input and result contract. + def spawn(model, port, launch_args): + # What: compute log path from artifacts and len and children and engine and log; why: with log path open wb as log later reads log path, so spawn must retain the computed value under that name. + log_path = artifacts / f"engine-{len(children)}.log" + # What: enter the log_path.open managed context before proc subprocess popen args python m freetoken cli serve; why: spawn releases this resource or lock after proc subprocess popen args python m freetoken cli serve on both success and failure paths. + with log_path.open("wb") as log: + # What: compute proc from popen and subprocess and python and model; why: child popen child proc str log path later reads proc, so spawn must retain the computed value under that name. + proc = subprocess.Popen([args.python, "-m", "freetoken.cli", "serve", + # What: call str with port; why: spawn invokes str while performing cwd args source env env stdout log; the call advances that operation through its result or side effect. + "--model", model, "--port", str(port), *launch_args], + # What: supply cwd to subprocess.Popen; why: spawn binds this source and args value to subprocess.Popen's cwd input. + cwd=args.source, env=env, stdout=log, stderr=subprocess.STDOUT, + # What: supply stdin to subprocess.Popen; why: spawn binds this devnull and subprocess value to subprocess.Popen's stdin input. + stdin=subprocess.DEVNULL, start_new_session=True) + # What: compute child from popen child and proc and str and log path; why: children append child later reads child, so spawn must retain the computed value under that name. + child = PopenChild(proc, str(log_path)) + # What: call children.append with child; why: spawn invokes children.append while performing return child; the call advances that operation through its result or side effect. + children.append(child) + # What: return child from spawn; why: spawn exposes child so its caller can continue with the function\'s computed outcome. + return child + + # What: compute probe from serve probe; why: prepare stop probe prepare stop read stats probe fresh stats later reads probe, so main must retain the computed value under that name. + probe = ServeProbe() + # What: compute ring from log ring; why: manager serve manager ring store spawn fn spawn later reads ring, so main must retain the computed value under that name. + ring = LogRing() + # What: compute store from serve state store and str and artifacts and serve and json; why: manager serve manager ring store spawn fn spawn later reads store, so main must retain the computed value under that name. + store = ServeStateStore(str(artifacts / "serve.json")) + # What: compute manager from serve manager and ring and store and spawn; why: app build app manager manager ring ring later reads manager, so main must retain the computed value under that name. + manager = ServeManager(ring, store, spawn_fn=spawn, apply_oom=False, + # What: supply prepare stop to ServeManager; why: main binds this prepare stop and probe value to ServeManager's prepare stop input. + prepare_stop=probe.prepare_stop, read_stats=probe.fresh_stats, + # What: supply grace s to ServeManager; why: main binds this 30 value to ServeManager's grace s input. + grace_s=30, reap_wait_s=15) + # What: compute maintenance from false; why: maintenance later reads maintenance, so main must retain the computed value under that name. + maintenance = False + + # What: define interrupt around the current object state; why: its direct callers call interrupt for interrupt and rely on this exact input and result contract. + def interrupt(*_): + # What: propagate the active failure to the caller; why: interrupt stops this rejected path before it can mutate state, dispatch work, or report success. + raise KeyboardInterrupt + + # What: call signal.signal with sigterm and signal and interrupt; why: main invokes signal.signal while performing signal signal signal sighup interrupt; the call advances that operation through its result or side effect. + signal.signal(signal.SIGTERM, interrupt) + # What: call signal.signal with sighup and signal and interrupt; why: main invokes signal.signal while performing try; the call advances that operation through its result or side effect. + signal.signal(signal.SIGHUP, interrupt) + # What: establish the handler boundary for the protected operation; why: main routes failures to base exception while preserving cleanup and success flow. + try: + # What: compute maintenance from true; why: if maintenance later reads maintenance, so main must retain the computed value under that name. + maintenance = True + # What: execute subprocess run service stop args protected service check True timeout 90; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + subprocess.run(service + ["stop", args.protected_service], check=True, timeout=90) + # What: preserve the exact print native maintenance started flush literal fragment; why: main passes this fragment verbatim through print("NATIVE_MAINTENANCE_STARTED", flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print("NATIVE_MAINTENANCE_STARTED", flush=True) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app manager manager ring ring; why: main releases this resource or lock after app build app manager manager ring ring on both success and failure paths. + with ThreadPoolExecutor(2) as lifecycle, ThreadPoolExecutor(2) as proxy: + # What: declare the pid input for main; why: main consumes pid during signature binding, so callers must bind it with the other signature inputs. + app = build_app(manager=manager, ring=ring, probe=probe, footprint_fn=lambda pid: {}, + # What: supply lifecycle pool to build_app; why: main binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, + # What: supply catalog to ModelCatalog.load; why: main binds this load and model catalog and str and catalog path value to ModelCatalog.load's catalog input. + catalog=ModelCatalog.load(str(catalog_path))) + # What: enter the TestClient managed context before response client post engine start profile json name; why: main releases this resource or lock after response client post engine start profile json name on both success and failure paths. + with TestClient(app) as client: + # What: map the name field as good; why: main carries name through response into status initial response json. + response = client.post("/engine/start-profile", json={"name": "good"}) + # What: compute status entry from json and response; why: status failed switch response json later reads status entry, so main must retain the computed value under that name. + status["initial"] = response.json() + # What: assert that response status code equals 200 and response json readiness ready; why: main requires response status code equals 200 and response json readiness ready to be true, so a false result stops the invalid state. + assert response.status_code == 200 and response.json()["readiness"]["ready"] + # What: compute raw and content from canary and port and args and native recovery and http; why: artifacts before failure json write bytes raw later reads raw and content, so main must retain the computed value under that name. + raw, content = canary(f"http://127.0.0.1:{args.port}", "native-recovery") + # What: preserve the exact artifacts before failure json write bytes raw literal fragment; why: main passes this fragment verbatim through (artifacts / "before-failure.json").write_bytes(raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / "before-failure.json").write_bytes(raw) + # What: assert that content equals 4; why: main requires content equals 4 to be true, so a false result stops the invalid state. + assert content == "4" + # What: preserve the exact print native baseline ok flush literal fragment; why: main passes this fragment verbatim through print("NATIVE_BASELINE_OK", flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print("NATIVE_BASELINE_OK", flush=True) + # What: map the name field as bad; why: main carries name through response into status failed switch response json. + response = client.post("/engine/switch-profile", json={"name": "bad"}) + # What: compute status entry from json and response; why: status accounting manager pending accounting later reads status entry, so main must retain the computed value under that name. + status["failedSwitch"] = response.json() + # What: assert that response status code equals 503; why: main requires response status code equals 503 to be true, so a false result stops the invalid state. + assert response.status_code == 503, "failed model must not report success" + # What: compute rollback from json and response and rollback; why: assert rollback launched and rollback readiness later reads rollback, so main must retain the computed value under that name. + rollback = response.json()["rollback"] + # What: assert that rollback launched and rollback readiness ready; why: main requires rollback launched and rollback readiness ready to be true, so a false result stops the invalid state. + assert rollback["launched"] and rollback["readiness"]["ready"] + # What: assert that len children equals 3 and children 1 proc poll not in; why: main requires len children equals 3 and children 1 proc poll not in to be true, so a false result stops the invalid state. + assert len(children) == 3 and children[1].proc.poll() not in (None, 0) + # What: require GGUF magic invalid in artifacts engine 1 log read text errors replace; why: the qualifier stops immediately when this protected invariant is false. + # What: require GGUF magic invalid in artifacts engine 1 log read text errors replace; why: the qualifier stops immediately when this protected invariant is false. + assert "GGUF magic invalid" in (artifacts / "engine-1.log").read_text(errors="replace"), \ + "replacement must fail for the intended invalid-GGUF reason" + # What: assert that store load model equals args model; why: main requires store load model equals args model to be true, so a false result stops the invalid state. + assert store.load().model == args.model + # What: compute raw and content from canary and port and args and native recovery and true; why: artifacts after recovery sse write bytes raw later reads raw and content, so main must retain the computed value under that name. + raw, content = canary(f"http://127.0.0.1:{args.port}", "native-recovery", True) + # What: preserve the exact artifacts after recovery sse write bytes raw literal fragment; why: main passes this fragment verbatim through (artifacts / "after-recovery.sse").write_bytes(raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / "after-recovery.sse").write_bytes(raw) + # What: assert that content equals 4; why: main requires content equals 4 to be true, so a false result stops the invalid state. + assert content == "4", "restored model failed generation" + # What: compute status entry from pending accounting and manager; why: for row in status accounting previous later reads status entry, so main must retain the computed value under that name. + status["accounting"] = manager.pending_accounting() + # What: require any row get drainComplete and not row get degraded; why: the qualifier stops immediately when this protected invariant is false. + assert any(row.get("drainComplete") and not row.get("degraded") + # What: execute for row in status accounting previous engine receipt must be sealed; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + for row in status["accounting"]), "previous engine receipt must be sealed" + # What: require any row get reason == engine crashed and row get degraded; why: the qualifier stops immediately when this protected invariant is false. + assert any(row.get("reason") == "engine-crashed" and row.get("degraded") + # What: execute for row in status accounting loader failure must retain explicit crash accounting; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + for row in status["accounting"]), "loader failure must retain explicit crash accounting" + # What: compute status entry from true; why: status error repr exc later reads status entry, so main must retain the computed value under that name. + status["passed"] = True + # What: preserve the exact print native model recovery ok flush literal fragment; why: main passes this fragment verbatim through print("NATIVE_MODEL_RECOVERY_OK", flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print("NATIVE_MODEL_RECOVERY_OK", flush=True) + # What: handle base exception by status error repr exc; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException as exc: + # What: compute status entry from repr and exc; why: status cleanup error repr exc later reads status entry, so main must retain the computed value under that name. + status["error"] = repr(exc) + # What: preserve the exact print native recovery failed repr exc flush literal fragment; why: main passes this fragment verbatim through print("NATIVE_RECOVERY_FAILED " + repr(exc), flush=True), because changing it would alter a protocol payload, serialized fixture, or public message. + print("NATIVE_RECOVERY_FAILED " + repr(exc), flush=True) + # What: run try on every exit path; why: main performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: establish the handler boundary for the protected operation; why: main routes failures to exception while preserving cleanup and success flow. + try: + # What: supply force to manager.stop; why: main binds this true value to manager.stop's force input. + manager.stop(force=True) + # What: handle exception by status cleanup error repr exc; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: + # What: compute status entry from repr and exc; why: status cleanup error repr exc later reads status entry, so main must retain the computed value under that name. + status["cleanupError"] = repr(exc) + # What: iterate across children to perform process lookup error and oserror and timeout expired and killpg and pid; why: main repeats the body only while or for the loop header admits an iteration. + for child in children: + # What: establish the handler boundary for the protected operation; why: main routes failures to oserror and timeout expired and subprocess while preserving cleanup and success flow. + try: + # What: establish the handler boundary for the protected operation; why: main routes failures to process lookup error while preserving cleanup and success flow. + try: + # What: call os.killpg with pid and child and sigkill and signal; why: main invokes os.killpg while performing except process lookup error; the call advances that operation through its result or side effect. + os.killpg(child.pid, signal.SIGKILL) + # What: handle process lookup error by pass; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except ProcessLookupError: + # What: ignore the anticipated exception handled by this branch; why: interrupt continues its retry or cleanup path instead of re-raising that transient failure. + pass + # What: gate on poll and proc and child before wait and proc and child; why: main admits wait and proc and child only for this predicate and excludes the opposite state. + if child.proc.poll() is None: + # What: supply timeout to child.proc.wait; why: main binds this 15 value to child.proc.wait's timeout input. + child.proc.wait(timeout=15) + # What: handle oserror and timeout expired and subprocess by status cleanup error repr exc; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except (OSError, subprocess.TimeoutExpired) as exc: + # What: compute status entry from repr and exc; why: status restored health wait health args protected url health later reads status entry, so main must retain the computed value under that name. + status["cleanupError"] = repr(exc) + # What: gate on maintenance before exception and run and status and wait health and raw; why: main admits exception and run and status and wait health and raw only for this predicate and excludes the opposite state. + if maintenance: + # What: establish the handler boundary for the protected operation; why: main routes failures to exception while preserving cleanup and success flow. + try: + # What: execute subprocess run service start args protected service check True timeout 180; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + subprocess.run(service + ["start", args.protected_service], check=True, timeout=180) + # What: execute status restoredHealth wait health args protected url health 300; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + status["restoredHealth"] = wait_health(args.protected_url + "/health", 300) + # What: evaluate and capture raw content canary args protected url protected model; why: the enclosing qualifier uses the captured result in its next validation or artifact step. + raw, content = canary(args.protected_url, protected_model) + # What: execute artifacts restored json write bytes raw; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + (artifacts / "restored.json").write_bytes(raw) + # What: compute status entry from content and 4; why: print restored str status restored flush later reads status entry, so main must retain the computed value under that name. + status["restored"] = content == "4" + # What: execute print RESTORED str status restored flush True; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + print("RESTORED " + str(status["restored"]), flush=True) + # What: handle exception by status restore error repr exc; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: + # What: compute status entry from repr and exc; why: artifacts result json write text json dumps status indent later reads status entry, so main must retain the computed value under that name. + status["restoreError"] = repr(exc) + # What: preserve the exact artifacts result json write text json dumps status indent literal fragment; why: main passes this fragment verbatim through (artifacts / "result.json").write_text(json.dumps(status, indent=2), enc, because changing it would alter a protocol payload, serialized fixture, or public mess. + (artifacts / "result.json").write_text(json.dumps(status, indent=2), encoding="utf-8") + # What: return status and 0 and 1 and passed and restored from main; why: main exposes status and 0 and 1 and passed and restored so its caller can continue with the function\'s computed outcome. + return 0 if status["passed"] and status["restored"] and "cleanupError" not in status else 1 + + +# What: gate on name before exit and sys and main; why: qualify_native_recovery admits exit and sys and main only for this predicate and excludes the opposite state. +if __name__ == "__main__": + # What: call sys.exit with main; why: qualify_native_recovery invokes sys.exit while performing the enclosing return; the call advances that operation through its result or side effect. + sys.exit(main()) diff --git a/benchmarks/swap/qualify_native_router.py b/benchmarks/swap/qualify_native_router.py new file mode 100644 index 0000000000..8fb4181088 --- /dev/null +++ b/benchmarks/swap/qualify_native_router.py @@ -0,0 +1,2745 @@ +"""Opt-in native freetoken-swap routing benchmark for an approved Linux window. + +It keeps raw requests, responses, daemon logs, catalog paths, and host details +inside a newly created private artifact directory. It never changes protected +service enablement or configuration, and always attempts restoration after a +maintenance stop. This is evidence collection, not a production launcher. +""" +# What: document opt in native freetoken swap routing benchmark for in the qualify_native_router docstring; why: introspection and maintainers read this exact docstring fragment to understand qualify native router behavior without executing it. +# What: document it keeps raw requests responses daemon in the qualify_native_router docstring; why: introspection and maintainers read this exact docstring fragment to understand qualify native router behavior without executing it. +# What: document inside a newly created private artifact in the qualify_native_router docstring; why: introspection and maintainers read this exact docstring fragment to understand qualify native router behavior without executing it. +# What: document service enablement or configuration and always in the qualify_native_router docstring; why: introspection and maintainers read this exact docstring fragment to understand qualify native router behavior without executing it. +# What: document maintenance stop this is evidence collection in the qualify_native_router docstring; why: introspection and maintainers read this exact docstring fragment to understand qualify native router behavior without executing it. +# What: preserve the paragraph boundary in the the qualify_native_router docstring; why: introspection and maintainers read this paragraph break to understand qualify native router behavior without executing it. + +# What: enable postponed evaluation of annotations; why: type hints in qualify_native_router can reference runtime types without eager imports or forward-reference failures. +from __future__ import annotations + +# What: import argparse for main using argparse; why: main uses argparse argument parser, making that imported dependency available to its named operation. +import argparse + +# What: import base64 for control plane canary using base64; why: control_plane_canary uses base64 b64encode, making that imported dependency available to its named operation. +import base64 + +# What: import json for request json using json; why: request_json uses json loads, making that imported dependency available to its named operation. +import json + +# What: import os for stop process group using os; why: stop_process_group uses os killpg, making that imported dependency available to its named operation. +import os + +# What: import secrets for main using secrets; why: main uses secrets token urlsafe, making that imported dependency available to its named operation. +import secrets + +# What: import signal for stop process group using signal; why: stop_process_group uses signal sigterm, making that imported dependency available to its named operation. +import signal + +# What: import socket for require expected hostname using socket; why: require_expected_hostname uses socket gethostname, making that imported dependency available to its named operation. +import socket + +# What: import subprocess for stop process group using subprocess; why: stop_process_group uses subprocess timeout expired, making that imported dependency available to its named operation. +import subprocess + +# What: import sys for main using sys; why: main uses sys platform startswith, making that imported dependency available to its named operation. +import sys + +# What: import threading for concurrent canaries using threading; why: concurrent_canaries uses threading lock, making that imported dependency available to its named operation. +import threading + +# What: import time for canary using time; why: canary uses time monotonic, making that imported dependency available to its named operation. +import time + +# What: import urllib error for request json using urllib and error; why: request_json uses urllib request request, making that imported dependency available to its named operation. +import urllib.error + +# What: import urllib request for request json using urllib and request; why: request_json uses urllib request request, making that imported dependency available to its named operation. +import urllib.request + +# What: import path for upstream model rewrite canary using pathlib and path; why: upstream_model_rewrite_canary uses the path annotation in upstream model rewrite canary, making that imported dependency available to its named operation. +from pathlib import Path + +# What: compute native auth base from the named fixture input; why: global native auth base native api key later reads native auth base, so qualify_native_router must retain the computed value under that name. +_NATIVE_AUTH_BASE: str | None = None +# What: compute native api key from the named fixture input; why: global native auth base native api key later reads native api key, so qualify_native_router must retain the computed value under that name. +_NATIVE_API_KEY: str | None = None + + +# What: define require_expected_hostname and its declared inputs; why: callers use require_expected_hostname to perform the behavior named by this helper without duplicating its boundary checks. +def require_expected_hostname(expected: str, *, actual: str | None = None) -> str: + """Require an exact operator-supplied host without disclosing either name.""" + # What: document require an exact operator supplied host without in the require_expected_hostname docstring; why: introspection and maintainers read this exact docstring fragment to understand require expected hostname behavior without executing it. + # What: evaluate and capture actual socket gethostname if actual is None else actual; why: the enclosing qualifier uses the captured result in its next validation or artifact step. + actual = socket.gethostname() if actual is None else actual + # What: gate on expected and actual before runtime error; why: require_expected_hostname admits runtime error only for this predicate and excludes the opposite state. + if not expected or "\x00" in expected or actual != expected: + # What: raise RuntimeError for the caller; why: require_expected_hostname stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError( + # What: execute qualification host does not match the operator supplied expected hostname; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + "qualification host does not match the operator-supplied expected hostname" + # What: complete the RuntimeError call with ordered positional inputs; why: require_expected_hostname groups the supplied clauses as one RuntimeError call before its value is consumed. + ) + # What: return actual from require_expected_hostname; why: require_expected_hostname exposes actual so its caller can continue with the function\'s computed outcome. + return actual + + +# What: define the protected service command selector; why: qualification must restore either a system or user-owned workload through its real manager. +def protected_service_command(scope: str) -> list[str]: + # What: select the system service manager; why: existing qualification callers retain their passwordless sudo behavior by default. + if scope == "system": + # What: return the non-interactive systemctl prefix; why: maintenance must never block waiting for a password prompt. + return ["sudo", "-n", "systemctl"] + # What: select the invoking user's service manager; why: the target AMD host protects Nemotron with a user-scoped unit. + if scope == "user": + # What: return the user systemctl prefix; why: stopping the wrong system scope would fail restoration or touch unrelated services. + return ["systemctl", "--user"] + # What: reject unrecognized service scope; why: lifecycle ownership must be explicit before any maintenance mutation. + raise ValueError("protected service scope must be 'system' or 'user'") + + +# What: define configure_native_auth around base and api key; why: its direct callers call configure_native_auth for configure native auth and rely on this exact input and result contract. +def configure_native_auth(base: str, api_key: str) -> None: + """Scope private router credentials to the exact temporary daemon origin.""" + # What: document scope private router credentials to the in the configure_native_auth docstring; why: introspection and maintainers read this exact docstring fragment to understand configure native auth behavior without executing it. + # What: apply the global native auth base native api key portion of the enclosing predicate; why: this clause remains in configure_native_auth\'s enclosing expression so its grouping and evaluation order stay intact. + global _NATIVE_AUTH_BASE, _NATIVE_API_KEY + # What: compute native auth base from rstrip and base and value; why: the enclosing return or state update later reads native auth base, so configure_native_auth must retain the computed value under that name. + _NATIVE_AUTH_BASE = base.rstrip("/") + # What: compute native api key from api key; why: the enclosing return or state update later reads native api key, so configure_native_auth must retain the computed value under that name. + _NATIVE_API_KEY = api_key + + +# What: define _native_headers around url; why: its direct callers call _native_headers for native headers and rely on this exact input and result contract. +def _native_headers(url: str) -> dict[str, str]: + # What: gate on native auth base and native api key and url and startswith before native api key; why: _native_headers admits native api key only for this predicate and excludes the opposite state. + if ( + # What: apply the native auth base is not portion of the enclosing predicate; why: this clause remains in _native_headers\'s enclosing expression so its grouping and evaluation order stay intact. + _NATIVE_AUTH_BASE is not None + # What: apply the and native api key is not portion of the enclosing predicate; why: this clause remains in _native_headers\'s enclosing expression so its grouping and evaluation order stay intact. + and _NATIVE_API_KEY is not None + # What: call url.startswith with native auth base and value; why: _native_headers consumes the url.startswith return value while evaluating and (url == _NATIVE_AUTH_BASE or url.startswith(_NATIVE_AUTH_BASE + "/"). + and (url == _NATIVE_AUTH_BASE or url.startswith(_NATIVE_AUTH_BASE + "/")) + # What: complete the enclosing predicate with if native auth base is not and native api key is not and; why: _native_headers groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: map the authorization field as native api key and bearer; why: _native_headers carries authorization into return {"Authorization": f"Bearer {_NATIVE_API_KEY}"}. + return {"Authorization": f"Bearer {_NATIVE_API_KEY}"} + # What: return no value from _native_headers; why: _native_headers returns no value to callers that depend on its completed result. + return {} + + +# What: define request_json around url and body and timeout and method; why: its direct callers call request_json for request json and rely on this exact input and result contract. +def request_json( + # What: declare the url input for request_json; why: request_json consumes url during url, so callers must bind it with the other signature inputs. + url: str, + # What: declare the body input for request_json; why: request_json consumes body during data if body is else json dumps, so callers must bind it with the other signature inputs. + body: dict | None = None, + # What: mark the remaining parameters as keyword-only; why: request_json prevents callers from confusing adjacent lifecycle and timing arguments. + *, + # What: declare the timeout input for request_json; why: request_json consumes timeout during with urllib request urlopen request timeout timeout as, so callers must bind it with the other signature inputs. + timeout: float = 30, + # What: declare the method input for request_json; why: request_json consumes method during method method, so callers must bind it with the other signature inputs. + method: str | None = None, +# What: complete the enclosing predicate collection with bytes and dict; why: request_json groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. +) -> tuple[bytes, dict]: + # What: compute data from body and encode and dumps and json and utf 8; why: data data later reads data, so request_json must retain the computed value under that name. + data = None if body is None else json.dumps(body).encode("utf-8") + # What: begin with origin-scoped authentication headers; why: read-only GET requests must not falsely declare an absent body as JSON. + headers = _native_headers(url) + # What: gate the JSON content type on an actual serialized body; why: the router correctly rejects empty requests that claim to contain JSON. + if data is not None: + # What: add the JSON media type while preserving authentication; why: body-bearing management requests still require explicit and valid content metadata. + headers = {"Content-Type": "application/json", **headers} + # What: compute request from request and url and request and data; why: with urllib request urlopen request timeout timeout as later reads request, so request_json must retain the computed value under that name. + request = urllib.request.Request( + # What: apply the url portion of request; why: request_json uses this clause to evaluate request as one grouped value. + url, + # What: supply data to urllib.request.Request; why: request_json binds this data value to urllib.request.Request's data input. + data=data, + # What: supply the body-aware headers; why: request_json must authenticate every native request without mislabeling empty GET requests as JSON payloads. + headers=headers, + # What: supply method to urllib.request.Request; why: request_json binds this method value to urllib.request.Request's method input. + method=method, + # What: complete the urllib.request.Request call with data and headers and method; why: request_json groups the supplied clauses as one urllib.request.Request call before its value is consumed. + ) + # What: enter the urllib.request.urlopen managed context before raw response read; why: request_json releases this resource or lock after raw response read on both success and failure paths. + with urllib.request.urlopen(request, timeout=timeout) as response: + # What: compute raw from read and response; why: return raw json loads raw later reads raw, so request_json must retain the computed value under that name. + raw = response.read() + # What: return raw and loads and json from request_json; why: request_json exposes raw and loads and json so its caller can continue with the function\'s computed outcome. + return raw, json.loads(raw) + + +# What: define request_bytes around url and timeout; why: its direct callers call request_bytes for request bytes and rely on this exact input and result contract. +def request_bytes(url: str, *, timeout: float = 30) -> bytes: + # What: compute request from request and url and request and urllib; why: with urllib request urlopen request timeout timeout as later reads request, so request_bytes must retain the computed value under that name. + request = urllib.request.Request(url, headers=_native_headers(url)) + # What: enter the urllib.request.urlopen managed context before return response read; why: request_bytes releases this resource or lock after return response read on both success and failure paths. + with urllib.request.urlopen(request, timeout=timeout) as response: + # What: return read and response from request_bytes; why: request_bytes exposes read and response so its caller can continue with the function\'s computed outcome. + return response.read() + + +# What: define wait_json around url and seconds; why: its direct callers call wait_json for wait json and rely on this exact input and result contract. +def wait_json(url: str, *, seconds: float) -> dict: + # What: compute deadline from seconds and monotonic and time; why: while time monotonic deadline later reads deadline, so wait_json must retain the computed value under that name. + deadline = time.monotonic() + seconds + # What: compute last from the named fixture input; why: last exc later reads last, so wait_json must retain the computed value under that name. + last: Exception | None = None + # What: iterate across deadline and monotonic and time to perform oserror and value error and httperror and last and exc; why: wait_json repeats the body only while or for the loop header admits an iteration. + while time.monotonic() < deadline: + # What: establish the handler boundary for the protected operation; why: wait_json routes failures to oserror and value error and httperror and error and urllib while preserving cleanup and success flow. + try: + # What: return request json and url and 1 and 3 from wait_json; why: wait_json exposes request json and url and 1 and 3 so its caller can continue with the function\'s computed outcome. + return request_json(url, timeout=3)[1] + # What: handle oserror and value error and httperror and error and urllib by last exc; why: wait_json converts that failure into this concrete recovery, response, or cleanup behavior. + except (OSError, ValueError, urllib.error.HTTPError) as exc: + # What: compute last from exc; why: raise timeout error f endpoint did not later reads last, so wait_json must retain the computed value under that name. + last = exc + # What: call time.sleep with 0 25; why: wait_json invokes time.sleep while performing raise timeout error f endpoint did not; the call advances that operation through its result or side effect. + time.sleep(0.25) + # What: raise TimeoutError for the caller; why: wait_json stops this rejected path before it can mutate state, dispatch work, or report success. + raise TimeoutError(f"endpoint did not become available: {last!r}") + + +# What: define canary around url and model and direct; why: its direct callers call canary for canary and rely on this exact input and result contract. +def canary(url: str, model: str, *, direct: bool) -> tuple[bytes, dict]: + """Make one deterministic request and retain raw bytes only in private artifacts.""" + # What: document make one deterministic request and retain in the canary docstring; why: introspection and maintainers read this exact docstring fragment to understand canary behavior without executing it. + # What: compute body from model and model and messages and temperature and max tokens; why: data json dumps body encode utf 8 later reads body, so canary must retain the computed value under that name. + body = { + # What: map the model field as model; why: canary sends this field through body so the router selects the canonical model or alias for upstream dispatch. + "model": model, + # What: map the role field as user; why: canary carries role through body into data json dumps body encode utf 8. + "messages": [{"role": "user", "content": "What is 2 + 2? Reply with only the single digit."}], + # What: map the temperature field as 0; why: canary carries temperature through body into data json dumps body encode utf 8. + "temperature": 0, + # What: map the maximum completion budget as 128 tokens; why: always-on reasoning models need enough bounded space to emit a final deterministic answer after their private analysis. + "max_tokens": 128, + # What: map the stream field as true; why: canary carries stream through body into data json dumps body encode utf 8. + "stream": True, + # What: map the include usage field as true; why: canary carries include usage through body into data json dumps body encode utf 8. + "stream_options": {"include_usage": True}, + # What: disable optional thinking while selecting low effort for always-on Harmony models; why: one bounded payload must render correctly for both Qwen-style toggles and gpt-oss's graded reasoning template. + "chat_template_kwargs": {"enable_thinking": False, "reasoning_effort": "low"}, + # What: complete the body mapping with model and messages and temperature and max tokens and stream; why: canary groups the supplied clauses as one body mapping before its value is consumed. + } + # What: compute request from request and request and url and urllib; why: with urllib request urlopen request timeout as response later reads request, so canary must retain the computed value under that name. + request = urllib.request.Request( + # What: apply the url v1 chat completions portion of request; why: canary uses this clause to evaluate request as one grouped value. + url + "/v1/chat/completions", + # What: supply data to operation.encode; why: canary binds this encode and dumps and body and json and utf 8 value to operation.encode's data input. + data=json.dumps(body).encode("utf-8"), + # What: map the content type field as application and json; why: canary carries content type through request into with urllib request urlopen request timeout 660 as response. + headers={"Content-Type": "application/json", **_native_headers(url)}, + # What: complete the urllib.request.Request call with data and headers; why: canary groups the supplied clauses as one urllib.request.Request call before its value is consumed. + ) + # What: compute raw from bytearray; why: raw extend chunk later reads raw, so canary must retain the computed value under that name. + raw = bytearray() + # What: initialize content as an empty runtime accumulator; why: canary appends or maps entries into it during value choice get delta get content or before consuming the aggregate. + content: list[str] = [] + # What: compute started from monotonic and time; why: observed s time monotonic started later reads started, so canary must retain the computed value under that name. + started = time.monotonic() + # What: compute first byte s from the named fixture input; why: if first byte s is later reads first byte s, so canary must retain the computed value under that name. + first_byte_s: float | None = None + # What: compute first token s from the named fixture input; why: if value and first token s is later reads first token s, so canary must retain the computed value under that name. + first_token_s: float | None = None + # What: compute completion tokens from the named fixture input; why: if isinstance usage dict and isinstance later reads completion tokens, so canary must retain the computed value under that name. + completion_tokens: int | None = None + # What: compute response models from set; why: response models add response model later reads response models, so canary must retain the computed value under that name. + response_models: set[str] = set() + # What: attempt the bounded HTTP request before streaming its body; why: an error response must be captured privately instead of being reduced to an opaque status code. + try: + # What: open the candidate endpoint with the existing timeout; why: successful responses still use the same bounded network contract. + response_context = urllib.request.urlopen(request, timeout=660) + # What: catch an HTTP protocol failure from the candidate; why: qualification needs the server's precise rejection reason to choose a safe corrective action. + except urllib.error.HTTPError as exc: + # What: read and decode the bounded error payload; why: the private exception text should preserve actionable diagnostics without publishing request artifacts. + error_body = exc.read(64 * 1024).decode("utf-8", errors="replace") + # What: raise a contextual runtime failure chained to the HTTP error; why: the harness must fail closed while retaining the exact private rejection evidence. + raise RuntimeError(f"canary HTTP {exc.code}: {error_body}") from exc + # What: enter the successful response managed context before reading chunks; why: canary releases the network resource on both normal completion and parse failure. + with response_context as response: + # What: iterate across response to perform observed s and float; why: canary repeats the body only while or for the loop header admits an iteration. + for chunk in response: + # What: compute observed s from the named fixture input; why: observed s time monotonic started later reads observed s, so canary must retain the computed value under that name. + observed_s: float | None = None + # What: gate on first byte s before observed s and started and monotonic and time; why: canary admits observed s and started and monotonic and time only for this predicate and excludes the opposite state. + if first_byte_s is None: + # What: compute observed s from started and monotonic and time; why: first byte s observed s later reads observed s, so canary must retain the computed value under that name. + observed_s = time.monotonic() - started + # What: compute first byte s from observed s; why: if first byte s is or first token s is later reads first byte s, so canary must retain the computed value under that name. + first_byte_s = observed_s + # What: call raw.extend with chunk; why: canary invokes raw.extend while performing if len raw; the call advances that operation through its result or side effect. + raw.extend(chunk) + # What: gate on len and raw before runtime error; why: canary admits runtime error only for this predicate and excludes the opposite state. + if len(raw) > 8 * 1024 * 1024: + # What: raise RuntimeError for the caller; why: canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("canary response exceeded private capture bound") + # What: gate on startswith and chunk and strip before event and loads and json and chunk; why: canary admits event and loads and json and chunk only for this predicate and excludes the opposite state. + if chunk.startswith(b"data: ") and chunk.strip() != b"data: [DONE]": + # What: compute event from loads and json and chunk and 6; why: response model event get model later reads event, so canary must retain the computed value under that name. + event = json.loads(chunk[6:]) + # What: compute response model from get and event and model; why: if isinstance response model str later reads response model, so canary must retain the computed value under that name. + response_model = event.get("model") + # What: gate on isinstance and response model and str before add and response model and response models; why: canary admits add and response model and response models only for this predicate and excludes the opposite state. + if isinstance(response_model, str): + # What: call response_models.add with response model; why: canary invokes response_models.add while performing usage event get usage; the call advances that operation through its result or side effect. + response_models.add(response_model) + # What: compute usage from get and event and usage; why: if isinstance usage dict and isinstance later reads usage, so canary must retain the computed value under that name. + usage = event.get("usage") + # What: gate on isinstance and usage and dict and int and get before completion tokens and usage; why: canary admits completion tokens and usage only for this predicate and excludes the opposite state. + if isinstance(usage, dict) and isinstance(usage.get("completion_tokens"), int): + # What: compute completion tokens from usage and completion tokens; why: if not isinstance completion tokens int or later reads completion tokens, so canary must retain the computed value under that name. + completion_tokens = usage["completion_tokens"] + # What: iterate across get and event to perform value and get and choice; why: canary repeats the body only while or for the loop header admits an iteration. + for choice in event.get("choices", []): + # What: compute value from get and choice and value and content and delta; why: if value and first token s is later reads value, so canary must retain the computed value under that name. + value = choice.get("delta", {}).get("content") or "" + # What: gate on value and first token s before observed s and started and monotonic and time; why: canary admits observed s and started and monotonic and time only for this predicate and excludes the opposite state. + if value and first_token_s is None: + # What: gate on observed s before observed s and started and monotonic and time; why: canary admits observed s and started and monotonic and time only for this predicate and excludes the opposite state. + if observed_s is None: + # What: compute observed s from started and monotonic and time; why: first token s observed s later reads observed s, so canary must retain the computed value under that name. + observed_s = time.monotonic() - started + # What: compute first token s from observed s; why: if first byte s is or first token s is later reads first token s, so canary must retain the computed value under that name. + first_token_s = observed_s + # What: call content.append with value; why: canary invokes content.append while performing duration s time monotonic started; the call advances that operation through its result or side effect. + content.append(value) + # What: compute duration s from started and monotonic and time; why: if first byte s is or first token s is later reads duration s, so canary must retain the computed value under that name. + duration_s = time.monotonic() - started + # What: compute answer from strip and join and content and value; why: if answer later reads answer, so canary must retain the computed value under that name. + answer = "".join(content).strip() + # What: gate on raw before runtime error; why: canary admits runtime error only for this predicate and excludes the opposite state. + if b"data: [DONE]" not in raw: + # What: raise RuntimeError for the caller; why: canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("SSE completion marker missing") + # What: gate on answer before runtime error; why: canary admits runtime error only for this predicate and excludes the opposite state. + if answer != "4": + # What: raise RuntimeError for the caller; why: canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("deterministic quality gate failed") + # What: gate on completion tokens and isinstance and int before runtime error; why: canary admits runtime error only for this predicate and excludes the opposite state. + if not isinstance(completion_tokens, int) or completion_tokens <= 0: + # What: raise RuntimeError for the caller; why: canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("streamed completion usage missing") + # What: gate on first byte s and first token s and duration s before runtime error; why: canary admits runtime error only for this predicate and excludes the opposite state. + if first_byte_s is None or first_token_s is None or duration_s <= first_token_s: + # What: raise RuntimeError for the caller; why: canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("stream timing did not permit token-throughput measurement") + # What: compute decode s from duration s and first token s; why: completion tokens per second completion tokens decode s later reads decode s, so canary must retain the computed value under that name. + decode_s = duration_s - first_token_s + # What: compute completion tokens per second from completion tokens and decode s; why: completion tokens per second completion tokens per second later reads completion tokens per second, so canary must retain the computed value under that name. + completion_tokens_per_second = completion_tokens / decode_s + # What: gate on len and response models before runtime error; why: canary admits runtime error only for this predicate and excludes the opposite state. + if len(response_models) > 1: + # What: raise RuntimeError for the caller; why: canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("SSE completion reported inconsistent upstream model names") + # What: return bytes and raw and model and first byte s from canary; why: canary exposes bytes and raw and model and first byte s so its caller can continue with the function\'s computed outcome. + return bytes(raw), { + # What: map the route field as direct and direct and native router; why: canary carries route into "route": "direct" if direct else "native_router". + "route": "direct" if direct else "native_router", + # What: map the model field as model; why: canary sends this field through "model": model so the router selects the canonical model or alias for upstream dispatch. + "model": model, + # What: map the response model field as next and iter and response models; why: canary carries response model into "responseModel": next(iter(response_models), None). + "responseModel": next(iter(response_models), None), + # What: map the first byte seconds field as first byte s; why: canary carries first byte seconds into "firstByteSeconds": first_byte_s. + "firstByteSeconds": first_byte_s, + # What: map the first token seconds field as first token s; why: canary carries first token seconds into "firstTokenSeconds": first_token_s. + "firstTokenSeconds": first_token_s, + # What: map the duration seconds field as duration s; why: canary carries duration seconds into "durationSeconds": duration_s. + "durationSeconds": duration_s, + # What: map the decode seconds field as decode s; why: canary carries decode seconds into "decodeSeconds": decode_s. + "decodeSeconds": decode_s, + # What: map the completion tokens field as completion tokens; why: canary carries completion tokens into "completionTokens": completion_tokens. + "completionTokens": completion_tokens, + # What: map the completion tokens per second field as completion tokens per second; why: canary carries completion tokens per second into "completionTokensPerSecond": completion_tokens_per_second. + "completionTokensPerSecond": completion_tokens_per_second, + # What: map the response bytes field as len and raw; why: canary carries response bytes into "responseBytes": len(raw). + "responseBytes": len(raw), + # What: map the passed field as true; why: canary carries passed into "passed": True. + "passed": True, + # What: complete the enclosing predicate collection with bytes and raw and model and first byte s and first token s and duration s; why: canary groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. + } + + +# What: define upstream_model_rewrite_canary around base and artifacts; why: its direct callers call upstream_model_rewrite_canary for upstream model rewrite canary and rely on this exact input and result contract. +def upstream_model_rewrite_canary(base: str, artifacts: Path) -> dict: + """Prove an alias is rewritten upstream without changing routing identity.""" + # What: document prove an alias is rewritten upstream in the upstream_model_rewrite_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand upstream model rewrite canary behavior without executing it. + # What: compute and before from request json and base and router and status; why: value after request json base router status later reads and before, so upstream_model_rewrite_canary must retain the computed value under that name. + _, before = request_json(base + "/router/status") + # What: compute prior activations from get and before and activations; why: if before get active profile model a or not later reads prior activations, so upstream_model_rewrite_canary must retain the computed value under that name. + prior_activations = before.get("activations") + # What: gate on get and isinstance and prior activations and int and before before runtime error; why: upstream_model_rewrite_canary admits runtime error only for this predicate and excludes the opposite state. + if before.get("activeProfile") != "model-a" or not isinstance(prior_activations, int): + # What: raise RuntimeError for the caller; why: upstream_model_rewrite_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("upstream model rewrite canary requires resident model-a") + # What: compute raw and completion from canary and base and compat and model a and false; why: artifacts upstream model rewrite sse write bytes raw later reads raw and completion, so upstream_model_rewrite_canary must retain the computed value under that name. + raw, completion = canary(base, "compat/model-a", direct=False) + # What: compute and after from request json and base and router and status; why: the enclosing return or state update later reads and after, so upstream_model_rewrite_canary must retain the computed value under that name. + _, after = request_json(base + "/router/status") + # What: gate on prior activations and get and completion and after before runtime error; why: upstream_model_rewrite_canary admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call completion.get with response model; why: upstream_model_rewrite_canary invokes completion.get while performing or after get active profile model a; the call advances that operation through its result or side effect. + completion.get("responseModel") != "model-a" + # What: call after.get with active profile; why: upstream_model_rewrite_canary invokes after.get while performing or after get active requests; the call advances that operation through its result or side effect. + or after.get("activeProfile") != "model-a" + # What: call after.get with active requests; why: upstream_model_rewrite_canary invokes after.get while performing or after get activations prior activations; the call advances that operation through its result or side effect. + or after.get("activeRequests") != 0 + # What: call after.get with activations; why: upstream_model_rewrite_canary consumes the after.get return value while evaluating or after.get("activations") != prior_activations. + or after.get("activations") != prior_activations + # What: complete the enclosing predicate with if completion get response model differs from model a or after get active profile; why: upstream_model_rewrite_canary groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: upstream_model_rewrite_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("alias was not rewritten upstream with stable routing residency") + # What: preserve the exact artifacts upstream model rewrite sse write bytes raw literal fragment; why: upstream_model_rewrite_canary passes this fragment verbatim through (artifacts / "upstream-model-rewrite.sse").write_bytes(raw), because changing it would alter a protocol payload, serialized fixture, or public m. + (artifacts / "upstream-model-rewrite.sse").write_bytes(raw) + # What: return; why: the caller consumes this value as the function’s success-path result. + return { + # What: map the requested model field as compat and model a; why: upstream_model_rewrite_canary carries requested model into "requestedModel": "compat/model-a". + "requestedModel": "compat/model-a", + # What: map the upstream response model field as model a; why: upstream_model_rewrite_canary carries upstream response model into "upstreamResponseModel": "model-a". + "upstreamResponseModel": "model-a", + # What: map the resident profile field as model a; why: upstream_model_rewrite_canary carries resident profile into "residentProfile": "model-a". + "residentProfile": "model-a", + # What: map the activation delta field as 0; why: upstream_model_rewrite_canary carries activation delta into "activationDelta": 0. + "activationDelta": 0, + # What: map the passed field as true; why: upstream_model_rewrite_canary carries passed into "passed": True. + "passed": True, + # What: complete the enclosing predicate mapping with requested model and upstream response model and resident profile and activation delta and passed; why: upstream_model_rewrite_canary groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + +# What: define validate_loading_feedback around raw and expected; why: its direct callers call validate_loading_feedback for validate loading feedback and rely on this exact input and result contract. +def validate_loading_feedback(raw: bytes, *, expected: bool) -> dict: + """Require the private SSE capture to match the expected router loading state.""" + # What: document require the private sse capture to in the validate_loading_feedback docstring; why: introspection and maintainers read this exact docstring fragment to understand validate loading feedback behavior without executing it. + # What: initialize reasoning as an empty runtime accumulator; why: validate_loading_feedback appends or maps entries into it during reasoning append value before consuming the aggregate. + reasoning: list[str] = [] + # What: iterate across splitlines and raw to perform line and startswith; why: validate_loading_feedback repeats the body only while or for the loop header admits an iteration. + for line in raw.splitlines(): + # What: gate on line and startswith before the computed value; why: validate_loading_feedback admits the computed value only for this predicate and excludes the opposite state. + if not line.startswith(b"data: ") or line == b"data: [DONE]": + # What: apply the continue portion of the enclosing predicate; why: this clause remains in validate_loading_feedback\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: establish the handler boundary for the protected operation; why: validate_loading_feedback routes failures to unicode decode error and jsondecode error and json while preserving cleanup and success flow. + try: + # What: compute event from loads and json and line and 6; why: for choice in event get choices later reads event, so validate_loading_feedback must retain the computed value under that name. + event = json.loads(line[6:]) + # What: handle unicode decode error and jsondecode error and json by continue; why: validate_loading_feedback converts that failure into this concrete recovery, response, or cleanup behavior. + except (UnicodeDecodeError, json.JSONDecodeError): + # What: apply the continue portion of the enclosing predicate; why: this clause remains in validate_loading_feedback\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: iterate across get and event to perform delta and isinstance and choice and dict and get; why: validate_loading_feedback repeats the body only while or for the loop header admits an iteration. + for choice in event.get("choices", []): + # What: compute delta from isinstance and choice and dict and get and delta; why: value delta get reasoning content if isinstance delta later reads delta, so validate_loading_feedback must retain the computed value under that name. + delta = choice.get("delta", {}) if isinstance(choice, dict) else {} + # What: compute value from isinstance and delta and dict and get and reasoning content; why: if isinstance value str later reads value, so validate_loading_feedback must retain the computed value under that name. + value = delta.get("reasoning_content") if isinstance(delta, dict) else None + # What: gate on isinstance and value and str before append and value and reasoning; why: validate_loading_feedback admits append and value and reasoning only for this predicate and excludes the opposite state. + if isinstance(value, str): + # What: call reasoning.append with value; why: validate_loading_feedback invokes reasoning.append while performing combined join reasoning; the call advances that operation through its result or side effect. + reasoning.append(value) + # What: compute combined from join and reasoning and value; why: observed freetoken swap loading model in combined later reads combined, so validate_loading_feedback must retain the computed value under that name. + combined = "".join(reasoning) + # What: compute observed from combined and freetoken swap and loading and model; why: if observed expected later reads observed, so validate_loading_feedback must retain the computed value under that name. + observed = "freetoken-swap loading model:" in combined + # What: gate on observed and expected before state and expected; why: validate_loading_feedback admits state and expected only for this predicate and excludes the opposite state. + if observed != expected: + # What: compute state from expected and missing and unexpected; why: raise runtime error f router loading feedback later reads state, so validate_loading_feedback must retain the computed value under that name. + state = "missing" if expected else "unexpected" + # What: raise RuntimeError for the caller; why: validate_loading_feedback stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError(f"router loading feedback was {state} for this qualification trial") + # What: map the expected field as expected; why: validate_loading_feedback carries expected into return {"expected": expected, "observed": observed, "passed": True}. + return {"expected": expected, "observed": observed, "passed": True} + + +# What: define concurrent_canaries around base and model and seconds; why: its direct callers call concurrent_canaries for concurrent canaries and rely on this exact input and result contract. +def concurrent_canaries(base: str, model: str, *, seconds: float = 180) -> tuple[list[tuple[bytes, dict]], dict]: + """Run two same-profile streams and prove they did not trigger a model swap.""" + # What: document run two same profile streams and prove in the concurrent_canaries docstring; why: introspection and maintainers read this exact docstring fragment to understand concurrent canaries behavior without executing it. + # What: compute and before from request json and base and router and status; why: value after request json base router status later reads and before, so concurrent_canaries must retain the computed value under that name. + _, before = request_json(base + "/router/status") + # What: compute prior activations from get and before and activations; why: if before get active profile model or not later reads prior activations, so concurrent_canaries must retain the computed value under that name. + prior_activations = before.get("activations") + # What: gate on model and get and isinstance and prior activations and int before runtime error; why: concurrent_canaries admits runtime error only for this predicate and excludes the opposite state. + if before.get("activeProfile") != model or not isinstance(prior_activations, int): + # What: raise RuntimeError for the caller; why: concurrent_canaries stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("same-model concurrency requires an already active profile") + # What: initialize results as an empty runtime accumulator; why: concurrent_canaries appends or maps entries into it during results append value before consuming the aggregate. + results: list[tuple[bytes, dict]] = [] + # What: initialize errors as an empty runtime accumulator; why: concurrent_canaries appends or maps entries into it during errors append exc before consuming the aggregate. + errors: list[BaseException] = [] + # What: compute lock from lock and threading; why: with lock later reads lock, so concurrent_canaries must retain the computed value under that name. + lock = threading.Lock() + # What: compute gate from barrier and threading and 3; why: gate wait timeout seconds later reads gate, so concurrent_canaries must retain the computed value under that name. + gate = threading.Barrier(3) + + # What: define run_one around the current object state; why: its direct callers call run_one for run one and rely on this exact input and result contract. + def run_one() -> None: + # What: establish the handler boundary for the protected operation; why: run_one routes failures to base exception while preserving cleanup and success flow. + try: + # What: supply timeout to gate.wait; why: run_one binds this seconds value to gate.wait's timeout input. + gate.wait(timeout=seconds) + # What: compute value from canary and base and model and false; why: results append value later reads value, so run_one must retain the computed value under that name. + value = canary(base, model, direct=False) + # What: enter the lock managed context before results append value; why: run_one releases this resource or lock after results append value on both success and failure paths. + with lock: + # What: call results.append with value; why: run_one invokes results.append while performing except base exception as exc; the call advances that operation through its result or side effect. + results.append(value) + # What: handle base exception by with lock; why: run_one converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException as exc: # noqa: BLE001 -- cleanup must record interrupts as qualification failures. + # What: enter the lock managed context before errors append exc; why: run_one releases this resource or lock after errors append exc on both success and failure paths. + with lock: + # What: call errors.append with exc; why: run_one invokes errors.append while performing the enclosing return; the call advances that operation through its result or side effect. + errors.append(exc) + + # What: compute workers from thread and index and threading and run one; why: for worker in workers later reads workers, so concurrent_canaries must retain the computed value under that name. + workers = [threading.Thread(target=run_one, name=f"native-router-concurrent-{index}", daemon=True) + # What: call range with 2; why: concurrent_canaries invokes range while performing for worker in workers; the call advances that operation through its result or side effect. + for index in range(2)] + # What: iterate across workers to perform start and worker; why: concurrent_canaries repeats the body only while or for the loop header admits an iteration. + for worker in workers: + # What: call worker.start with the declared inputs; why: concurrent_canaries invokes worker.start while performing gate wait timeout seconds; the call advances that operation through its result or side effect. + worker.start() + # What: supply timeout to gate.wait; why: concurrent_canaries binds this seconds value to gate.wait's timeout input. + gate.wait(timeout=seconds) + # What: iterate across workers to perform join and seconds and worker; why: concurrent_canaries repeats the body only while or for the loop header admits an iteration. + for worker in workers: + # What: call worker.join with seconds; why: concurrent_canaries invokes worker.join while performing if any worker is alive for worker in; the call advances that operation through its result or side effect. + worker.join(seconds) + # What: gate on any and is alive and worker and workers before timeout error; why: concurrent_canaries admits timeout error only for this predicate and excludes the opposite state. + if any(worker.is_alive() for worker in workers): + # What: raise TimeoutError for the caller; why: concurrent_canaries stops this rejected path before it can mutate state, dispatch work, or report success. + raise TimeoutError("same-model concurrent streams did not finish") + # What: gate on errors before runtime error and errors; why: concurrent_canaries admits runtime error and errors only for this predicate and excludes the opposite state. + if errors: + # What: raise RuntimeError for the caller; why: concurrent_canaries stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("same-model concurrent stream failed") from errors[0] + # What: compute and after from request json and base and router and status; why: or not all row get passed is later reads and after, so concurrent_canaries must retain the computed value under that name. + _, after = request_json(base + "/router/status") + # What: gate on model and prior activations and len and results and all before runtime error; why: concurrent_canaries admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call len with results; why: concurrent_canaries invokes len while performing or not all row get passed is; the call advances that operation through its result or side effect. + len(results) != 2 + # What: call all with results and get and value and row and true; why: concurrent_canaries invokes all while performing or after get active requests; the call advances that operation through its result or side effect. + or not all(row.get("passed") is True for _, row in results) + # What: call after.get with active requests; why: concurrent_canaries invokes after.get while performing or after get active profile model; the call advances that operation through its result or side effect. + or after.get("activeRequests") != 0 + # What: call after.get with active profile; why: concurrent_canaries invokes after.get while performing or after get activations prior activations; the call advances that operation through its result or side effect. + or after.get("activeProfile") != model + # What: call after.get with activations; why: concurrent_canaries consumes the after.get return value while evaluating or after.get("activations") != prior_activations. + or after.get("activations") != prior_activations + # What: complete the enclosing predicate with if len results differs from 2 or not all; why: concurrent_canaries groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: concurrent_canaries stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("same-model concurrency changed native routing residency") + # What: return results and model and route and model and requests from concurrent_canaries; why: concurrent_canaries exposes results and model and route and model and requests so its caller can continue with the function\'s computed outcome. + return results, { + # What: map the route field as native router; why: concurrent_canaries carries route into "route": "native_router". + "route": "native_router", + # What: map the model field as model; why: concurrent_canaries sends this field through "model": model so the router selects the canonical model or alias for upstream dispatch. + "model": model, + # What: map the requests field as 2; why: concurrent_canaries carries requests into "requests": 2. + "requests": 2, + # What: map the activation delta field as 0; why: concurrent_canaries carries activation delta into "activationDelta": 0. + "activationDelta": 0, + # What: map the active requests after field as 0; why: concurrent_canaries carries active requests after into "activeRequestsAfter": 0. + "activeRequestsAfter": 0, + # What: map the passed field as true; why: concurrent_canaries carries passed into "passed": True. + "passed": True, + # What: complete the enclosing predicate collection with results and model and route and model and requests and activation delta; why: concurrent_canaries groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. + } + + +# What: define cancellation_canary around base and model and seconds; why: its direct callers call cancellation_canary for cancellation canary and rely on this exact input and result contract. +def cancellation_canary(base: str, model: str, *, seconds: float = 90) -> tuple[bytes, dict]: + """Prove native router cancellation reaches idle without a normal completion credit. + + The raw partial SSE remains a private artifact. The returned observation is + deliberately limited to lifecycle counters and timing-safe booleans. + """ + # What: document prove native router cancellation reaches idle in the cancellation_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand cancellation canary behavior without executing it. + # What: document the raw partial sse remains a in the cancellation_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand cancellation canary behavior without executing it. + # What: document deliberately limited to lifecycle counters and in the cancellation_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand cancellation canary behavior without executing it. + # What: preserve the paragraph boundary in the the cancellation_canary docstring; why: introspection and maintainers read this paragraph break to understand cancellation canary behavior without executing it. + # What: compute request id from native qualification cancel; why: content type application json x ft request id request id later reads request id, so cancellation_canary must retain the computed value under that name. + request_id = "native-qualification-cancel" + # What: compute and before from request json and base and router and status; why: value cancelled request json base f router later reads and before, so cancellation_canary must retain the computed value under that name. + _, before = request_json(base + "/router/status") + # What: compute prior cancellations from get and before and cancellations; why: if not isinstance prior cancellations int or later reads prior cancellations, so cancellation_canary must retain the computed value under that name. + prior_cancellations = before.get("cancellations") + # What: compute prior terminal from get and before and terminal streams; why: if not isinstance prior cancellations int or later reads prior terminal, so cancellation_canary must retain the computed value under that name. + prior_terminal = before.get("terminalStreams") + # What: gate on isinstance and prior cancellations and int and prior terminal before runtime error; why: cancellation_canary admits runtime error only for this predicate and excludes the opposite state. + if not isinstance(prior_cancellations, int) or not isinstance(prior_terminal, int): + # What: raise RuntimeError for the caller; why: cancellation_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("router status lacks cancellation counters") # noqa: TRY004 -- malformed remote status is an operational failure. + # What: compute body from model and model and messages and temperature and max tokens; why: base v1 chat completions data json dumps later reads body, so cancellation_canary must retain the computed value under that name. + body = { + # What: map the model field as model; why: cancellation_canary sends this field through body so the router selects the canonical model or alias for upstream dispatch. + "model": model, + # What: map the role field as user; why: cancellation_canary carries role through body into base v1 chat completions data json dumps body. + "messages": [{"role": "user", "content": "Count upward slowly and do not stop."}], + # What: map the temperature field as 0; why: cancellation_canary carries temperature through body into base v1 chat completions data json dumps body. + "temperature": 0, + # What: map the max tokens field as 2048; why: cancellation_canary carries max tokens through body into base v1 chat completions data json dumps body. + "max_tokens": 2048, + # What: map the stream field as true; why: cancellation_canary carries stream through body into base v1 chat completions data json dumps body. + "stream": True, + # What: complete the body mapping with model and messages and temperature and max tokens and stream; why: cancellation_canary groups the supplied clauses as one body mapping before its value is consumed. + } + # What: compute request from request and request and base and urllib; why: with urllib request urlopen request timeout seconds as later reads request, so cancellation_canary must retain the computed value under that name. + request = urllib.request.Request( + # What: supply data to operation.encode; why: cancellation_canary binds this encode and dumps and body and json and utf 8 value to operation.encode's data input. + base + "/v1/chat/completions", data=json.dumps(body).encode("utf-8"), + # What: supply headers to urllib.request.Request; why: cancellation_canary binds this request id and native headers and base and content type and x ft request id value to urllib.request.Request's headers input. + headers={ + # What: map the content type field as application and json; why: cancellation_canary carries content type through request into with urllib request urlopen request timeout seconds as response. + "Content-Type": "application/json", "X-FT-Request-ID": request_id, + # What: call _native_headers with base; why: cancellation_canary consumes the _native_headers return value while evaluating **_native_headers(base). + **_native_headers(base), + # What: complete the request mapping with content type and x ft request id; why: cancellation_canary groups the supplied clauses as one request mapping before its value is consumed. + }, + # What: complete the urllib.request.Request call with data and headers; why: cancellation_canary groups the supplied clauses as one urllib.request.Request call before its value is consumed. + ) + # What: compute raw from bytearray; why: raw extend chunk later reads raw, so cancellation_canary must retain the computed value under that name. + raw = bytearray() + # What: compute first chunk from event and threading; why: first chunk set later reads first chunk, so cancellation_canary must retain the computed value under that name. + first_chunk = threading.Event() + # What: compute finished from event and threading; why: finished set later reads finished, so cancellation_canary must retain the computed value under that name. + finished = threading.Event() + # What: initialize errors as an empty runtime accumulator; why: cancellation_canary appends or maps entries into it during errors append exc before consuming the aggregate. + errors: list[BaseException] = [] + + # What: define consume around the current object state; why: its direct callers call consume for consume and rely on this exact input and result contract. + def consume() -> None: + # What: establish the handler boundary for the protected operation; why: consume routes failures to exception while preserving cleanup and success flow. + try: + # What: enter the urllib.request.urlopen managed context before for chunk in response; why: consume releases this resource or lock after for chunk in response on both success and failure paths. + with urllib.request.urlopen(request, timeout=seconds) as response: + # What: iterate across response to perform extend and chunk and raw; why: consume repeats the body only while or for the loop header admits an iteration. + for chunk in response: + # What: call raw.extend with chunk; why: consume invokes raw.extend while performing first chunk set; the call advances that operation through its result or side effect. + raw.extend(chunk) + # What: call first_chunk.set with the declared inputs; why: consume invokes first_chunk.set while performing except exception as exc cancellation may; the call advances that operation through its result or side effect. + first_chunk.set() + # What: handle exception by errors append exc; why: consume converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: # noqa: BLE001 -- cancellation may close a blocking HTTP read + # What: call errors.append with exc; why: consume invokes errors.append while performing finally; the call advances that operation through its result or side effect. + errors.append(exc) + # What: run finished set on every exit path; why: consume performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: call finished.set with the declared inputs; why: consume invokes finished.set while performing the enclosing return; the call advances that operation through its result or side effect. + finished.set() + + # What: compute worker from thread and threading and consume and native router cancel and true; why: worker start later reads worker, so cancellation_canary must retain the computed value under that name. + worker = threading.Thread(target=consume, name="native-router-cancel", daemon=True) + # What: compute started from monotonic and time; why: duration seconds time monotonic started later reads started, so cancellation_canary must retain the computed value under that name. + started = time.monotonic() + # What: call worker.start with the declared inputs; why: cancellation_canary invokes worker.start while performing if not first chunk wait seconds; the call advances that operation through its result or side effect. + worker.start() + # What: gate on wait and seconds and first chunk before timeout error; why: cancellation_canary admits timeout error only for this predicate and excludes the opposite state. + if not first_chunk.wait(seconds): + # What: raise TimeoutError for the caller; why: cancellation_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise TimeoutError("cancellation stream produced no first chunk") + # What: compute and cancelled from request json and base and request id and 30 and router; why: the enclosing return or state update later reads and cancelled, so cancellation_canary must retain the computed value under that name. + _, cancelled = request_json(base + f"/router/requests/{request_id}/cancel", {}, timeout=30) + # What: map the cancelled field as true; why: cancellation_canary carries cancelled through if cancelled != {"cancelled": True, "id": request_id} into raise runtime error router did not acknowledge the. + if cancelled != {"cancelled": True, "id": request_id}: + # What: raise RuntimeError for the caller; why: cancellation_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("router did not acknowledge the active cancellation request") + # What: gate on wait and seconds and finished before timeout error; why: cancellation_canary admits timeout error only for this predicate and excludes the opposite state. + if not finished.wait(seconds): + # What: raise TimeoutError for the caller; why: cancellation_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise TimeoutError("cancelled stream did not close") + # What: compute deadline from seconds and monotonic and time; why: while time monotonic deadline later reads deadline, so cancellation_canary must retain the computed value under that name. + deadline = time.monotonic() + seconds + # What: compute status from the named fixture input; why: status request json base router status timeout later reads status, so cancellation_canary must retain the computed value under that name. + status: dict | None = None + # What: iterate across deadline and monotonic and time to perform status and request json and base; why: cancellation_canary repeats the body only while or for the loop header admits an iteration. + while time.monotonic() < deadline: + # What: compute status from request json and base and 1 and router and status; why: if status get active requests later reads status, so cancellation_canary must retain the computed value under that name. + status = request_json(base + "/router/status", timeout=3)[1] + # What: gate on get and status before the computed value; why: cancellation_canary admits the computed value only for this predicate and excludes the opposite state. + if status.get("activeRequests") == 0: + # What: apply the break portion of the enclosing predicate; why: this clause remains in cancellation_canary\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: call time.sleep with 0 1; why: cancellation_canary invokes time.sleep while performing if status is or status get active requests; the call advances that operation through its result or side effect. + time.sleep(0.1) + # What: gate on status and get before timeout error; why: cancellation_canary admits timeout error only for this predicate and excludes the opposite state. + if status is None or status.get("activeRequests") != 0: + # What: raise TimeoutError for the caller; why: cancellation_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise TimeoutError("router did not return to idle after cancellation") + # What: gate on get and prior cancellations and status before runtime error; why: cancellation_canary admits runtime error only for this predicate and excludes the opposite state. + if status.get("cancellations") != prior_cancellations + 1: + # What: raise RuntimeError for the caller; why: cancellation_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("router cancellation counter did not increment") + # What: gate on prior terminal and get and status before runtime error; why: cancellation_canary admits runtime error only for this predicate and excludes the opposite state. + if status.get("terminalStreams") != prior_terminal: + # What: raise RuntimeError for the caller; why: cancellation_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("cancelled stream was credited as a normal completion") + # What: gate on raw before runtime error; why: cancellation_canary admits runtime error only for this predicate and excludes the opposite state. + if b"data: [DONE]" in raw: + # What: raise RuntimeError for the caller; why: cancellation_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("cancelled stream reached a normal terminal event") + # What: return bytes and raw and model and request id from cancellation_canary; why: cancellation_canary exposes bytes and raw and model and request id so its caller can continue with the function\'s computed outcome. + return bytes(raw), { + # What: map the route field as native router; why: cancellation_canary carries route into "route": "native_router". + "route": "native_router", + # What: map the model field as model; why: cancellation_canary sends this field through "model": model so the router selects the canonical model or alias for upstream dispatch. + "model": model, + # What: map the request id field as request id; why: cancellation_canary carries request id into "requestId": request_id. + "requestId": request_id, + # What: map the duration seconds field as started and monotonic and time; why: cancellation_canary carries duration seconds into "durationSeconds": time.monotonic() - started. + "durationSeconds": time.monotonic() - started, + # What: map the response bytes field as len and raw; why: cancellation_canary carries response bytes into "responseBytes": len(raw). + "responseBytes": len(raw), + # What: map the cancellation incremented field as true; why: cancellation_canary carries cancellation incremented into "cancellationIncremented": True. + "cancellationIncremented": True, + # What: map the normal completion credited field as false; why: cancellation_canary carries normal completion credited into "normalCompletionCredited": False. + "normalCompletionCredited": False, + # What: map the stream read error field as errors and repr and 0; why: cancellation_canary carries stream read error into "streamReadError": repr(errors[0]) if errors else None. + "streamReadError": repr(errors[0]) if errors else None, + # What: map the passed field as true; why: cancellation_canary carries passed into "passed": True. + "passed": True, + # What: complete the enclosing predicate collection with bytes and raw and model and request id and started and len; why: cancellation_canary groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. + } + + +# What: define conflicting_request_canary around base and active model and waiting model and seconds; why: its direct callers call conflicting_request_canary for conflicting request canary and rely on this exact input and result contract. +def conflicting_request_canary( + # What: declare the base input for conflicting_request_canary; why: conflicting_request_canary consumes base during restored raw restored row canary base active model direct, so callers must bind it with the other signature inputs. + base: str, active_model: str, waiting_model: str, *, seconds: float = 180 +# What: complete the enclosing predicate collection with bytes and bytes and bytes and dict; why: conflicting_request_canary groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. +) -> tuple[bytes, bytes, bytes, dict]: + """Hold A, prove B queues, cancel A, then complete B and restore A.""" + # What: document hold a prove b queues cancel in the conflicting_request_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand conflicting request canary behavior without executing it. + # What: compute request id from native qualification conflict; why: content type application json x ft request id request id later reads request id, so conflicting_request_canary must retain the computed value under that name. + request_id = "native-qualification-conflict" + # What: compute and before from request json and base and router and status; why: value after waiting request json base router status later reads and before, so conflicting_request_canary must retain the computed value under that name. + _, before = request_json(base + "/router/status") + # What: compute prior activations from get and before and activations; why: if before get active profile active model or not later reads prior activations, so conflicting_request_canary must retain the computed value under that name. + prior_activations = before.get("activations") + # What: gate on active model and get and isinstance and prior activations and int before runtime error; why: conflicting_request_canary admits runtime error only for this predicate and excludes the opposite state. + if before.get("activeProfile") != active_model or not isinstance(prior_activations, int): + # What: raise RuntimeError for the caller; why: conflicting_request_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("conflicting-request qualification requires active model A") + # What: compute body from active model and model and messages and temperature and max tokens; why: base v1 chat completions data json dumps later reads body, so conflicting_request_canary must retain the computed value under that name. + body = { + # What: map the model field as active model; why: conflicting_request_canary sends this field through body so the router selects the canonical model or alias for upstream dispatch. + "model": active_model, + # What: map the role field as user; why: conflicting_request_canary carries role through body into base v1 chat completions data json dumps body. + "messages": [{"role": "user", "content": "Count upward slowly and do not stop."}], + # What: map the temperature field as 0; why: conflicting_request_canary carries temperature through body into base v1 chat completions data json dumps body. + "temperature": 0, "max_tokens": 2048, "stream": True, + # What: complete the body mapping with model and messages and temperature and max tokens and stream; why: conflicting_request_canary groups the supplied clauses as one body mapping before its value is consumed. + } + # What: compute request from request and request and base and urllib; why: with urllib request urlopen request timeout seconds as later reads request, so conflicting_request_canary must retain the computed value under that name. + request = urllib.request.Request( + # What: supply data to operation.encode; why: conflicting_request_canary binds this encode and dumps and body and json and utf 8 value to operation.encode's data input. + base + "/v1/chat/completions", data=json.dumps(body).encode("utf-8"), + # What: supply headers to urllib.request.Request; why: conflicting_request_canary binds this request id and native headers and base and content type and x ft request id value to urllib.request.Request's headers input. + headers={ + # What: map the content type field as application and json; why: conflicting_request_canary carries content type through request into with urllib request urlopen request timeout seconds as response. + "Content-Type": "application/json", "X-FT-Request-ID": request_id, + # What: call _native_headers with base; why: conflicting_request_canary consumes the _native_headers return value while evaluating **_native_headers(base). + **_native_headers(base), + # What: complete the request mapping with content type and x ft request id; why: conflicting_request_canary groups the supplied clauses as one request mapping before its value is consumed. + }, + # What: complete the urllib.request.Request call with data and headers; why: conflicting_request_canary groups the supplied clauses as one urllib.request.Request call before its value is consumed. + ) + # What: compute active raw from bytearray; why: active raw extend chunk later reads active raw, so conflicting_request_canary must retain the computed value under that name. + active_raw = bytearray() + # What: compute first chunk from event and threading; why: first chunk set later reads first chunk, so conflicting_request_canary must retain the computed value under that name. + first_chunk = threading.Event() + # What: compute active finished from event and threading; why: active finished set later reads active finished, so conflicting_request_canary must retain the computed value under that name. + active_finished = threading.Event() + # What: initialize waiting result as an empty runtime accumulator; why: conflicting_request_canary appends or maps entries into it during waiting result append canary base waiting model direct false before consuming the aggregate. + waiting_result: list[tuple[bytes, dict]] = [] + # What: initialize active errors as an empty runtime accumulator; why: conflicting_request_canary appends or maps entries into it during active errors append exc before consuming the aggregate. + active_errors: list[BaseException] = [] + # What: initialize waiting errors as an empty runtime accumulator; why: conflicting_request_canary appends or maps entries into it during waiting errors append exc before consuming the aggregate. + waiting_errors: list[BaseException] = [] + + # What: define consume_active around the current object state; why: its direct callers call consume_active for consume active and rely on this exact input and result contract. + def consume_active() -> None: + # What: establish the handler boundary for the protected operation; why: consume_active routes failures to exception while preserving cleanup and success flow. + try: + # What: enter the urllib.request.urlopen managed context before for chunk in response; why: consume_active releases this resource or lock after for chunk in response on both success and failure paths. + with urllib.request.urlopen(request, timeout=seconds) as response: + # What: iterate across response to perform extend and chunk and active raw; why: consume_active repeats the body only while or for the loop header admits an iteration. + for chunk in response: + # What: call active_raw.extend with chunk; why: consume_active invokes active_raw.extend while performing first chunk set; the call advances that operation through its result or side effect. + active_raw.extend(chunk) + # What: call first_chunk.set with the declared inputs; why: consume_active invokes first_chunk.set while performing except exception as exc; the call advances that operation through its result or side effect. + first_chunk.set() + # What: handle exception by active errors append exc; why: consume_active converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: # noqa: BLE001 -- worker failures are returned to the coordinating test thread. + # What: call active_errors.append with exc; why: consume_active invokes active_errors.append while performing finally; the call advances that operation through its result or side effect. + active_errors.append(exc) + # What: run active finished set on every exit path; why: consume_active performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: call active_finished.set with the declared inputs; why: consume_active invokes active_finished.set while performing the enclosing return; the call advances that operation through its result or side effect. + active_finished.set() + + # What: define consume_waiting around the current object state; why: its direct callers call consume_waiting for consume waiting and rely on this exact input and result contract. + def consume_waiting() -> None: + # What: establish the handler boundary for the protected operation; why: consume_waiting routes failures to base exception while preserving cleanup and success flow. + try: + # What: supply direct to waiting_result.append; why: consume_waiting binds this false value to waiting_result.append's direct input. + waiting_result.append(canary(base, waiting_model, direct=False)) + # What: handle base exception by waiting errors append exc; why: consume_waiting converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException as exc: # noqa: BLE001 -- cleanup must record interrupts as qualification failures. + # What: call waiting_errors.append with exc; why: consume_waiting invokes waiting_errors.append while performing the enclosing return; the call advances that operation through its result or side effect. + waiting_errors.append(exc) + + # What: compute active worker from thread and threading and consume active and true; why: active worker start later reads active worker, so conflicting_request_canary must retain the computed value under that name. + active_worker = threading.Thread(target=consume_active, daemon=True) + # What: call active_worker.start with the declared inputs; why: conflicting_request_canary invokes active_worker.start while performing if not first chunk wait seconds; the call advances that operation through its result or side effect. + active_worker.start() + # What: gate on wait and seconds and first chunk before timeout error; why: conflicting_request_canary admits timeout error only for this predicate and excludes the opposite state. + if not first_chunk.wait(seconds): + # What: raise TimeoutError for the caller; why: conflicting_request_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise TimeoutError("active conflicting stream produced no first chunk") + # What: compute waiting worker from thread and threading and consume waiting and true; why: waiting worker start later reads waiting worker, so conflicting_request_canary must retain the computed value under that name. + waiting_worker = threading.Thread(target=consume_waiting, daemon=True) + # What: call waiting_worker.start with the declared inputs; why: conflicting_request_canary invokes waiting_worker.start while performing deadline time monotonic seconds; the call advances that operation through its result or side effect. + waiting_worker.start() + # What: compute deadline from seconds and monotonic and time; why: while time monotonic deadline later reads deadline, so conflicting_request_canary must retain the computed value under that name. + deadline = time.monotonic() + seconds + # What: compute queued from the named fixture input; why: queued request json base router status timeout later reads queued, so conflicting_request_canary must retain the computed value under that name. + queued: dict | None = None + # What: iterate across deadline and monotonic and time to perform queued and request json and base; why: conflicting_request_canary repeats the body only while or for the loop header admits an iteration. + while time.monotonic() < deadline: + # What: compute queued from request json and base and 1 and router and status; why: if queued get queued requests later reads queued, so conflicting_request_canary must retain the computed value under that name. + queued = request_json(base + "/router/status", timeout=3)[1] + # What: gate on get and queued before the computed value; why: conflicting_request_canary admits the computed value only for this predicate and excludes the opposite state. + if queued.get("queuedRequests") == 1: + # What: apply the break portion of the enclosing predicate; why: this clause remains in conflicting_request_canary\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: call time.sleep with 0 1; why: conflicting_request_canary invokes time.sleep while performing if; the call advances that operation through its result or side effect. + time.sleep(0.1) + # What: gate on queued and active model and get before runtime error; why: conflicting_request_canary admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call queued.get with queued requests; why: conflicting_request_canary invokes queued.get while performing or queued get active profile active model; the call advances that operation through its result or side effect. + queued is None or queued.get("queuedRequests") != 1 + # What: call queued.get with active profile; why: conflicting_request_canary invokes queued.get while performing or queued get active requests; the call advances that operation through its result or side effect. + or queued.get("activeProfile") != active_model + # What: call queued.get with active requests; why: conflicting_request_canary invokes queued.get while performing or queued get active identity matches engine is not; the call advances that operation through its result or side effect. + or queued.get("activeRequests") != 1 + # What: call queued.get with active identity matches engine; why: conflicting_request_canary consumes the queued.get return value while evaluating or queued.get("activeIdentityMatchesEngine") is not True. + or queued.get("activeIdentityMatchesEngine") is not True + # What: complete the enclosing predicate with if queued is or queued get queued requests differs from 1; why: conflicting_request_canary groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: conflicting_request_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("waiting model did not queue behind the active stream") + # What: gate on request id and request json and base before runtime error; why: conflicting_request_canary admits runtime error only for this predicate and excludes the opposite state. + if request_json(base + f"/router/requests/{request_id}/cancel", {}, timeout=30)[1] != { + # What: map the cancelled field as true; why: conflicting_request_canary carries cancelled into "cancelled": True, "id": request_id. + "cancelled": True, "id": request_id, + # What: apply the grouped expression portion of the enclosing predicate; why: this clause remains in conflicting_request_canary\'s enclosing expression so its grouping and evaluation order stay intact. + }: + # What: raise RuntimeError for the caller; why: conflicting_request_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("active conflicting stream cancellation was not acknowledged") + # What: gate on wait and seconds and active finished before timeout error; why: conflicting_request_canary admits timeout error only for this predicate and excludes the opposite state. + if not active_finished.wait(seconds): + # What: raise TimeoutError for the caller; why: conflicting_request_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise TimeoutError("active conflicting stream did not close") + # What: call waiting_worker.join with seconds; why: conflicting_request_canary invokes waiting_worker.join while performing if waiting worker is alive or waiting errors or len; the call advances that operation through its result or side effect. + waiting_worker.join(seconds) + # What: gate on waiting errors and is alive and waiting worker and len and waiting result before runtime error; why: conflicting_request_canary admits runtime error only for this predicate and excludes the opposite state. + if waiting_worker.is_alive() or waiting_errors or len(waiting_result) != 1: + # What: propagate raise RuntimeError waiting model did not complete after active stream cancellation as a qualification failure; why: callers must not continue after this violated precondition or observed result. + raise RuntimeError("waiting model did not complete after active-stream cancellation") + # What: compute waiting raw and waiting row from waiting result and 0; why: return bytes active raw waiting raw restored raw later reads waiting raw and waiting row, so conflicting_request_canary must retain the computed value under that name. + waiting_raw, waiting_row = waiting_result[0] + # What: compute and after waiting from request json and base and router and status; why: value restored request json base router status later reads and after waiting, so conflicting_request_canary must retain the computed value under that name. + _, after_waiting = request_json(base + "/router/status") + # What: gate on waiting model and get and prior activations and waiting row and after waiting before runtime error; why: conflicting_request_canary admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call waiting_row.get with passed; why: conflicting_request_canary invokes waiting_row.get while performing or after waiting get active profile waiting model; the call advances that operation through its result or side effect. + waiting_row.get("passed") is not True + # What: call after_waiting.get with active profile; why: conflicting_request_canary invokes after_waiting.get while performing or after waiting get active requests; the call advances that operation through its result or side effect. + or after_waiting.get("activeProfile") != waiting_model + # What: call after_waiting.get with active requests; why: conflicting_request_canary invokes after_waiting.get while performing or after waiting get activations prior activations; the call advances that operation through its result or side effect. + or after_waiting.get("activeRequests") != 0 + # What: call after_waiting.get with activations; why: conflicting_request_canary consumes the after_waiting.get return value while evaluating or after_waiting.get("activations") != prior_activations + 1. + or after_waiting.get("activations") != prior_activations + 1 + # What: complete the enclosing predicate with if waiting row get passed is not true or after waiting get active profile; why: conflicting_request_canary groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: conflicting_request_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("waiting model did not receive exactly one post-drain activation") + # What: compute restored raw and restored row from canary and base and active model and false; why: return bytes active raw waiting raw restored raw later reads restored raw and restored row, so conflicting_request_canary must retain the computed value under that name. + restored_raw, restored_row = canary(base, active_model, direct=False) + # What: compute and restored from request json and base and router and status; why: the enclosing return or state update later reads and restored, so conflicting_request_canary must retain the computed value under that name. + _, restored = request_json(base + "/router/status") + # What: gate on get and prior activations and restored row and restored before runtime error; why: conflicting_request_canary admits runtime error only for this predicate and excludes the opposite state. + if restored_row.get("passed") is not True or restored.get("activations") != prior_activations + 2: + # What: raise RuntimeError for the caller; why: conflicting_request_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("conflicting-request qualification did not restore model A") + # What: gate on active raw before runtime error; why: conflicting_request_canary admits runtime error only for this predicate and excludes the opposite state. + if b"data: [DONE]" in active_raw: + # What: raise RuntimeError for the caller; why: conflicting_request_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("active conflicting stream completed normally instead of being cancelled") + # What: return waiting raw and restored raw and bytes and active raw from conflicting_request_canary; why: conflicting_request_canary exposes waiting raw and restored raw and bytes and active raw so its caller can continue with the function\'s computed outcome. + return bytes(active_raw), waiting_raw, restored_raw, { + # What: map the active profile field as active model; why: conflicting_request_canary carries active profile into "activeProfile": active_model, "waitingProfile": waiting_model. + "activeProfile": active_model, "waitingProfile": waiting_model, + # What: map the queued behind active field as true; why: conflicting_request_canary carries queued behind active into "queuedBehindActive": True, "activeIdentityPreservedWhileQueued": True. + "queuedBehindActive": True, "activeIdentityPreservedWhileQueued": True, + # What: map the activation delta field as 2; why: conflicting_request_canary carries activation delta into "activationDelta": 2, "restoredProfile": active_model, "passed": True. + "activationDelta": 2, "restoredProfile": active_model, "passed": True, + # What: execute the grouped source fragment; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + } + + +# What: define stop_process_group around proc; why: its direct callers call stop_process_group for stop process group and rely on this exact input and result contract. +def stop_process_group(proc: subprocess.Popen[bytes]) -> None: + # What: gate on poll and proc before the computed value; why: stop_process_group admits the computed value only for this predicate and excludes the opposite state. + if proc.poll() is not None: + # What: return no value from stop_process_group; why: stop_process_group returns no value to callers that depend on its completed result. + return + # What: call os.killpg with pid and proc and sigterm and signal; why: stop_process_group invokes os.killpg while performing try; the call advances that operation through its result or side effect. + os.killpg(proc.pid, signal.SIGTERM) + # What: establish the handler boundary for the protected operation; why: stop_process_group routes failures to timeout expired and subprocess while preserving cleanup and success flow. + try: + # What: supply timeout to proc.wait; why: stop_process_group binds this 45 value to proc.wait's timeout input. + proc.wait(timeout=45) + # What: handle timeout expired and subprocess by os killpg proc pid signal sigkill; why: stop_process_group converts that failure into this concrete recovery, response, or cleanup behavior. + except subprocess.TimeoutExpired: + # What: call os.killpg with pid and proc and sigkill and signal; why: stop_process_group invokes os.killpg while performing proc wait timeout; the call advances that operation through its result or side effect. + os.killpg(proc.pid, signal.SIGKILL) + # What: supply timeout to proc.wait; why: stop_process_group binds this 10 value to proc.wait's timeout input. + proc.wait(timeout=10) + + +# What: define validate_routed_trial around router and alias and prior activations and expected delta; why: its direct callers call validate_routed_trial for validate routed trial and rely on this exact input and result contract. +def validate_routed_trial(router: dict, *, alias: str, prior_activations: int, expected_delta: int) -> int: + """Prove that a labeled routed benchmark actually used its intended state. + + Timings alone cannot distinguish a warm request from an accidental reload. + The bounded router state makes each performance label auditable without + retaining a prompt or model path in the public summary. + """ + # What: document prove that a labeled routed benchmark in the validate_routed_trial docstring; why: introspection and maintainers read this exact docstring fragment to understand validate routed trial behavior without executing it. + # What: document timings alone cannot distinguish a warm in the validate_routed_trial docstring; why: introspection and maintainers read this exact docstring fragment to understand validate routed trial behavior without executing it. + # What: document the bounded router state makes each in the validate_routed_trial docstring; why: introspection and maintainers read this exact docstring fragment to understand validate routed trial behavior without executing it. + # What: document retaining a prompt or model path in the validate_routed_trial docstring; why: introspection and maintainers read this exact docstring fragment to understand validate routed trial behavior without executing it. + # What: preserve the paragraph boundary in the the validate_routed_trial docstring; why: introspection and maintainers read this paragraph break to understand validate routed trial behavior without executing it. + # What: compute activations from get and router and activations; why: if not isinstance activations int or later reads activations, so validate_routed_trial must retain the computed value under that name. + activations = router.get("activations") + # What: gate on alias and get and router before runtime error; why: validate_routed_trial admits runtime error only for this predicate and excludes the opposite state. + if router.get("activeProfile") != alias or router.get("activeRequests") != 0: + # What: raise RuntimeError for the caller; why: validate_routed_trial stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("routed trial did not settle on the expected idle profile") + # What: gate on activations and isinstance and int and prior activations and expected delta before runtime error; why: validate_routed_trial admits runtime error only for this predicate and excludes the opposite state. + if not isinstance(activations, int) or activations != prior_activations + expected_delta: + # What: raise RuntimeError for the caller; why: validate_routed_trial stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("routed trial activation count did not match its scenario") + # What: return activations from validate_routed_trial; why: validate_routed_trial exposes activations so its caller can continue with the function\'s computed outcome. + return activations + + +# What: define valid_periodic_performance around performance; why: its direct callers call valid_periodic_performance for valid periodic performance and rely on this exact input and result contract. +def valid_periodic_performance(performance: dict) -> bool: + # What: compute rows from get and performance and sys stats; why: or not isinstance rows list later reads rows, so valid_periodic_performance must retain the computed value under that name. + rows = performance.get("sys_stats") + # What: gate on get and isinstance and rows and list and all before the computed value; why: valid_periodic_performance admits the computed value only for this predicate and excludes the opposite state. + if ( + # What: call performance.get with enabled; why: valid_periodic_performance invokes performance.get while performing or performance get gpu stats; the call advances that operation through its result or side effect. + performance.get("enabled") is not True + # What: call performance.get with gpu stats; why: valid_periodic_performance invokes performance.get while performing or not isinstance rows list; the call advances that operation through its result or side effect. + or performance.get("gpu_stats") != [] + # What: call isinstance with rows and list; why: valid_periodic_performance invokes isinstance while performing or not len rows; the call advances that operation through its result or side effect. + or not isinstance(rows, list) + # What: call len with rows; why: valid_periodic_performance invokes len while performing or not all; the call advances that operation through its result or side effect. + or not 1 <= len(rows) <= 720 + # What: call all with row and rows and isinstance and dict; why: valid_periodic_performance invokes all while performing isinstance row dict; the call advances that operation through its result or side effect. + or not all( + # What: call isinstance with row and dict; why: valid_periodic_performance invokes isinstance while performing and row get scope engine process tree; the call advances that operation through its result or side effect. + isinstance(row, dict) + # What: call row.get with scope; why: valid_periodic_performance invokes row.get while performing and not any key in row; the call advances that operation through its result or side effect. + and row.get("scope") == "engine-process-tree" + # What: call any with key and row and pids and model and path; why: valid_periodic_performance invokes any while performing for row in rows; the call advances that operation through its result or side effect. + and not any(key in row for key in ("pids", "model", "path", "command")) + # What: apply the for row in rows portion of the enclosing predicate; why: this clause remains in valid_periodic_performance\'s enclosing expression so its grouping and evaluation order stay intact. + for row in rows + # What: complete the all call with row; why: valid_periodic_performance groups the supplied clauses as one all call before its value is consumed. + ) + # What: complete the enclosing predicate with if performance get enabled is not true or performance get gpu stats; why: valid_periodic_performance groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: return false from valid_periodic_performance; why: valid_periodic_performance exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: compute latest from rows and 1; why: latest get ram available is later reads latest, so valid_periodic_performance must retain the computed value under that name. + latest = rows[-1] + # What: return isinstance and int and all and get from valid_periodic_performance; why: valid_periodic_performance exposes isinstance and int and all and get so its caller can continue with the function\'s computed outcome. + return ( + # What: call latest.get with ram available; why: valid_periodic_performance invokes latest.get while performing and latest get vram available is; the call advances that operation through its result or side effect. + latest.get("ram_available") is True + # What: call latest.get with vram available; why: valid_periodic_performance invokes latest.get while performing and isinstance latest get ram bytes int; the call advances that operation through its result or side effect. + and latest.get("vram_available") is True + # What: call isinstance with get and latest and ram bytes and int; why: valid_periodic_performance invokes isinstance while performing and latest ram bytes; the call advances that operation through its result or side effect. + and isinstance(latest.get("ram_bytes"), int) + # What: apply the and latest ram bytes portion of the enclosing predicate; why: this clause remains in valid_periodic_performance\'s enclosing expression so its grouping and evaluation order stay intact. + and latest["ram_bytes"] > 0 + # What: call isinstance with get and latest and vram bytes and int; why: valid_periodic_performance invokes isinstance while performing and latest vram bytes; the call advances that operation through its result or side effect. + and isinstance(latest.get("vram_bytes"), int) + # What: apply the and latest vram bytes portion of the enclosing predicate; why: this clause remains in valid_periodic_performance\'s enclosing expression so its grouping and evaluation order stay intact. + and latest["vram_bytes"] > 0 + # What: call all with key and isinstance and str and latest; why: valid_periodic_performance invokes all while performing isinstance latest get key str and latest; the call advances that operation through its result or side effect. + and all( + # What: call isinstance with get and key and latest and str; why: valid_periodic_performance invokes isinstance while performing for key in timestamp ram source vram source; the call advances that operation through its result or side effect. + isinstance(latest.get(key), str) and latest[key] + # What: apply the for key in timestamp ram source vram source portion of the enclosing predicate; why: this clause remains in valid_periodic_performance\'s enclosing expression so its grouping and evaluation order stay intact. + for key in ("timestamp", "ram_source", "vram_source") + # What: complete the all call with key; why: valid_periodic_performance groups the supplied clauses as one all call before its value is consumed. + ) + # What: complete the valid_periodic_performance signature with performance; why: valid_periodic_performance groups the supplied clauses as one valid_periodic_performance signature before its value is consumed. + ) + + +# What: define control_plane_canary around base and artifacts; why: its direct callers call control_plane_canary for control plane canary and rely on this exact input and result contract. +def control_plane_canary(base: str, artifacts: Path) -> dict: + """Qualify authenticated management, metrics, and bounded router-log access.""" + # What: document qualify authenticated management metrics and bounded in the control_plane_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand control plane canary behavior without executing it. + # What: initialize unauthorized as an empty runtime accumulator; why: control_plane_canary appends or maps entries into it during unauthorized path exc code before consuming the aggregate. + unauthorized: dict[str, int] = {} + # What: compute protected paths from router and status and v1 and models and models; why: for path in protected paths later reads protected paths, so control_plane_canary must retain the computed value under that name. + protected_paths = ("/router/status", "/v1/models", "/models", "/api/performance") + # What: iterate across protected paths to perform request and request and base and path and urllib; why: control_plane_canary repeats the body only while or for the loop header admits an iteration. + for path in protected_paths: + # What: compute request from request and request and base and path; why: with urllib request urlopen request timeout later reads request, so control_plane_canary must retain the computed value under that name. + request = urllib.request.Request(base + path) + # What: establish the handler boundary for the protected operation; why: control_plane_canary routes failures to httperror and error and urllib while preserving cleanup and success flow. + try: + # What: enter the urllib.request.urlopen managed context before pass; why: control_plane_canary releases this resource or lock after pass on both success and failure paths. + with urllib.request.urlopen(request, timeout=10): + # What: ignore the anticipated exception handled by this branch; why: control_plane_canary continues its retry or cleanup path instead of re-raising that transient failure. + pass + # What: handle httperror and error and urllib by unauthorized path exc code; why: control_plane_canary converts that failure into this concrete recovery, response, or cleanup behavior. + except urllib.error.HTTPError as exc: + # What: compute unauthorized entry from code and exc; why: if unauthorized path for path in later reads unauthorized entry, so control_plane_canary must retain the computed value under that name. + unauthorized[path] = exc.code + # What: call exc.close with the declared inputs; why: control_plane_canary invokes exc.close while performing else; the call advances that operation through its result or side effect. + exc.close() + # What: select the remaining branch that performs raise runtime error f unauthenticated request unexpectedly; why: control_plane_canary covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: raise RuntimeError for the caller; why: control_plane_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError(f"unauthenticated request unexpectedly succeeded: {path}") + # What: gate on unauthorized and path and protected paths before runtime error; why: control_plane_canary admits runtime error only for this predicate and excludes the opposite state. + if unauthorized != {path: 401 for path in protected_paths}: + # What: raise RuntimeError for the caller; why: control_plane_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("native router did not reject unauthenticated control and inference") + + # What: gate on native auth base and native api key and rstrip and base before runtime error; why: control_plane_canary admits runtime error only for this predicate and excludes the opposite state. + if _NATIVE_AUTH_BASE != base.rstrip("/") or _NATIVE_API_KEY is None: + # What: raise RuntimeError for the caller; why: control_plane_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("native router credentials are not scoped to the qualification origin") + # What: compute basic from decode and b64encode and base64 and encode; why: basic authorization f basic basic later reads basic, so control_plane_canary must retain the computed value under that name. + basic = base64.b64encode(f"operator:{_NATIVE_API_KEY}".encode()).decode() + # What: initialize alternate auth raw as an empty runtime accumulator; why: control_plane_canary appends or maps entries into it during alternate auth raw name raw before consuming the aggregate. + alternate_auth_raw: dict[str, bytes] = {} + # What: iterate across native api key and basic to perform request and request and base and headers and urllib; why: control_plane_canary repeats the body only while or for the loop header admits an iteration. + for name, headers in ( + # What: map the authorization field as basic and basic; why: control_plane_canary carries authorization through ("basic", {"Authorization": f"Basic {basic}"}) into raise runtime error models is not equivalent to. + ("basic", {"Authorization": f"Basic {basic}"}), + # What: map the x api key field as native api key; why: control_plane_canary carries x api key through ("x-api-key", {"X-Api-Key": _NATIVE_API_KEY}) into raise runtime error models is not equivalent to. + ("x-api-key", {"X-Api-Key": _NATIVE_API_KEY}), + # What: complete the enclosing predicate collection with basic and basic and authorization and basic and native api key and x api key and x api key; why: control_plane_canary groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. + ): + # What: compute request from request and request and base and headers; why: with urllib request urlopen request timeout as response later reads request, so control_plane_canary must retain the computed value under that name. + request = urllib.request.Request(base + "/router/status", headers=headers) + # What: enter the urllib.request.urlopen managed context before raw response read; why: control_plane_canary releases this resource or lock after raw response read on both success and failure paths. + with urllib.request.urlopen(request, timeout=10) as response: + # What: compute raw from read and response; why: status json loads raw later reads raw, so control_plane_canary must retain the computed value under that name. + raw = response.read() + # What: compute status from loads and raw and json; why: if status get active profile model a later reads status, so control_plane_canary must retain the computed value under that name. + status = json.loads(raw) + # What: gate on get and status before runtime error and name; why: control_plane_canary admits runtime error and name only for this predicate and excludes the opposite state. + if status.get("activeProfile") != "model-a": + # What: raise RuntimeError for the caller; why: control_plane_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError(f"{name} authentication did not expose exact model-a residency") + # What: compute alternate auth raw entry from raw; why: for name raw in alternate auth raw items later reads alternate auth raw entry, so control_plane_canary must retain the computed value under that name. + alternate_auth_raw[name] = raw + + # What: compute models raw and models from request json and base and v1 and models and 10; why: artifacts control v1 models json write bytes models raw later reads models raw and models, so control_plane_canary must retain the computed value under that name. + models_raw, models = request_json(base + "/v1/models", timeout=10) + # What: compute and models alias from request json and base and models and 10; why: value namespaced stats request json later reads and models alias, so control_plane_canary must retain the computed value under that name. + _, models_alias = request_json(base + "/models", timeout=10) + # What: establish a diagnostic boundary around namespaced upstream stats; why: a live proxy rejection must retain its private response body instead of collapsing to an opaque HTTP status. + try: + # What: request stats through the slash-namespaced alias; why: the control plane must prove longest-prefix alias resolution against the resident engine. + _, namespaced_stats = request_json( + # What: supply the bounded timeout to the stats request; why: qualification must not wait indefinitely on a broken upstream proxy. + base + "/upstream/compat/model-a/v1/stats", timeout=10 + # What: complete the namespaced stats request; why: the returned mapping is validated with the remaining authenticated control-plane evidence. + ) + # What: catch an HTTP rejection from the live proxy; why: the exact private error payload is required to distinguish path parsing, residency, and upstream failures. + except urllib.error.HTTPError as exc: + # What: read and decode the bounded rejection body; why: diagnostics must remain useful without allowing an unbounded error response into artifacts. + error_body = exc.read(64 * 1024).decode("utf-8", errors="replace") + # What: raise a contextual qualification error chained to the protocol failure; why: the harness must fail closed while preserving the actionable reason. + raise RuntimeError(f"namespaced stats HTTP {exc.code}: {error_body}") from exc + # What: compute routed raw and routed from request json and base and router and models and 10; why: artifacts control router models json write bytes routed raw later reads routed raw and routed, so control_plane_canary must retain the computed value under that name. + routed_raw, routed = request_json(base + "/router/models", timeout=10) + # What: compute profiles raw and profiles from request json and base and router and profiles and 10; why: artifacts control router profiles json write bytes profiles raw later reads profiles raw and profiles, so control_plane_canary must retain the computed value under that name. + profiles_raw, profiles = request_json(base + "/router/profiles", timeout=10) + # What: compute performance raw from the named fixture input; why: performance raw performance request json base api performance later reads performance raw, so control_plane_canary must retain the computed value under that name. + performance_raw = b"" + # What: initialize performance as an empty runtime accumulator; why: control_plane_canary appends or maps entries into it during performance raw performance request json base api performance timeout before consuming the aggregate. + performance: dict = {} + # What: compute performance deadline from monotonic and time and 15; why: if time monotonic performance deadline later reads performance deadline, so control_plane_canary must retain the computed value under that name. + performance_deadline = time.monotonic() + 15 + # What: iterate across the computed value to perform performance raw and performance and request json and base; why: control_plane_canary repeats the body only while or for the loop header admits an iteration. + while True: + # What: compute performance raw and performance from request json and base and api and performance and 10; why: control_plane_canary consumes performance raw and performance during artifacts control performance json write bytes performance raw, so performance raw and performance value receives the computed val. + performance_raw, performance = request_json(base + "/api/performance", timeout=10) + # What: gate on valid periodic performance and performance before the computed value; why: control_plane_canary admits the computed value only for this predicate and excludes the opposite state. + if valid_periodic_performance(performance): + # What: apply the break portion of the enclosing predicate; why: this clause remains in control_plane_canary\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: gate on performance deadline and monotonic and time before runtime error; why: control_plane_canary admits runtime error only for this predicate and excludes the opposite state. + if time.monotonic() >= performance_deadline: + # What: raise RuntimeError for the caller; why: control_plane_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("periodic performance lacked a positive owned-process sample") + # What: call time.sleep with 0 25; why: control_plane_canary invokes time.sleep while performing metrics raw request bytes base metrics timeout; the call advances that operation through its result or side effect. + time.sleep(0.25) + # What: compute metrics raw from request bytes and base and metrics and 10; why: or b freetoken swap admissions total not in metrics raw later reads metrics raw, so control_plane_canary must retain the computed value under that name. + metrics_raw = request_bytes(base + "/metrics", timeout=10) + # What: compute model rows from get and models and data; why: for rows in model rows alias rows routed rows later reads model rows, so control_plane_canary must retain the computed value under that name. + model_rows = models.get("data") + # What: compute alias rows from get and models alias and data; why: for rows in model rows alias rows routed rows later reads alias rows, so control_plane_canary must retain the computed value under that name. + alias_rows = models_alias.get("data") + # What: compute routed rows from get and routed and data; why: for rows in model rows alias rows routed rows later reads routed rows, so control_plane_canary must retain the computed value under that name. + routed_rows = routed.get("data") + # What: compute profile rows from get and profiles and data; why: for rows in model rows alias rows routed rows later reads profile rows, so control_plane_canary must retain the computed value under that name. + profile_rows = profiles.get("data") + # What: compute routing profiles from get and profiles and routing profiles; why: item for item in routing profiles later reads routing profiles, so control_plane_canary must retain the computed value under that name. + routing_profiles = profiles.get("routingProfiles") + # What: gate on all and rows and isinstance and list and model rows before runtime error; why: control_plane_canary admits runtime error only for this predicate and excludes the opposite state. + if not all( + # What: call isinstance with rows and list; why: control_plane_canary invokes isinstance while performing for rows in model rows alias rows routed rows; the call advances that operation through its result or side effect. + isinstance(rows, list) and all(isinstance(item, dict) for item in rows) + # What: apply the for rows in model rows alias rows routed rows portion of the enclosing predicate; why: this clause remains in control_plane_canary\'s enclosing expression so its grouping and evaluation order stay intact. + for rows in (model_rows, alias_rows, routed_rows, profile_rows) + # What: complete the all call with rows; why: control_plane_canary groups the supplied clauses as one all call before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: control_plane_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("authenticated native control-plane responses have invalid shapes") + # The pinned alias invokes the same handler independently, so request-time + # `created` values may differ by one second. Everything else must match. + # What: compute normalized models from k and v and item and model rows; why: if model envelope alias envelope or normalized models normalized alias later reads normalized models, so control_plane_canary must retain the computed value under that name. + normalized_models = [{k: v for k, v in item.items() if k != "created"} for item in model_rows] + # What: compute normalized alias from k and v and item and alias rows; why: if model envelope alias envelope or normalized models normalized alias later reads normalized alias, so control_plane_canary must retain the computed value under that name. + normalized_alias = [{k: v for k, v in item.items() if k != "created"} for item in alias_rows] + # What: compute model envelope from k and v and items and models and data; why: if model envelope alias envelope or normalized models normalized alias later reads model envelope, so control_plane_canary must retain the computed value under that name. + model_envelope = {k: v for k, v in models.items() if k != "data"} + # What: compute alias envelope from k and v and items and models alias and data; why: if model envelope alias envelope or normalized models normalized alias later reads alias envelope, so control_plane_canary must retain the computed value under that name. + alias_envelope = {k: v for k, v in models_alias.items() if k != "data"} + # What: gate on model envelope and alias envelope and normalized models and normalized alias before runtime error; why: control_plane_canary admits runtime error only for this predicate and excludes the opposite state. + if model_envelope != alias_envelope or normalized_models != normalized_alias: + # What: raise RuntimeError for the caller; why: control_plane_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("/models is not equivalent to the /v1/models compatibility listing") + # What: compute aliases from sorted and item and model rows and isinstance; why: not model a model b compat model a preferred model later reads aliases, so control_plane_canary must retain the computed value under that name. + aliases = sorted(item["id"] for item in model_rows if isinstance(item.get("id"), str)) + # What: compute routed names from sorted and item and routed rows and isinstance; why: or routed names profile names later reads routed names, so control_plane_canary must retain the computed value under that name. + routed_names = sorted( + # What: call isinstance with get and item and name and str; why: control_plane_canary consumes the isinstance return value while evaluating item["name"] for item in routed_rows if isinstance(item.get("name"), str. + item["name"] for item in routed_rows if isinstance(item.get("name"), str) + # What: complete the sorted call with item; why: control_plane_canary groups the supplied clauses as one sorted call before its value is consumed. + ) + # What: compute profile names from sorted and item and profile rows and isinstance; why: or routed names profile names later reads profile names, so control_plane_canary must retain the computed value under that name. + profile_names = sorted( + # What: call isinstance with get and item and name and str; why: control_plane_canary consumes the isinstance return value while evaluating item["name"] for item in profile_rows if isinstance(item.get("name"), st. + item["name"] for item in profile_rows if isinstance(item.get("name"), str) + # What: complete the sorted call with item; why: control_plane_canary groups the supplied clauses as one sorted call before its value is consumed. + ) + # What: compute coding profile from isinstance and routing profiles and list and next; why: or not isinstance coding profile dict later reads coding profile, so control_plane_canary must retain the computed value under that name. + coding_profile = next( + # What: complete the next call with item; why: control_plane_canary groups the supplied clauses as one next call before its value is consumed. + ( + # What: apply the item for item in routing profiles portion of coding profile; why: control_plane_canary uses this clause to evaluate coding profile as one grouped value. + item for item in routing_profiles + # What: call isinstance with item and dict; why: control_plane_canary consumes the isinstance return value while evaluating if isinstance(item, dict) and item.get("name") == "coding". + if isinstance(item, dict) and item.get("name") == "coding" + # What: complete the next call with item; why: control_plane_canary groups the supplied clauses as one next call before its value is consumed. + ), + # What: apply the grouped expression portion of coding profile; why: control_plane_canary uses this clause to evaluate coding profile as one grouped value. + None, + # What: call isinstance with routing profiles and list; why: control_plane_canary invokes isinstance while performing resident item get name for item in; the call advances that operation through its result or side effect. + ) if isinstance(routing_profiles, list) else None + # What: compute resident from get and item and routed rows and name and resident; why: or resident model a later reads resident, so control_plane_canary must retain the computed value under that name. + resident = [item.get("name") for item in routed_rows if item.get("resident")] + # What: compute routed a from next and item and routed rows and get and model a; why: or not isinstance routed a dict later reads routed a, so control_plane_canary must retain the computed value under that name. + routed_a = next((item for item in routed_rows if item.get("name") == "model-a"), None) + # What: compute listed a from next and item and model rows and get and model a; why: or not isinstance listed a dict later reads listed a, so control_plane_canary must retain the computed value under that name. + listed_a = next((item for item in model_rows if item.get("id") == "model-a"), None) + # What: compute listed alias a from next and item and model rows and get and compat; why: or not isinstance listed alias a dict later reads listed alias a, so control_plane_canary must retain the computed value under that name. + listed_alias_a = next( + # What: call item.get with id; why: control_plane_canary consumes the item.get return value while evaluating (item for item in model_rows if item.get("id") == "compat/model-a"), Non. + (item for item in model_rows if item.get("id") == "compat/model-a"), None + # What: complete the next call with item; why: control_plane_canary groups the supplied clauses as one next call before its value is consumed. + ) + # What: gate on routed names and profile names and resident and metrics raw and issubset before runtime error; why: control_plane_canary admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call operation.issubset with aliases; why: control_plane_canary invokes operation.issubset while performing or routed names profile names; the call advances that operation through its result or side effect. + not {"model-a", "model-b", "compat/model-a", "preferred-model"}.issubset(aliases) + # What: apply the or routed names profile names portion of the enclosing predicate; why: this clause remains in control_plane_canary\'s enclosing expression so its grouping and evaluation order stay intact. + or routed_names != profile_names + # What: call operation.issubset with routed names; why: control_plane_canary invokes operation.issubset while performing or resident model a; the call advances that operation through its result or side effect. + or not {"model-a", "model-b"}.issubset(routed_names) + # What: apply the or resident model a portion of the enclosing predicate; why: this clause remains in control_plane_canary\'s enclosing expression so its grouping and evaluation order stay intact. + or resident != ["model-a"] + # What: call profiles.get with active profile; why: control_plane_canary invokes profiles.get while performing or profiles get active routing profile is not; the call advances that operation through its result or side effect. + or profiles.get("activeProfile") != "model-a" + # What: call profiles.get with active routing profile; why: control_plane_canary invokes profiles.get while performing or not isinstance routing profiles list; the call advances that operation through its result or side effect. + or profiles.get("activeRoutingProfile") is not None + # What: call isinstance with routing profiles and list; why: control_plane_canary invokes isinstance while performing or not isinstance coding profile dict; the call advances that operation through its result or side effect. + or not isinstance(routing_profiles, list) + # What: call isinstance with coding profile and dict; why: control_plane_canary invokes isinstance while performing or coding profile get pins; the call advances that operation through its result or side effect. + or not isinstance(coding_profile, dict) + # What: call coding_profile.get with pins; why: control_plane_canary invokes coding_profile.get while performing disabled model profile model preferred model; the call advances that operation through its result or side effect. + or coding_profile.get("pins") != { + # What: map the disabled model field as the fixture input; why: control_plane_canary carries disabled model through "disabled-model": None, "profile-model": "preferred-model" into raise runtime error router log stream lacked the. + "disabled-model": None, "profile-model": "preferred-model", + # What: complete the enclosing predicate mapping with disabled model and profile model; why: control_plane_canary groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + # What: call isinstance with routed a and dict; why: control_plane_canary invokes isinstance while performing or routed a get check endpoint ready; the call advances that operation through its result or side effect. + or not isinstance(routed_a, dict) + # What: call routed_a.get with check endpoint; why: control_plane_canary invokes routed_a.get while performing or routed a get use model name model a; the call advances that operation through its result or side effect. + or routed_a.get("checkEndpoint") != "/ready" + # What: call routed_a.get with use model name; why: control_plane_canary invokes routed_a.get while performing or routed a get upstream timeout s; the call advances that operation through its result or side effect. + or routed_a.get("useModelName") != "model-a" + # What: call routed_a.get with upstream timeout s; why: control_plane_canary invokes routed_a.get while performing or routed a get display name qualification model a; the call advances that operation through its result or side effect. + or routed_a.get("upstreamTimeoutS") != 659 + # What: call routed_a.get with display name; why: control_plane_canary invokes routed_a.get while performing or routed a get metadata tier qualification type; the call advances that operation through its result or side effect. + or routed_a.get("displayName") != "Qualification model A" + # What: map the tier field as qualification; why: control_plane_canary carries tier through or routed_a.get("metadata") != {"tier": "qualification", "type": "operat into raise runtime error router log stream lacked the. + or routed_a.get("metadata") != {"tier": "qualification", "type": "operator"} + # What: call isinstance with listed a and dict; why: control_plane_canary invokes isinstance while performing or listed a get name qualification model a; the call advances that operation through its result or side effect. + or not isinstance(listed_a, dict) + # What: call listed_a.get with name; why: control_plane_canary invokes listed_a.get while performing or listed a get meta get freetoken; the call advances that operation through its result or side effect. + or listed_a.get("name") != "Qualification model A" + # What: call operation.get with freetoken; why: control_plane_canary invokes operation.get while performing aliases compat model a tier qualification type; the call advances that operation through its result or side effect. + or listed_a.get("meta", {}).get("freetoken") != { + # What: map the aliases field as compat and model a; why: control_plane_canary carries aliases through "aliases": ["compat/model-a"], "tier": "qualification", "type": "model" into raise runtime error router log stream lacked the. + "aliases": ["compat/model-a"], "tier": "qualification", "type": "model", + # What: complete the enclosing predicate mapping with aliases and tier and type; why: control_plane_canary groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + # What: call isinstance with listed alias a and dict; why: control_plane_canary invokes isinstance while performing or listed alias a get name qualification model a; the call advances that operation through its result or side effect. + or not isinstance(listed_alias_a, dict) + # What: call listed_alias_a.get with name; why: control_plane_canary invokes listed_alias_a.get while performing or listed alias a get meta get freetoken; the call advances that operation through its result or side effect. + or listed_alias_a.get("name") != "Qualification model A" + # What: call operation.get with freetoken; why: control_plane_canary invokes operation.get while performing model id model a tier qualification type alias; the call advances that operation through its result or side effect. + or listed_alias_a.get("meta", {}).get("freetoken") != { + # What: map the model id field as model a; why: control_plane_canary carries model id through "modelID": "model-a", "tier": "qualification", "type": "alias" into raise runtime error router log stream lacked the. + "modelID": "model-a", "tier": "qualification", "type": "alias", + # What: complete the enclosing predicate mapping with model id and tier and type; why: control_plane_canary groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + # What: call isinstance with namespaced stats and dict; why: control_plane_canary invokes isinstance while performing or b freetoken swap admissions total not in metrics raw; the call advances that operation through its result or side effect. + or not isinstance(namespaced_stats, dict) + # What: apply the or b freetoken swap admissions total not in metrics raw portion of the enclosing predicate; why: this clause remains in control_plane_canary\'s enclosing expression so its grouping and evaluation order stay intact. + or b"freetoken_swap_admissions_total" not in metrics_raw + # What: complete the enclosing predicate with if not model a model b compat model a preferred model issubset aliases; why: control_plane_canary groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: control_plane_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("authenticated native control-plane responses are inconsistent") + + # What: compute log request from request and request and base and urllib; why: with urllib request urlopen log request timeout as response later reads log request, so control_plane_canary must retain the computed value under that name. + log_request = urllib.request.Request( + # What: supply headers to _native_headers; why: control_plane_canary binds this native headers and base value to _native_headers's headers input. + base + "/router/logs?since=0", headers=_native_headers(base) + # What: complete the urllib.request.Request call with headers; why: control_plane_canary groups the supplied clauses as one urllib.request.Request call before its value is consumed. + ) + # What: compute log frame from bytearray; why: while len log frame later reads log frame, so control_plane_canary must retain the computed value under that name. + log_frame = bytearray() + # What: enter the urllib.request.urlopen managed context before if response headers get content type text event stream; why: control_plane_canary releases this resource or lock after if response headers get content type text event stream on both success and failure paths. + with urllib.request.urlopen(log_request, timeout=10) as response: + # What: gate on get content type and headers and response before runtime error; why: control_plane_canary admits runtime error only for this predicate and excludes the opposite state. + if response.headers.get_content_type() != "text/event-stream": + # What: raise RuntimeError for the caller; why: control_plane_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("router log endpoint did not return SSE") + # What: iterate across len and log frame to perform line and readline and response; why: control_plane_canary repeats the body only while or for the loop header admits an iteration. + while len(log_frame) <= 64 * 1024: + # What: compute line from readline and response; why: if not line later reads line, so control_plane_canary must retain the computed value under that name. + line = response.readline() + # What: gate on line before the computed value; why: control_plane_canary admits the computed value only for this predicate and excludes the opposite state. + if not line: + # What: apply the break portion of the enclosing predicate; why: this clause remains in control_plane_canary\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: call log_frame.extend with line; why: control_plane_canary invokes log_frame.extend while performing if b management loaded in log frame; the call advances that operation through its result or side effect. + log_frame.extend(line) + # What: gate on log frame before the computed value; why: control_plane_canary admits the computed value only for this predicate and excludes the opposite state. + if b"management_loaded" in log_frame: + # What: apply the break portion of the enclosing predicate; why: this clause remains in control_plane_canary\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: gate on log frame and len before runtime error; why: control_plane_canary admits runtime error only for this predicate and excludes the opposite state. + if len(log_frame) > 64 * 1024 or b"management_loaded" not in log_frame: + # What: raise RuntimeError for the caller; why: control_plane_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("router log stream lacked the bounded management event") + + # What: preserve the exact artifacts control v1 models json write bytes models raw literal fragment; why: control_plane_canary passes this fragment verbatim through (artifacts / "control-v1-models.json").write_bytes(models_raw), because changing it would alter a protocol payload, serialized fixture, or public mess. + (artifacts / "control-v1-models.json").write_bytes(models_raw) + # What: preserve the exact artifacts control router models json write bytes routed raw literal fragment; why: control_plane_canary passes this fragment verbatim through (artifacts / "control-router-models.json").write_bytes(routed_raw), because changing it would alter a protocol payload, serialized fixture, or pub. + (artifacts / "control-router-models.json").write_bytes(routed_raw) + # What: preserve the exact artifacts control router profiles json write bytes profiles raw literal fragment; why: control_plane_canary passes this fragment verbatim through (artifacts / "control-router-profiles.json").write_bytes(profiles_raw), because changing it would alter a protocol payload, serialized fixture. + (artifacts / "control-router-profiles.json").write_bytes(profiles_raw) + # What: preserve the exact artifacts control performance json write bytes performance raw literal fragment; why: control_plane_canary passes this fragment verbatim through (artifacts / "control-performance.json").write_bytes(performance_raw), because changing it would alter a protocol payload, serialized fixture. + (artifacts / "control-performance.json").write_bytes(performance_raw) + # What: preserve the exact artifacts control metrics prom write bytes metrics raw literal fragment; why: control_plane_canary passes this fragment verbatim through (artifacts / "control-metrics.prom").write_bytes(metrics_raw), because changing it would alter a protocol payload, serialized fixture, or public messag. + (artifacts / "control-metrics.prom").write_bytes(metrics_raw) + # What: preserve the exact artifacts control router log sse write bytes log frame literal fragment; why: control_plane_canary passes this fragment verbatim through (artifacts / "control-router-log.sse").write_bytes(log_frame), because changing it would alter a protocol payload, serialized fixture, or public messag. + (artifacts / "control-router-log.sse").write_bytes(log_frame) + # What: iterate across items and alternate auth raw to perform write bytes and raw and artifacts and name; why: control_plane_canary repeats the body only while or for the loop header admits an iteration. + for name, raw in alternate_auth_raw.items(): + # What: preserve the exact artifacts f control auth name json write bytes literal fragment; why: control_plane_canary passes this fragment verbatim through (artifacts / f"control-auth-{name}.json").write_bytes(raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / f"control-auth-{name}.json").write_bytes(raw) + # What: return; why: the caller consumes this value as the function’s success-path result. + return { + # What: map the unauthenticated control rejected field as true; why: control_plane_canary carries unauthenticated control rejected into "unauthenticatedControlRejected": True. + "unauthenticatedControlRejected": True, + # What: map the unauthenticated inference rejected field as true; why: control_plane_canary carries unauthenticated inference rejected into "unauthenticatedInferenceRejected": True. + "unauthenticatedInferenceRejected": True, + # What: map the alias count field as len and aliases; why: control_plane_canary carries alias count into "aliasCount": len(aliases). + "aliasCount": len(aliases), + # What: map the selector listed field as aliases and preferred model; why: control_plane_canary carries selector listed into "selectorListed": "preferred-model" in aliases. + "selectorListed": "preferred-model" in aliases, + # What: map the profile count field as len and profile names; why: control_plane_canary carries profile count into "profileCount": len(profile_names). + "profileCount": len(profile_names), + # What: map the routing profile listed field as true; why: control_plane_canary carries routing profile listed into "routingProfileListed": True. + "routingProfileListed": True, + # What: map the configured readiness target verified field as true; why: control_plane_canary carries configured readiness target verified into "configuredReadinessTargetVerified": True. + "configuredReadinessTargetVerified": True, + # What: map the configured upstream model name verified field as true; why: control_plane_canary carries configured upstream model name verified into "configuredUpstreamModelNameVerified": True. + "configuredUpstreamModelNameVerified": True, + # What: map the configured upstream timeout verified field as true; why: control_plane_canary carries configured upstream timeout verified into "configuredUpstreamTimeoutVerified": True. + "configuredUpstreamTimeoutVerified": True, + # What: map the configured model metadata verified field as true; why: control_plane_canary carries configured model metadata verified into "configuredModelMetadataVerified": True. + "configuredModelMetadataVerified": True, + # What: map the resident profile field as model a; why: control_plane_canary carries resident profile into "residentProfile": "model-a". + "residentProfile": "model-a", + # What: map the model list alias verified field as true; why: control_plane_canary carries model list alias verified into "modelListAliasVerified": True. + "modelListAliasVerified": True, + # What: map the namespaced upstream verified field as true; why: control_plane_canary carries namespaced upstream verified into "namespacedUpstreamVerified": True. + "namespacedUpstreamVerified": True, + # What: map the api key forms verified field as bearer and basic and x api key; why: control_plane_canary carries api key forms verified into "apiKeyFormsVerified": ["bearer", "basic", "x-api-key"]. + "apiKeyFormsVerified": ["bearer", "basic", "x-api-key"], + # What: map the metrics available field as true; why: control_plane_canary carries metrics available into "metricsAvailable": True. + "metricsAvailable": True, + # What: map the periodic performance available field as true; why: control_plane_canary carries periodic performance available into "periodicPerformanceAvailable": True. + "periodicPerformanceAvailable": True, + # What: map the router log sse available field as true; why: control_plane_canary carries router log sse available into "routerLogSseAvailable": True. + "routerLogSseAvailable": True, + # What: map the passed field as true; why: control_plane_canary carries passed into "passed": True. + "passed": True, + # What: execute the grouped source fragment; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + } + + +# What: define selector_canary around base and artifacts; why: its direct callers call selector_canary for selector canary and rely on this exact input and result contract. +def selector_canary(base: str, artifacts: Path) -> dict: + """Prove a warm virtual ID reuses the resident target without a swap.""" + # What: document prove a warm virtual id reuses in the selector_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand selector canary behavior without executing it. + # What: compute and before from request json and base and router and status; why: value after request json base router status later reads and before, so selector_canary must retain the computed value under that name. + _, before = request_json(base + "/router/status") + # What: compute prior activations from get and before and activations; why: if before get active profile model a or not later reads prior activations, so selector_canary must retain the computed value under that name. + prior_activations = before.get("activations") + # What: gate on get and isinstance and prior activations and int and before before runtime error; why: selector_canary admits runtime error only for this predicate and excludes the opposite state. + if before.get("activeProfile") != "model-a" or not isinstance(prior_activations, int): + # What: raise RuntimeError for the caller; why: selector_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("warm selector canary requires resident model-a") + # What: compute raw and completion from canary and base and preferred model and false; why: artifacts warm selector sse write bytes raw later reads raw and completion, so selector_canary must retain the computed value under that name. + raw, completion = canary(base, "preferred-model", direct=False) + # What: compute and after from request json and base and router and status; why: the enclosing return or state update later reads and after, so selector_canary must retain the computed value under that name. + _, after = request_json(base + "/router/status") + # What: gate on prior activations and get and completion and after before runtime error; why: selector_canary admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call completion.get with passed; why: selector_canary invokes completion.get while performing or after get active profile model a; the call advances that operation through its result or side effect. + completion.get("passed") is not True + # What: call after.get with active profile; why: selector_canary invokes after.get while performing or after get active requests; the call advances that operation through its result or side effect. + or after.get("activeProfile") != "model-a" + # What: call after.get with active requests; why: selector_canary invokes after.get while performing or after get activations prior activations; the call advances that operation through its result or side effect. + or after.get("activeRequests") != 0 + # What: call after.get with activations; why: selector_canary consumes the after.get return value while evaluating or after.get("activations") != prior_activations. + or after.get("activations") != prior_activations + # What: complete the enclosing predicate with if completion get passed is not true or after get active profile; why: selector_canary groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: selector_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("warm selector did not reuse the resident target") + # What: preserve the exact artifacts warm selector sse write bytes raw literal fragment; why: selector_canary passes this fragment verbatim through (artifacts / "warm-selector.sse").write_bytes(raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / "warm-selector.sse").write_bytes(raw) + # What: return strategy and resolved profile and activation delta and passed and warm from selector_canary; why: selector_canary exposes strategy and resolved profile and activation delta and passed and warm so its caller can continue with the function\'s computed outcome. + return { + # What: map the strategy field as warm; why: selector_canary carries strategy into "strategy": "warm". + "strategy": "warm", + # What: map the resolved profile field as model a; why: selector_canary carries resolved profile into "resolvedProfile": "model-a". + "resolvedProfile": "model-a", + # What: map the activation delta field as 0; why: selector_canary carries activation delta into "activationDelta": 0. + "activationDelta": 0, + # What: map the passed field as true; why: selector_canary carries passed into "passed": True. + "passed": True, + # What: complete the enclosing predicate mapping with strategy and resolved profile and activation delta and passed; why: selector_canary groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + +# What: define routing_profile_canary around base and artifacts; why: its direct callers call routing_profile_canary for routing profile canary and rely on this exact input and result contract. +def routing_profile_canary(base: str, artifacts: Path) -> dict: + """Prove an active profile pin composes through a warm selector, then clear it.""" + # What: document prove an active profile pin composes in the routing_profile_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand routing profile canary behavior without executing it. + # What: compute and before from request json and base and router and status; why: value activated request json later reads and before, so routing_profile_canary must retain the computed value under that name. + _, before = request_json(base + "/router/status") + # What: compute prior activations from get and before and activations; why: if before get active profile model a or not later reads prior activations, so routing_profile_canary must retain the computed value under that name. + prior_activations = before.get("activations") + # What: gate on get and isinstance and prior activations and int and before before runtime error; why: routing_profile_canary admits runtime error only for this predicate and excludes the opposite state. + if before.get("activeProfile") != "model-a" or not isinstance(prior_activations, int): + # What: raise RuntimeError for the caller; why: routing_profile_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("routing profile canary requires resident model-a") + # What: compute raw from the named fixture input; why: raw completion canary base profile model direct later reads raw, so routing_profile_canary must retain the computed value under that name. + raw = b"" + # What: compute listed raw from the named fixture input; why: listed raw listed request json base v1 models later reads listed raw, so routing_profile_canary must retain the computed value under that name. + listed_raw = b"" + # What: establish the handler boundary for the protected operation; why: routing_profile_canary routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: compute and activated from request json and base and router and profiles and active; why: value after request json base router status later reads and activated, so routing_profile_canary must retain the computed value under that name. + _, activated = request_json( + # What: map the name field as coding; why: routing_profile_canary carries name through and activated into value after request json base router status. + base + "/router/profiles/active", {"name": "coding"}, method="PUT" + # What: complete the request_json call with method; why: routing_profile_canary groups the supplied clauses as one request_json call before its value is consumed. + ) + # What: map the active field as coding; why: routing_profile_canary carries active through if activated != {"active": "coding"} into raise runtime error routing profile pin did not. + if activated != {"active": "coding"}: + # What: raise RuntimeError for the caller; why: routing_profile_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("routing profile activation was not acknowledged") + # What: compute listed raw and listed from request json and base and v1 and models and 10; why: artifacts routing profile models json write bytes listed raw later reads listed raw and listed, so routing_profile_canary must retain the computed value under that name. + listed_raw, listed = request_json(base + "/v1/models", timeout=10) + # What: compute listed ids from get and item and isinstance and dict; why: if profile model not in listed ids or later reads listed ids, so routing_profile_canary must retain the computed value under that name. + listed_ids = { + # What: call item.get with id; why: routing_profile_canary consumes the item.get return value while evaluating item.get("id") for item in listed.get("data", []) if isinstance(item, di. + item.get("id") for item in listed.get("data", []) if isinstance(item, dict) + # What: complete the listed_ids expression with listed ids item get id for item in listed get data if; why: routing_profile_canary groups the supplied clauses as one listed_ids expression before its value is consumed. + } + # What: gate on listed ids before runtime error; why: routing_profile_canary admits runtime error only for this predicate and excludes the opposite state. + if "profile-model" not in listed_ids or "disabled-model" in listed_ids: + # What: raise RuntimeError for the caller; why: routing_profile_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("active routing profile model listing is inconsistent") + # What: compute raw and completion from canary and base and profile model and false; why: artifacts routing profile sse write bytes raw later reads raw and completion, so routing_profile_canary must retain the computed value under that name. + raw, completion = canary(base, "profile-model", direct=False) + # What: compute and after from request json and base and router and status; why: value cleared request json later reads and after, so routing_profile_canary must retain the computed value under that name. + _, after = request_json(base + "/router/status") + # What: gate on prior activations and get and completion and after before runtime error; why: routing_profile_canary admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call completion.get with passed; why: routing_profile_canary invokes completion.get while performing or after get active routing profile coding; the call advances that operation through its result or side effect. + completion.get("passed") is not True + # What: call after.get with active routing profile; why: routing_profile_canary invokes after.get while performing or after get active profile model a; the call advances that operation through its result or side effect. + or after.get("activeRoutingProfile") != "coding" + # What: call after.get with active profile; why: routing_profile_canary invokes after.get while performing or after get active requests; the call advances that operation through its result or side effect. + or after.get("activeProfile") != "model-a" + # What: call after.get with active requests; why: routing_profile_canary invokes after.get while performing or after get activations prior activations; the call advances that operation through its result or side effect. + or after.get("activeRequests") != 0 + # What: call after.get with activations; why: routing_profile_canary consumes the after.get return value while evaluating or after.get("activations") != prior_activations. + or after.get("activations") != prior_activations + # What: complete the enclosing predicate with if completion get passed is not true or after get active routing profile; why: routing_profile_canary groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: routing_profile_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("routing profile pin did not reuse the resident selector target") + # What: run value cleared request json on every exit path; why: routing_profile_canary performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: compute and cleared from request json and base and router and profiles and active; why: the enclosing return or state update later reads and cleared, so routing_profile_canary must retain the computed value under that name. + _, cleared = request_json( + # What: map the name field as the fixture input; why: routing_profile_canary carries name through and cleared into the enclosing return or state update. + base + "/router/profiles/active", {"name": None}, method="PUT" + # What: complete the request_json call with method; why: routing_profile_canary groups the supplied clauses as one request_json call before its value is consumed. + ) + # What: map the active field as the fixture input; why: routing_profile_canary carries active into if cleared != {"active": None}. + if cleared != {"active": None}: + # What: raise RuntimeError for the caller; why: routing_profile_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("routing profile was not cleared after its canary") + # What: preserve the exact artifacts routing profile sse write bytes raw literal fragment; why: routing_profile_canary passes this fragment verbatim through (artifacts / "routing-profile.sse").write_bytes(raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / "routing-profile.sse").write_bytes(raw) + # What: preserve the exact artifacts routing profile models json write bytes listed raw literal fragment; why: routing_profile_canary passes this fragment verbatim through (artifacts / "routing-profile-models.json").write_bytes(listed_raw), because changing it would alter a protocol payload, serialized fixture, or. + (artifacts / "routing-profile-models.json").write_bytes(listed_raw) + # What: return; why: the caller consumes this value as the function’s success-path result. + return { + # What: map the profile activated field as true; why: routing_profile_canary carries profile activated into "profileActivated": True. + "profileActivated": True, + # What: map the profile cleared field as true; why: routing_profile_canary carries profile cleared into "profileCleared": True. + "profileCleared": True, + # What: map the selector composed field as true; why: routing_profile_canary carries selector composed into "selectorComposed": True. + "selectorComposed": True, + # What: map the resolved profile field as model a; why: routing_profile_canary carries resolved profile into "resolvedProfile": "model-a". + "resolvedProfile": "model-a", + # What: map the activation delta field as 0; why: routing_profile_canary carries activation delta into "activationDelta": 0. + "activationDelta": 0, + # What: map the passed field as true; why: routing_profile_canary carries passed into "passed": True. + "passed": True, + # What: complete the enclosing predicate mapping with profile activated and profile cleared and selector composed and resolved profile and activation delta; why: routing_profile_canary groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + +# What: define capture_hardware around base and artifacts and label; why: its direct callers call capture_hardware for capture hardware and rely on this exact input and result contract. +def capture_hardware(base: str, artifacts: Path, label: str) -> dict: + """Keep per-trial process and memory observations in the private artifact set.""" + # What: document keep per trial process and memory observations in the capture_hardware docstring; why: introspection and maintainers read this exact docstring fragment to understand capture hardware behavior without executing it. + # What: compute raw and hardware from request json and base and router and hardware; why: artifacts f label hardware json write bytes raw later reads raw and hardware, so capture_hardware must retain the computed value under that name. + raw, hardware = request_json(base + "/router/hardware") + # What: compute engine from get and hardware and engine; why: if not isinstance engine dict or later reads engine, so capture_hardware must retain the computed value under that name. + engine = hardware.get("engine") + # What: compute memory from get and hardware and memory; why: if not isinstance engine dict or later reads memory, so capture_hardware must retain the computed value under that name. + memory = hardware.get("memory") + # What: gate on isinstance and engine and dict and memory before runtime error; why: capture_hardware admits runtime error only for this predicate and excludes the opposite state. + if not isinstance(engine, dict) or not isinstance(memory, dict): + # What: raise RuntimeError for the caller; why: capture_hardware stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("router hardware observation has an invalid shape") # noqa: TRY004 -- malformed remote telemetry is an operational failure. + # What: gate on get and isinstance and int and engine before runtime error; why: capture_hardware admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call engine.get with running; why: capture_hardware invokes engine.get while performing or not isinstance engine get pid int; the call advances that operation through its result or side effect. + not engine.get("running") + # What: call isinstance with get and engine and pid and int; why: capture_hardware invokes isinstance while performing or engine pid; the call advances that operation through its result or side effect. + or not isinstance(engine.get("pid"), int) + # What: apply the or engine pid portion of the enclosing predicate; why: this clause remains in capture_hardware\'s enclosing expression so its grouping and evaluation order stay intact. + or engine["pid"] <= 0 + # What: call isinstance with get and engine and port and int; why: capture_hardware invokes isinstance while performing or not engine port; the call advances that operation through its result or side effect. + or not isinstance(engine.get("port"), int) + # What: apply the or not engine port portion of the enclosing predicate; why: this clause remains in capture_hardware\'s enclosing expression so its grouping and evaluation order stay intact. + or not 1 <= engine["port"] <= 65535 + # What: complete the enclosing predicate with if not engine get running or not isinstance engine get pid; why: capture_hardware groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: capture_hardware stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("router hardware observation does not identify a running engine") + # What: gate on all and isinstance and int and key and get before runtime error; why: capture_hardware admits runtime error only for this predicate and excludes the opposite state. + if not all(isinstance(memory.get(key), int) for key in ("ramBytes", "vramBytes")): + # What: raise RuntimeError for the caller; why: capture_hardware stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("router hardware observation lacks byte measurements") + # What: gate on get and memory before runtime error; why: capture_hardware admits runtime error only for this predicate and excludes the opposite state. + if memory.get("ramAvailable") is not True or memory.get("vramAvailable") is not True: + # What: raise RuntimeError for the caller; why: capture_hardware stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("router hardware observation contains unavailable memory measurements") + # What: gate on memory before runtime error; why: capture_hardware admits runtime error only for this predicate and excludes the opposite state. + if memory["ramBytes"] <= 0 or memory["vramBytes"] <= 0: + # What: raise RuntimeError for the caller; why: capture_hardware stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("router hardware observation contains non-positive memory measurements") + # What: gate on all and key and isinstance and str and memory before runtime error; why: capture_hardware admits runtime error only for this predicate and excludes the opposite state. + if not all(isinstance(memory.get(key), str) and memory[key] for key in ("ramSource", "vramSource")): + # What: raise RuntimeError for the caller; why: capture_hardware stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("router hardware observation lacks memory measurement sources") + # What: preserve the exact artifacts f label hardware json write bytes raw literal fragment; why: capture_hardware passes this fragment verbatim through (artifacts / f"{label}.hardware.json").write_bytes(raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / f"{label}.hardware.json").write_bytes(raw) + # What: return hardware from capture_hardware; why: capture_hardware exposes hardware so its caller can continue with the function\'s computed outcome. + return hardware + + +# What: define validate_re_adoption around before and after and router; why: its direct callers call validate_re_adoption for validate re adoption and rely on this exact input and result contract. +def validate_re_adoption(before: dict, after: dict, router: dict) -> dict: + """Validate that a replacement daemon bound, rather than replaced, one engine.""" + # What: document validate that a replacement daemon bound in the validate_re_adoption docstring; why: introspection and maintainers read this exact docstring fragment to understand validate re adoption behavior without executing it. + # What: compute old pid and old port from get and before and pid and port; why: or not isinstance old pid int or later reads old pid and old port, so validate_re_adoption must retain the computed value under that name. + old_pid, old_port = before.get("pid"), before.get("port") + # What: gate on old pid and get and isinstance and int and old port before runtime error; why: validate_re_adoption admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call before.get with running; why: validate_re_adoption invokes before.get while performing or not isinstance old pid int or; the call advances that operation through its result or side effect. + not before.get("running") + # What: call isinstance with old pid and int; why: validate_re_adoption invokes isinstance while performing or not isinstance old port int or; the call advances that operation through its result or side effect. + or not isinstance(old_pid, int) or old_pid <= 0 + # What: call isinstance with old port and int; why: validate_re_adoption consumes the isinstance return value while evaluating or not isinstance(old_port, int) or not 1 <= old_port <= 65535. + or not isinstance(old_port, int) or not 1 <= old_port <= 65535 + # What: complete the enclosing predicate with if not before get running or not isinstance old pid int; why: validate_re_adoption groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: validate_re_adoption stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("pre-restart engine identity is invalid") + # What: gate on old pid and old port and get and after and router before runtime error; why: validate_re_adoption admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call after.get with pid; why: validate_re_adoption invokes after.get while performing or after get port old port; the call advances that operation through its result or side effect. + after.get("pid") != old_pid + # What: call after.get with port; why: validate_re_adoption invokes after.get while performing or after get adopted is not; the call advances that operation through its result or side effect. + or after.get("port") != old_port + # What: call after.get with adopted; why: validate_re_adoption invokes after.get while performing or router get active profile model a; the call advances that operation through its result or side effect. + or after.get("adopted") is not True + # What: call router.get with active profile; why: validate_re_adoption invokes router.get while performing or router get active identity matches engine is not; the call advances that operation through its result or side effect. + or router.get("activeProfile") != "model-a" + # What: call router.get with active identity matches engine; why: validate_re_adoption invokes router.get while performing or router get activations; the call advances that operation through its result or side effect. + or router.get("activeIdentityMatchesEngine") is not True + # What: call router.get with activations; why: validate_re_adoption consumes the router.get return value while evaluating or router.get("activations") != 0. + or router.get("activations") != 0 + # What: complete the enclosing predicate with if after get pid differs from old pid or after get port; why: validate_re_adoption groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: validate_re_adoption stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("replacement daemon did not bind the exact adopted residency") + # What: return profile and same pid and same port and manager adopted and activation delta from validate_re_adoption; why: validate_re_adoption exposes profile and same pid and same port and manager adopted and activation delta so its caller can continue with the function\'s computed outcome. + return { + # What: map the profile field as model a; why: validate_re_adoption carries profile into "profile": "model-a", "samePid": True, "samePort": True. + "profile": "model-a", "samePid": True, "samePort": True, + # What: map the manager adopted field as true; why: validate_re_adoption carries manager adopted into "managerAdopted": True, "activationDelta": 0. + "managerAdopted": True, "activationDelta": 0, + # What: complete the enclosing predicate mapping with profile and same pid and same port and manager adopted and activation delta; why: validate_re_adoption groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + +# What: define require_listener_closed around port; why: its direct callers call require_listener_closed for require listener closed and rely on this exact input and result contract. +def require_listener_closed(port: int) -> None: + """Fail the qualification if a temporary engine listener survived cleanup.""" + # What: document fail the qualification if a temporary in the require_listener_closed docstring; why: introspection and maintainers read this exact docstring fragment to understand require listener closed behavior without executing it. + # What: enter the socket.socket managed context before connection settimeout; why: require_listener_closed releases this resource or lock after connection settimeout on both success and failure paths. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as connection: + # What: call connection.settimeout with 1; why: require_listener_closed invokes connection.settimeout while performing if connection connect ex port; the call advances that operation through its result or side effect. + connection.settimeout(1) + # What: gate on connect ex and connection and port before runtime error; why: require_listener_closed admits runtime error only for this predicate and excludes the opposite state. + if connection.connect_ex(("127.0.0.1", port)) == 0: + # What: raise RuntimeError for the caller; why: require_listener_closed stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("temporary engine listener remains reachable after cleanup") + + +# What: define require_listener_open around port; why: its direct callers call require_listener_open for require listener open and rely on this exact input and result contract. +def require_listener_open(port: int) -> None: + """Require a detached test-owned engine to remain reachable for re-adoption.""" + # What: document require a detached test owned engine to in the require_listener_open docstring; why: introspection and maintainers read this exact docstring fragment to understand require listener open behavior without executing it. + # What: enter the socket.socket managed context before connection settimeout; why: require_listener_open releases this resource or lock after connection settimeout on both success and failure paths. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as connection: + # What: call connection.settimeout with 1; why: require_listener_open invokes connection.settimeout while performing if connection connect ex port; the call advances that operation through its result or side effect. + connection.settimeout(1) + # What: gate on connect ex and connection and port before runtime error; why: require_listener_open admits runtime error only for this predicate and excludes the opposite state. + if connection.connect_ex(("127.0.0.1", port)) != 0: + # What: raise RuntimeError for the caller; why: require_listener_open stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("detached engine listener did not survive daemon restart") + + +# What: define stop_detached_engine around pid and port; why: its direct callers call stop_detached_engine for stop detached engine and rely on this exact input and result contract. +def stop_detached_engine(pid: int, port: int) -> None: + """Best-effort cleanup for the exact test-owned engine during a restart gap.""" + # What: document best effort cleanup for the exact test owned in the stop_detached_engine docstring; why: introspection and maintainers read this exact docstring fragment to understand stop detached engine behavior without executing it. + # What: establish the handler boundary for the protected operation; why: stop_detached_engine routes failures to process lookup error while preserving cleanup and success flow. + try: + # What: call os.killpg with pid and sigterm and signal; why: stop_detached_engine invokes os.killpg while performing except process lookup error; the call advances that operation through its result or side effect. + os.killpg(pid, signal.SIGTERM) + # What: handle process lookup error by return; why: stop_detached_engine converts that failure into this concrete recovery, response, or cleanup behavior. + except ProcessLookupError: + # What: return no value from stop_detached_engine; why: stop_detached_engine returns no value to callers that depend on its completed result. + return + # What: compute deadline from monotonic and time and 15; why: while time monotonic deadline later reads deadline, so stop_detached_engine must retain the computed value under that name. + deadline = time.monotonic() + 15 + # What: iterate across deadline and monotonic and time to perform runtime error and require listener closed and port and sleep and time; why: stop_detached_engine repeats the body only while or for the loop header admits an iteration. + while time.monotonic() < deadline: + # What: establish the handler boundary for the protected operation; why: stop_detached_engine routes failures to runtime error while preserving cleanup and success flow. + try: + # What: call require_listener_closed with port; why: stop_detached_engine invokes require_listener_closed while performing return; the call advances that operation through its result or side effect. + require_listener_closed(port) + # What: return no value from stop_detached_engine; why: stop_detached_engine returns no value to callers that depend on its completed result. + return + # What: handle runtime error by time sleep 0 1; why: stop_detached_engine converts that failure into this concrete recovery, response, or cleanup behavior. + except RuntimeError: + # What: call time.sleep with 0 1; why: stop_detached_engine invokes time.sleep while performing try; the call advances that operation through its result or side effect. + time.sleep(0.1) + # What: establish the handler boundary for the protected operation; why: stop_detached_engine routes failures to process lookup error while preserving cleanup and success flow. + try: + # What: call os.killpg with pid and sigkill and signal; why: stop_detached_engine invokes os.killpg while performing except process lookup error; the call advances that operation through its result or side effect. + os.killpg(pid, signal.SIGKILL) + # What: handle process lookup error by pass; why: stop_detached_engine converts that failure into this concrete recovery, response, or cleanup behavior. + except ProcessLookupError: + # What: ignore the anticipated exception handled by this branch; why: stop_detached_engine continues its retry or cleanup path instead of re-raising that transient failure. + pass + # What: compute deadline from monotonic and time and 5; why: while time monotonic deadline later reads deadline, so stop_detached_engine must retain the computed value under that name. + deadline = time.monotonic() + 5 + # What: iterate across deadline and monotonic and time to perform runtime error and require listener closed and port and sleep and time; why: stop_detached_engine repeats the body only while or for the loop header admits an iteration. + while time.monotonic() < deadline: + # What: establish the handler boundary for the protected operation; why: stop_detached_engine routes failures to runtime error while preserving cleanup and success flow. + try: + # What: call require_listener_closed with port; why: stop_detached_engine invokes require_listener_closed while performing return; the call advances that operation through its result or side effect. + require_listener_closed(port) + # What: return no value from stop_detached_engine; why: stop_detached_engine returns no value to callers that depend on its completed result. + return + # What: handle runtime error by time sleep 0 1; why: stop_detached_engine converts that failure into this concrete recovery, response, or cleanup behavior. + except RuntimeError: + # What: call time.sleep with 0 1; why: stop_detached_engine invokes time.sleep while performing raise runtime error detached test owned engine survived; the call advances that operation through its result or side effect. + time.sleep(0.1) + # What: raise RuntimeError for the caller; why: stop_detached_engine stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("detached test-owned engine survived cleanup") + + +# What: discover the exact engine process currently owned by the temporary daemon; why: failure cleanup must not leak a model process that consumes unified memory after the daemon exits. +def running_engine_identity(base: str) -> tuple[int, int] | None: + # What: query the daemon's authoritative engine status; why: process cleanup must use the manager-recorded PID and listener rather than broad process matching. + try: + # What: retain the decoded engine status response; why: validated running state, PID, and port are all required before a process can be treated as test-owned. + _, engine = request_json(base + "/engine/status", timeout=10) + # What: treat an unavailable daemon status endpoint as no capturable engine; why: cleanup continues through existing process-group and detached-engine safeguards. + except (OSError, ValueError, urllib.error.HTTPError): + # What: return no identity when ownership cannot be proven; why: cleanup must never signal an unverified process. + return None + # What: extract the manager-recorded process identity fields; why: both values are needed to terminate the process group and verify listener closure. + pid, port = engine.get("pid"), engine.get("port") + # What: accept only a running engine with valid integer identity fields; why: malformed or idle status must not be converted into an unsafe signal target. + if engine.get("running") is True and isinstance(pid, int) and isinstance(port, int): + # What: return the exact test-owned process identity; why: the caller can perform bounded targeted cleanup on every exit path. + return pid, port + # What: return no identity for an idle or invalid engine status; why: there is no proven process for cleanup to stop. + return None + + +# What: define reload_conflict_canary around base and catalog path and model a and model b and api key; why: its direct callers call reload_conflict_canary for reload conflict canary and rely on this exact input and result contract. +def reload_conflict_canary( + # What: declare the base input for reload_conflict_canary; why: reload_conflict_canary consumes base during value status request json base router status, so callers must bind it with the other signature inputs. + base: str, catalog_path: Path, model_a: str, model_b: str, *, api_key: str | None = None +# What: complete the enclosing predicate with dict; why: reload_conflict_canary groups the supplied clauses as one enclosing predicate expression before its value is consumed. +) -> dict: + """Prove an active profile's scheduler policy cannot change under its engine.""" + # What: document prove an active profile s scheduler in the reload_conflict_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand reload conflict canary behavior without executing it. + # What: call catalog_path.write_text with native catalog text and model a and model b and api key and 1; why: reload_conflict_canary invokes catalog_path.write_text while performing native catalog text model a model b model a priority api key api key; the call advances that operation through its result or side effect. + catalog_path.write_text( + # What: supply model a priority to native_catalog_text; why: reload_conflict_canary binds this 1 value to native_catalog_text's model a priority input. + native_catalog_text(model_a, model_b, model_a_priority=1, api_key=api_key), + # What: preserve the exact encoding utf 8 literal fragment; why: reload_conflict_canary passes this fragment verbatim through encoding="utf-8", because changing it would alter a protocol payload, serialized fixture, or public message. + encoding="utf-8", + # What: complete the catalog_path.write_text call with encoding; why: reload_conflict_canary groups the supplied clauses as one catalog_path.write_text call before its value is consumed. + ) + # What: establish the handler boundary for the protected operation; why: reload_conflict_canary routes failures to httperror and error and urllib while preserving cleanup and success flow. + try: + # What: preserve the exact request json base router reload timeout literal fragment; why: reload_conflict_canary passes this fragment verbatim through request_json(base + "/router/reload", {}, timeout=30), because changing it would alter a protocol payload, serialized fixture, or public message. + request_json(base + "/router/reload", {}, timeout=30) + # What: handle httperror and error and urllib by if exc code differs from 409; why: reload_conflict_canary converts that failure into this concrete recovery, response, or cleanup behavior. + except urllib.error.HTTPError as exc: + # What: gate on code and exc before exc and runtime error; why: reload_conflict_canary admits exc and runtime error only for this predicate and excludes the opposite state. + if exc.code != 409: + # What: raise RuntimeError for the caller; why: reload_conflict_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("active catalog conflict returned the wrong status") from exc + # What: select the remaining branch that performs raise runtime error active catalog scheduler redefinition; why: reload_conflict_canary covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: raise RuntimeError for the caller; why: reload_conflict_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("active catalog scheduler redefinition was accepted") + # What: compute and status from request json and base and router and status and 30; why: the enclosing return or state update later reads and status, so reload_conflict_canary must retain the computed value under that name. + _, status = request_json(base + "/router/status", timeout=30) + # What: gate on get and status before runtime error; why: reload_conflict_canary admits runtime error only for this predicate and excludes the opposite state. + if status.get("activeProfile") != "model-a" or status.get("activeIdentityMatchesEngine") is not True: + # What: raise RuntimeError for the caller; why: reload_conflict_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("rejected catalog replacement changed active engine identity") + # What: return active profile and rejected status and active identity preserved and passed and model a from reload_conflict_canary; why: qualify_native_router needs this line to preserve the surrounding expression or collection structure. + return { + # What: map the active profile field as model a; why: reload_conflict_canary carries active profile into "activeProfile": "model-a". + "activeProfile": "model-a", + # What: map the rejected status field as 409; why: reload_conflict_canary carries rejected status into "rejectedStatus": 409. + "rejectedStatus": 409, + # What: map the active identity preserved field as true; why: reload_conflict_canary carries active identity preserved into "activeIdentityPreserved": True. + "activeIdentityPreserved": True, + # What: map the passed field as true; why: reload_conflict_canary carries passed into "passed": True. + "passed": True, + # What: complete the enclosing predicate mapping with active profile and rejected status and active identity preserved and passed; why: reload_conflict_canary groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + +# What: define failed_switch_canary around base and model and restored model; why: its direct callers call failed_switch_canary for failed switch canary and rely on this exact input and result contract. +def failed_switch_canary(base: str, model: str, restored_model: str) -> tuple[bytes, bytes, dict]: + """Require a failed disposable load to restore the prior resident engine.""" + # What: document require a failed disposable load to in the failed_switch_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand failed switch canary behavior without executing it. + # What: compute and before from request json and base and router and status; why: value pending before request json base accounting pending later reads and before, so failed_switch_canary must retain the computed value under that name. + _, before = request_json(base + "/router/status") + # What: compute and pending before from request json and base and accounting and pending; why: value after request json base router status later reads and pending before, so failed_switch_canary must retain the computed value under that name. + _, pending_before = request_json(base + "/accounting/pending") + # What: compute receipts before from get and pending before and receipts; why: if not isinstance receipts before list later reads receipts before, so failed_switch_canary must retain the computed value under that name. + receipts_before = pending_before.get("receipts") + # What: gate on isinstance and receipts before and list before runtime error; why: failed_switch_canary admits runtime error only for this predicate and excludes the opposite state. + if not isinstance(receipts_before, list): + # What: raise RuntimeError for the caller; why: failed_switch_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("accounting outbox response has an invalid shape") # noqa: TRY004 -- malformed remote state is an operational failure. + # What: compute before ids from get and receipt and receipts before and isinstance; why: new receipts after ids before ids later reads before ids, so failed_switch_canary must retain the computed value under that name. + before_ids = { + # What: call receipt.get with receipt id; why: failed_switch_canary invokes receipt.get while performing if isinstance receipt dict and isinstance; the call advances that operation through its result or side effect. + receipt.get("receiptId") for receipt in receipts_before + # What: call isinstance with receipt and dict; why: failed_switch_canary consumes the isinstance return value while evaluating if isinstance(receipt, dict) and isinstance(receipt.get("receiptId"), st. + if isinstance(receipt, dict) and isinstance(receipt.get("receiptId"), str) + # What: complete the before_ids expression with before ids receipt get receipt id for receipt in receipts before if isinstance; why: failed_switch_canary groups the supplied clauses as one before_ids expression before its value is consumed. + } + # What: compute prior failures from get and before and activation failures; why: or not isinstance prior failures int later reads prior failures, so failed_switch_canary must retain the computed value under that name. + prior_failures = before.get("activationFailures") + # What: gate on restored model and get and isinstance and prior failures and int before runtime error; why: failed_switch_canary admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call before.get with active profile; why: failed_switch_canary invokes before.get while performing or before get active identity matches engine is not; the call advances that operation through its result or side effect. + before.get("activeProfile") != restored_model + # What: call before.get with active identity matches engine; why: failed_switch_canary invokes before.get while performing or not isinstance prior failures int; the call advances that operation through its result or side effect. + or before.get("activeIdentityMatchesEngine") is not True + # What: call isinstance with prior failures and int; why: failed_switch_canary consumes the isinstance return value while evaluating or not isinstance(prior_failures, int). + or not isinstance(prior_failures, int) + # What: complete the enclosing predicate with if before get active profile differs from restored model or before get active identity matches engine; why: failed_switch_canary groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: failed_switch_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("failed-switch qualification requires an exact healthy resident") + # What: compute failure raw from the named fixture input; why: failure raw exc read later reads failure raw, so failed_switch_canary must retain the computed value under that name. + failure_raw = b"" + # What: establish the handler boundary for the protected operation; why: failed_switch_canary routes failures to httperror and error and urllib while preserving cleanup and success flow. + try: + # What: map the name field as model; why: failed_switch_canary carries name into request_json(base + "/router/load", {"name": model}, timeout=90). + request_json(base + "/router/load", {"name": model}, timeout=90) + # What: handle httperror and error and urllib by failure raw exc read 1024 1024 1; why: failed_switch_canary converts that failure into this concrete recovery, response, or cleanup behavior. + except urllib.error.HTTPError as exc: + # What: compute failure raw from read and exc and 1 and 1024 and 1024; why: if exc code or len failure raw later reads failure raw, so failed_switch_canary must retain the computed value under that name. + failure_raw = exc.read(1024 * 1024 + 1) + # What: gate on code and exc and len and failure raw before exc and runtime error; why: failed_switch_canary admits exc and runtime error only for this predicate and excludes the opposite state. + if exc.code != 503 or len(failure_raw) > 1024 * 1024: + # What: raise RuntimeError for the caller; why: failed_switch_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("failed replacement returned an invalid bounded response") from exc + # What: select the remaining branch that performs raise runtime error disposable invalid model unexpectedly; why: failed_switch_canary covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: raise RuntimeError for the caller; why: failed_switch_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("disposable invalid model unexpectedly activated") + # What: establish the handler boundary for the protected operation; why: failed_switch_canary routes failures to unicode decode error and jsondecode error and json while preserving cleanup and success flow. + try: + # What: compute failure from loads and failure raw and json; why: error failure get error if isinstance failure later reads failure, so failed_switch_canary must retain the computed value under that name. + failure = json.loads(failure_raw) + # What: handle unicode decode error and jsondecode error and json by raise runtime error failed replacement response was not; why: failed_switch_canary converts that failure into this concrete recovery, response, or cleanup behavior. + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + # What: raise RuntimeError for the caller; why: failed_switch_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("failed replacement response was not JSON") from exc + # What: compute error from isinstance and failure and dict and get and error; why: if not isinstance error dict or later reads error, so failed_switch_canary must retain the computed value under that name. + error = failure.get("error") if isinstance(failure, dict) else None + # What: compute recovery from isinstance and failure and dict and get and recovery; why: if not isinstance recovery dict or later reads recovery, so failed_switch_canary must retain the computed value under that name. + recovery = failure.get("recovery") if isinstance(failure, dict) else None + # What: gate on isinstance and error and dict and get before runtime error; why: failed_switch_canary admits runtime error only for this predicate and excludes the opposite state. + if not isinstance(error, dict) or error.get("type") not in {"engine_not_ready", "switch_launch_failed"}: + # What: raise RuntimeError for the caller; why: failed_switch_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("failed replacement did not report a lifecycle failure") + # What: gate on isinstance and recovery and dict and get before runtime error; why: failed_switch_canary admits runtime error only for this predicate and excludes the opposite state. + if not isinstance(recovery, dict) or recovery.get("launched") is not True: + # What: raise RuntimeError for the caller; why: failed_switch_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("failed replacement did not report successful rollback launch") + # What: compute and after from request json and base and router and status and 30; why: value pending after request json base accounting pending later reads and after, so failed_switch_canary must retain the computed value under that name. + _, after = request_json(base + "/router/status", timeout=30) + # What: gate on restored model and get and prior failures and after before runtime error; why: failed_switch_canary admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call after.get with active profile; why: failed_switch_canary invokes after.get while performing or after get active identity matches engine is not; the call advances that operation through its result or side effect. + after.get("activeProfile") != restored_model + # What: call after.get with active identity matches engine; why: failed_switch_canary invokes after.get while performing or after get active requests; the call advances that operation through its result or side effect. + or after.get("activeIdentityMatchesEngine") is not True + # What: call after.get with active requests; why: failed_switch_canary invokes after.get while performing or after get activation failures prior failures; the call advances that operation through its result or side effect. + or after.get("activeRequests") != 0 + # What: call after.get with activation failures; why: failed_switch_canary consumes the after.get return value while evaluating or after.get("activationFailures") != prior_failures + 1. + or after.get("activationFailures") != prior_failures + 1 + # What: complete the enclosing predicate with if after get active profile differs from restored model or after get active identity matches engine; why: failed_switch_canary groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: failed_switch_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("failed replacement did not restore exact idle residency") + # What: compute and pending after from request json and base and accounting and pending; why: the enclosing return or state update later reads and pending after, so failed_switch_canary must retain the computed value under that name. + _, pending_after = request_json(base + "/accounting/pending") + # What: compute receipts after from get and pending after and receipts; why: if not isinstance receipts after list later reads receipts after, so failed_switch_canary must retain the computed value under that name. + receipts_after = pending_after.get("receipts") + # What: gate on isinstance and receipts after and list before runtime error; why: failed_switch_canary admits runtime error only for this predicate and excludes the opposite state. + if not isinstance(receipts_after, list): + # What: raise RuntimeError for the caller; why: failed_switch_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("post-failure accounting outbox response has an invalid shape") # noqa: TRY004 -- malformed remote state is an operational failure. + # What: compute after ids from get and receipt and receipts after and isinstance; why: new receipts after ids before ids later reads after ids, so failed_switch_canary must retain the computed value under that name. + after_ids = { + # What: call receipt.get with receipt id; why: failed_switch_canary invokes receipt.get while performing if isinstance receipt dict and isinstance; the call advances that operation through its result or side effect. + receipt.get("receiptId") for receipt in receipts_after + # What: call isinstance with receipt and dict; why: failed_switch_canary consumes the isinstance return value while evaluating if isinstance(receipt, dict) and isinstance(receipt.get("receiptId"), st. + if isinstance(receipt, dict) and isinstance(receipt.get("receiptId"), str) + # What: complete the after_ids expression with after ids receipt get receipt id for receipt in receipts after if isinstance; why: failed_switch_canary groups the supplied clauses as one after_ids expression before its value is consumed. + } + # What: compute new receipts from after ids and before ids; why: if not new receipts later reads new receipts, so failed_switch_canary must retain the computed value under that name. + new_receipts = after_ids - before_ids + # What: gate on new receipts before runtime error; why: failed_switch_canary admits runtime error only for this predicate and excludes the opposite state. + if not new_receipts: + # What: raise RuntimeError for the caller; why: failed_switch_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("failed switch produced no new durable accounting receipt") + # What: read the restored engine status after rollback launch; why: router identity can be restored before the replacement process is actually ready to serve traffic. + _, restored_engine = request_json(base + "/engine/status", timeout=30) + # What: extract the restored engine listener port; why: readiness must be proven against the concrete replacement process rather than inferred from router metadata. + restored_port = restored_engine.get("port") + # What: reject a missing or invalid restored listener; why: waiting without an exact local port could probe the wrong process and create false recovery evidence. + if restored_engine.get("running") is not True or not isinstance(restored_port, int): + # What: stop qualification when rollback has no concrete running listener; why: a metadata-only rollback is not a usable restored service. + raise RuntimeError("failed replacement rollback has no running engine listener") + # What: wait for the restored process readiness endpoint; why: a launched process may still be loading model weights when router state already names it active. + wait_json(f"http://127.0.0.1:{restored_port}/ready", seconds=600) + # What: compute restored raw and restored from canary and base and restored model and false; why: return failure raw restored raw later reads restored raw and restored, so failed_switch_canary must retain the computed value under that name. + restored_raw, restored = canary(base, restored_model, direct=False) + # What: return failure raw and restored raw and model and restored model from failed_switch_canary; why: failed_switch_canary exposes failure raw and restored raw and model and restored model so its caller can continue with the function\'s computed outcome. + return failure_raw, restored_raw, { + # What: map the failed profile field as model; why: failed_switch_canary carries failed profile into "failedProfile": model. + "failedProfile": model, + # What: map the restored profile field as restored model; why: failed_switch_canary carries restored profile into "restoredProfile": restored_model. + "restoredProfile": restored_model, + # What: map the failure type field as error and type; why: failed_switch_canary carries failure type into "failureType": error["type"]. + "failureType": error["type"], + # What: map the rollback launched field as true; why: failed_switch_canary carries rollback launched into "rollbackLaunched": True. + "rollbackLaunched": True, + # What: map the activation failure incremented field as true; why: failed_switch_canary carries activation failure incremented into "activationFailureIncremented": True. + "activationFailureIncremented": True, + # What: map the new accounting receipt count field as len and new receipts; why: failed_switch_canary carries new accounting receipt count into "newAccountingReceiptCount": len(new_receipts). + "newAccountingReceiptCount": len(new_receipts), + # What: map the restored completion passed field as get and restored and true and passed; why: failed_switch_canary carries restored completion passed into "restoredCompletionPassed": restored.get("passed") is True. + "restoredCompletionPassed": restored.get("passed") is True, + # What: map the passed field as get and restored and true and passed; why: failed_switch_canary carries passed into "passed": restored.get("passed") is True. + "passed": restored.get("passed") is True, + # What: complete the enclosing predicate collection with failure raw and restored raw and model and restored model and error and len; why: failed_switch_canary groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. + } + + +# What: define persistent_capacity_canary around base and catalog path and model a and model b and api key; why: its direct callers call persistent_capacity_canary for persistent capacity canary and rely on this exact input and result contract. +def persistent_capacity_canary( + # What: declare the base input for persistent_capacity_canary; why: persistent_capacity_canary consumes base during value loaded a request json base router load, so callers must bind it with the other signature inputs. + base: str, catalog_path: Path, model_a: str, model_b: str, *, api_key: str | None = None +# What: complete the enclosing predicate collection with bytes and dict; why: persistent_capacity_canary groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. +) -> tuple[bytes, dict]: + """Prove a singleton persistent group reserves the sole resident slot.""" + # What: document prove a singleton persistent group reserves in the persistent_capacity_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand persistent capacity canary behavior without executing it. + # What: gate on get and request json and base before runtime error; why: persistent_capacity_canary admits runtime error only for this predicate and excludes the opposite state. + if request_json(base + "/router/unload", {}, timeout=45)[1].get("unloaded") is not True: + # What: raise RuntimeError for the caller; why: persistent_capacity_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("could not unload before persistent capacity qualification") + # What: call catalog_path.write_text; why: qualify_native_router needs this line to preserve the surrounding expression or collection structure. + catalog_path.write_text( + # What: supply persistent a to native_catalog_text; why: persistent_capacity_canary binds this true value to native_catalog_text's persistent a input. + native_catalog_text(model_a, model_b, persistent_a=True, api_key=api_key), + # What: preserve the exact encoding utf 8 literal fragment; why: persistent_capacity_canary passes this fragment verbatim through encoding="utf-8", because changing it would alter a protocol payload, serialized fixture, or public message. + encoding="utf-8", + # What: complete the catalog_path.write_text call with encoding; why: persistent_capacity_canary groups the supplied clauses as one catalog_path.write_text call before its value is consumed. + ) + # What: gate on get and request json and base before runtime error; why: persistent_capacity_canary admits runtime error only for this predicate and excludes the opposite state. + if request_json(base + "/router/reload", {}, timeout=30)[1].get("reloaded") is not True: + # What: raise RuntimeError for the caller; why: persistent_capacity_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("persistent catalog reload was not acknowledged") + # What: map the name field as model a; why: persistent_capacity_canary carries name through and loaded a into value still a request json base router status. + _, loaded_a = request_json(base + "/router/load", {"name": "model-a"}, timeout=660) + # What: compute router a and pid a from get and loaded a and router and pid; why: or not isinstance router a dict or later reads router a and pid a, so persistent_capacity_canary must retain the computed value under that name. + router_a, pid_a = loaded_a.get("router"), loaded_a.get("pid") + # What: gate on pid a and get and isinstance and int and router a before runtime error; why: persistent_capacity_canary admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call loaded_a.get with profile; why: persistent_capacity_canary invokes loaded_a.get while performing or not isinstance router a dict or; the call advances that operation through its result or side effect. + loaded_a.get("profile") != "model-a" or not isinstance(pid_a, int) or pid_a <= 0 + # What: call isinstance with router a and dict; why: persistent_capacity_canary invokes isinstance while performing or router a get active identity matches engine is not; the call advances that operation through its result or side effect. + or not isinstance(router_a, dict) or router_a.get("persistent") is not True + # What: call router_a.get with active identity matches engine; why: persistent_capacity_canary consumes the router_a.get return value while evaluating or router_a.get("activeIdentityMatchesEngine") is not True. + or router_a.get("activeIdentityMatchesEngine") is not True + # What: complete the enclosing predicate with if loaded a get profile differs from model a or not isinstance; why: persistent_capacity_canary groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: persistent_capacity_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("model-a did not occupy the persistent resident slot") + # What: compute rejection raw from the named fixture input; why: rejection raw exc read later reads rejection raw, so persistent_capacity_canary must retain the computed value under that name. + rejection_raw = b"" + # What: establish the handler boundary for the protected operation; why: persistent_capacity_canary routes failures to httperror and error and urllib while preserving cleanup and success flow. + try: + # What: map the name field as model b; why: persistent_capacity_canary carries name through request_json(base + "/router/load", {"name": "model-b"}, timeout=30) into raise runtime error persistent capacity conflict returned the. + request_json(base + "/router/load", {"name": "model-b"}, timeout=30) + # What: handle httperror and error and urllib by rejection raw exc read 1024 1024 1; why: persistent_capacity_canary converts that failure into this concrete recovery, response, or cleanup behavior. + except urllib.error.HTTPError as exc: + # What: compute rejection raw from read and exc and 1 and 1024 and 1024; why: if exc code or len rejection raw later reads rejection raw, so persistent_capacity_canary must retain the computed value under that name. + rejection_raw = exc.read(1024 * 1024 + 1) + # What: gate on code and exc and len and rejection raw before exc and runtime error; why: persistent_capacity_canary admits exc and runtime error only for this predicate and excludes the opposite state. + if exc.code != 409 or len(rejection_raw) > 1024 * 1024: + # What: raise RuntimeError for the caller; why: persistent_capacity_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("persistent capacity conflict returned an invalid response") from exc + # What: select the remaining branch that performs raise runtime error persistent resident allowed a; why: persistent_capacity_canary covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: raise RuntimeError for the caller; why: persistent_capacity_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("persistent resident allowed a conflicting activation") + # What: establish the handler boundary for the protected operation; why: persistent_capacity_canary routes failures to unicode decode error and jsondecode error and json while preserving cleanup and success flow. + try: + # What: compute rejection from loads and rejection raw and json; why: if rejection get error get type capacity unavailable later reads rejection, so persistent_capacity_canary must retain the computed value under that name. + rejection = json.loads(rejection_raw) + # What: handle unicode decode error and jsondecode error and json by raise runtime error persistent capacity response was not; why: persistent_capacity_canary converts that failure into this concrete recovery, response, or cleanup behavior. + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + # What: raise RuntimeError for the caller; why: persistent_capacity_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("persistent capacity response was not JSON") from exc + # What: gate on get and rejection before runtime error; why: persistent_capacity_canary admits runtime error only for this predicate and excludes the opposite state. + if rejection.get("error", {}).get("type") != "capacity_unavailable": + # What: raise RuntimeError for the caller; why: persistent_capacity_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("persistent capacity conflict returned the wrong error type") + # What: compute and still a from request json and base and router and status; why: value loaded b request json base router load later reads and still a, so persistent_capacity_canary must retain the computed value under that name. + _, still_a = request_json(base + "/router/status") + # What: gate on pid a and get and still a and request json and base before runtime error; why: persistent_capacity_canary admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call still_a.get with active profile; why: persistent_capacity_canary invokes still_a.get while performing or still a get active identity matches engine is not; the call advances that operation through its result or side effect. + still_a.get("activeProfile") != "model-a" + # What: call still_a.get with active identity matches engine; why: persistent_capacity_canary invokes still_a.get while performing or still a get persistent is not; the call advances that operation through its result or side effect. + or still_a.get("activeIdentityMatchesEngine") is not True + # What: call still_a.get with persistent; why: persistent_capacity_canary invokes still_a.get while performing or request json base engine status get; the call advances that operation through its result or side effect. + or still_a.get("persistent") is not True + # What: call operation.get with pid; why: persistent_capacity_canary consumes the operation.get return value while evaluating or request_json(base + "/engine/status")[1].get("pid") != pid_a. + or request_json(base + "/engine/status")[1].get("pid") != pid_a + # What: complete the enclosing predicate with if still a get active profile differs from model a or still a get active identity matches engine; why: persistent_capacity_canary groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: persistent_capacity_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("persistent capacity rejection disturbed the resident engine") + # What: map the name field as model a; why: persistent_capacity_canary carries name into if request_json(base + "/router/unload", {"name": "model-a"}, timeout=45. + if request_json(base + "/router/unload", {"name": "model-a"}, timeout=45)[1].get("unloaded") is not True: + # What: raise RuntimeError for the caller; why: persistent_capacity_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("explicit persistent unload failed") + # What: map the name field as model b; why: persistent_capacity_canary carries name through and loaded b into the enclosing return or state update. + _, loaded_b = request_json(base + "/router/load", {"name": "model-b"}, timeout=660) + # What: gate on get and loaded b before runtime error; why: persistent_capacity_canary admits runtime error only for this predicate and excludes the opposite state. + if loaded_b.get("profile") != "model-b" or loaded_b.get("router", {}).get("activeIdentityMatchesEngine") is not True: + # What: raise RuntimeError for the caller; why: persistent_capacity_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("released persistent capacity did not admit model-b") + # What: return rejection raw; why: the caller consumes this value as the function’s success-path result. + return rejection_raw, { + # What: map the persistent profile field as model a; why: persistent_capacity_canary carries persistent profile into "persistentProfile": "model-a", "conflictingProfile": "model-b". + "persistentProfile": "model-a", "conflictingProfile": "model-b", + # What: map the rejected status field as 409; why: persistent_capacity_canary carries rejected status into "rejectedStatus": 409, "residentPidPreserved": True. + "rejectedStatus": 409, "residentPidPreserved": True, + # What: map the explicit unload released capacity field as true; why: persistent_capacity_canary carries explicit unload released capacity into "explicitUnloadReleasedCapacity": True, "passed": True. + "explicitUnloadReleasedCapacity": True, "passed": True, + # What: execute the grouped source fragment; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + } + + +# What: define ttl_eviction_canary around base and catalog path and model a and model b and seconds and api key; why: its direct callers call ttl_eviction_canary for ttl eviction canary and rely on this exact input and result contract. +def ttl_eviction_canary( + # What: declare the base input for ttl_eviction_canary; why: ttl_eviction_canary consumes base during value before request json base router status, so callers must bind it with the other signature inputs. + base: str, catalog_path: Path, model_a: str, model_b: str, *, seconds: float = 180, + # What: declare the api key input for ttl_eviction_canary; why: ttl_eviction_canary consumes api key during native catalog text model a model b ttl s api key api key, so callers must bind it with the other signature inputs. + api_key: str | None = None, +# What: complete the enclosing predicate with dict; why: ttl_eviction_canary groups the supplied clauses as one enclosing predicate expression before its value is consumed. +) -> dict: + """Exercise idle-TTL ownership cleanup against the temporary catalog only.""" + # What: document exercise idle ttl ownership cleanup against the in the ttl_eviction_canary docstring; why: introspection and maintainers read this exact docstring fragment to understand ttl eviction canary behavior without executing it. + # What: compute and before from request json and base and router and status; why: value unloaded request json base router unload later reads and before, so ttl_eviction_canary must retain the computed value under that name. + _, before = request_json(base + "/router/status") + # What: compute prior evictions from get and before and evictions; why: if not isinstance prior evictions int later reads prior evictions, so ttl_eviction_canary must retain the computed value under that name. + prior_evictions = before.get("evictions") + # What: gate on isinstance and prior evictions and int before runtime error; why: ttl_eviction_canary admits runtime error only for this predicate and excludes the opposite state. + if not isinstance(prior_evictions, int): + # What: raise RuntimeError for the caller; why: ttl_eviction_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("router status lacks eviction counter") # noqa: TRY004 -- malformed remote status is an operational failure. + # What: compute and unloaded from request json and base and router and unload and 45; why: value reloaded request json base router reload later reads and unloaded, so ttl_eviction_canary must retain the computed value under that name. + _, unloaded = request_json(base + "/router/unload", {}, timeout=45) + # What: gate on get and unloaded before runtime error; why: ttl_eviction_canary admits runtime error only for this predicate and excludes the opposite state. + if unloaded.get("unloaded") is not True: + # What: raise RuntimeError for the caller; why: ttl_eviction_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("could not unload the prior resident before TTL qualification") + # What: extract the router state returned by explicit unload; why: manual unload is itself an eviction and advances the counter before the TTL event under test. + unloaded_router = unloaded.get("router") + # What: validate and retain the post-unload eviction baseline; why: the TTL assertion must measure one additional automatic eviction rather than compare against stale pre-unload accounting. + if not isinstance(unloaded_router, dict) or unloaded_router.get("evictions") != prior_evictions + 1: + # What: reject inconsistent explicit-unload accounting; why: a missing baseline would let the later TTL counter produce ambiguous evidence. + raise RuntimeError("explicit unload did not advance eviction accounting exactly once") + # What: retain the exact post-unload counter; why: the automatic TTL event must increment this current baseline by one. + unloaded_evictions = unloaded_router["evictions"] + # What: call catalog_path.write_text with native catalog text and model a and model b and api key and 2; why: ttl_eviction_canary invokes catalog_path.write_text while performing native catalog text model a model b ttl s api key api key; the call advances that operation through its result or side effect. + catalog_path.write_text( + # What: preserve the exact native catalog text model a model b ttl s api key api key literal fragment; why: ttl_eviction_canary passes this fragment verbatim through native_catalog_text(model_a, model_b, ttl_s=2, api_key=api_key), encodin, because changing it would alter a protocol payload, serialized fixture. + native_catalog_text(model_a, model_b, ttl_s=2, api_key=api_key), encoding="utf-8" + # What: complete the catalog_path.write_text call with encoding; why: ttl_eviction_canary groups the supplied clauses as one catalog_path.write_text call before its value is consumed. + ) + # What: compute and reloaded from request json and base and router and reload and 30; why: value loaded request json base router load later reads and reloaded, so ttl_eviction_canary must retain the computed value under that name. + _, reloaded = request_json(base + "/router/reload", {}, timeout=30) + # What: gate on get and reloaded before runtime error; why: ttl_eviction_canary admits runtime error only for this predicate and excludes the opposite state. + if reloaded.get("reloaded") is not True: + # What: raise RuntimeError for the caller; why: ttl_eviction_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("temporary TTL catalog reload was not acknowledged") + # What: map the name field as model a; why: ttl_eviction_canary carries name through and loaded into the enclosing return or state update. + _, loaded = request_json(base + "/router/load", {"name": "model-a"}, timeout=660) + # What: compute port from get and loaded and port; why: if loaded get profile model a or not later reads port, so ttl_eviction_canary must retain the computed value under that name. + port = loaded.get("port") + # What: gate on get and isinstance and port and int and loaded before runtime error; why: ttl_eviction_canary admits runtime error only for this predicate and excludes the opposite state. + if loaded.get("profile") != "model-a" or not isinstance(port, int) or not 1 <= port <= 65535: + # What: raise RuntimeError for the caller; why: ttl_eviction_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("TTL qualification did not activate a concrete model-a engine") + # What: complete one routed request against the TTL-enabled resident; why: idle eviction is scheduled when request ownership is released, not merely when an operator preloads a model. + _, ttl_completion = canary(base, "model-a", direct=False) + # What: reject a failed TTL trigger completion; why: eviction evidence is only meaningful after the resident has served and released real inference work. + if ttl_completion.get("passed") is not True: + # What: stop the TTL gate when the trigger request fails; why: waiting for eviction without proven prior use would test the wrong lifecycle contract. + raise RuntimeError("TTL qualification trigger completion failed") + # What: compute deadline from seconds and monotonic and time; why: while time monotonic deadline later reads deadline, so ttl_eviction_canary must retain the computed value under that name. + deadline = time.monotonic() + seconds + # What: compute status from the named fixture input; why: status request json base router status timeout later reads status, so ttl_eviction_canary must retain the computed value under that name. + status: dict | None = None + # What: iterate across deadline and monotonic and time to perform status and request json and base; why: ttl_eviction_canary repeats the body only while or for the loop header admits an iteration. + while time.monotonic() < deadline: + # What: compute status from request json and base and 1 and router and status; why: if status get active profile is and status get later reads status, so ttl_eviction_canary must retain the computed value under that name. + status = request_json(base + "/router/status", timeout=3)[1] + # What: gate on get and prior evictions and status before the computed value; why: ttl_eviction_canary admits the computed value only for this predicate and excludes the opposite state. + if status.get("activeProfile") is None and status.get("evictions") == unloaded_evictions + 1: + # What: apply the break portion of the enclosing predicate; why: this clause remains in ttl_eviction_canary\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: call time.sleep with 0 1; why: ttl_eviction_canary invokes time.sleep while performing if status is or status get active profile; the call advances that operation through its result or side effect. + time.sleep(0.1) + # What: gate on status and get and prior evictions before timeout error; why: ttl_eviction_canary admits timeout error only for this predicate and excludes the opposite state. + if status is None or status.get("activeProfile") is not None or status.get("evictions") != unloaded_evictions + 1: + # What: raise TimeoutError for the caller; why: ttl_eviction_canary stops this rejected path before it can mutate state, dispatch work, or report success. + raise TimeoutError("idle TTL did not evict the temporary resident engine") + # What: call require_listener_closed with port; why: ttl_eviction_canary invokes require_listener_closed while performing return; the call advances that operation through its result or side effect. + require_listener_closed(port) + # What: return port and profile and ttl seconds and port and eviction incremented from ttl_eviction_canary; why: ttl_eviction_canary exposes port and profile and ttl seconds and port and eviction incremented so its caller can continue with the function\'s computed outcome. + return { + # What: map the profile field as model a; why: ttl_eviction_canary carries profile into "profile": "model-a". + "profile": "model-a", + # What: map the ttl seconds field as 2; why: ttl_eviction_canary carries ttl seconds into "ttlSeconds": 2. + "ttlSeconds": 2, + # What: map the port field as port; why: ttl_eviction_canary carries port into "port": port. + "port": port, + # What: map the eviction incremented field as true; why: ttl_eviction_canary carries eviction incremented into "evictionIncremented": True. + "evictionIncremented": True, + # What: map the listener closed field as true; why: ttl_eviction_canary carries listener closed into "listenerClosed": True. + "listenerClosed": True, + # What: map the passed field as true; why: ttl_eviction_canary carries passed into "passed": True. + "passed": True, + # What: complete the enclosing predicate mapping with profile and ttl seconds and port and eviction incremented and listener closed; why: ttl_eviction_canary groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + +# What: define native_catalog_text around model a and model b and ttl s and model a priority and invalid model and persistent a and api key and startup; why: its direct callers call native_catalog_text for native catalog text and rely on this exact input and result contract. +def native_catalog_text( + # What: declare the model a input for native_catalog_text; why: native_catalog_text consumes model a during for alias model in model a model a, so callers must bind it with the other signature inputs. + model_a: str, model_b: str, *, ttl_s: int = 0, model_a_priority: int = 0, + # What: declare the invalid model input for native_catalog_text; why: native_catalog_text consumes invalid model during if invalid model is not, so callers must bind it with the other signature inputs. + invalid_model: str | None = None, persistent_a: bool = False, + # What: declare the api key input for native_catalog_text; why: native_catalog_text consumes api key during if api key is not, so callers must bind it with the other signature inputs. + api_key: str | None = None, startup: bool = False, +# What: complete the enclosing predicate with str; why: native_catalog_text groups the supplied clauses as one enclosing predicate expression before its value is consumed. +) -> str: + """Return the allowlisted, dynamic-port catalog used by the private run.""" + # What: document return the allowlisted dynamic port catalog used in the native_catalog_text docstring; why: introspection and maintainers read this exact docstring fragment to understand native catalog text behavior without executing it. + # What: compute common args from host and 127 0 0 1 and served model name and model id and max seq len override; why: args json dumps common args replace model id alias later reads common args, so native_catalog_text must retain the computed value under that name. + common_args = [ + # What: apply the host served model name model id portion of common args; why: native_catalog_text uses this clause to evaluate common args as one grouped value. + "--host", "127.0.0.1", "--served-model-name", "${MODEL_ID}", + # What: bound sequence, cache, and prefill capacity for the 24 GiB gfx1150 host; why: both qualified checkpoints need deterministic headroom instead of an avoidable recurrent-state allocation failure. + "--max-seq-len-override", "1024", "--num-tokens", "1024", "--max-prefill-length", "256", + # What: use one captured request, the naive cache, and a measured memory ratio; why: the target AMD host must avoid unsupported hybrid-state over-allocation while preserving enough memory for both exact artifacts. + "--max-running-requests", "1", "--graph", "1", "--cache-type", "naive", "--memory-ratio", "0.90", + # What: apply the attention backend triton moe backend fused disable pynccl portion of common args; why: native_catalog_text uses this clause to evaluate common args as one grouped value. + "--attention-backend", "triton", "--moe-backend", "fused", "--disable-pynccl", + # What: complete the common_args collection with host and 127 0 0 1 and served model name and model id; why: native_catalog_text groups the supplied clauses as one common_args collection before its value is consumed. + ] + # What: compute catalog from router and upstream timeout s and include aliases in list and true and send loading state; why: catalog f api keys json dumps api key later reads catalog, so native_catalog_text must retain the computed value under that name. + catalog = [ + # What: apply the router upstream timeout s include aliases in list true portion of catalog; why: native_catalog_text uses this clause to evaluate catalog as one grouped value. + "[router]", "upstream_timeout_s = 660", "include_aliases_in_list = true", + # What: apply the send loading state true performance every s portion of catalog; why: native_catalog_text uses this clause to evaluate catalog as one grouped value. + "send_loading_state = true", "performance_every_s = 5", "", + # What: complete the catalog collection with router and upstream timeout s and include aliases in list and true and send loading state and true; why: native_catalog_text groups the supplied clauses as one catalog collection before its value is consumed. + ] + # What: gate on api key before catalog and dumps and api key and json; why: native_catalog_text admits catalog and dumps and api key and json only for this predicate and excludes the opposite state. + if api_key is not None: + # What: compute catalog entry from dumps and api key and json and api keys and value; why: catalog later reads catalog entry, so native_catalog_text must retain the computed value under that name. + catalog[2:2] = [f"api_keys = [{json.dumps(api_key)}]"] + # What: gate on startup before catalog; why: native_catalog_text admits catalog only for this predicate and excludes the opposite state. + if startup: + # What: compute catalog entry from preload model and compat and model a and startup routing profile and coding; why: catalog extend later reads catalog entry, so native_catalog_text must retain the computed value under that name. + catalog[-1:-1] = [ + # What: apply the preload model compat model a portion of catalog entry; why: native_catalog_text uses this clause to evaluate catalog entry as one grouped value. + 'preload_model = "compat/model-a"', + # What: apply the startup routing profile coding portion of catalog entry; why: native_catalog_text uses this clause to evaluate catalog entry as one grouped value. + 'startup_routing_profile = "coding"', + # What: complete the catalog entry collection with preload model and compat and model a and startup routing profile and coding; why: native_catalog_text groups the supplied clauses as one catalog entry collection before its value is consumed. + ] + # What: call catalog.extend with selectors and preferred model and strategy and warm and targets; why: native_catalog_text invokes catalog.extend while performing selectors preferred model strategy warm; the call advances that operation through its result or side effect. + catalog.extend(( + # What: preserve the exact selectors preferred model strategy warm literal fragment; why: native_catalog_text passes this fragment verbatim through "[selectors.preferred-model]", 'strategy = "warm"', because changing it would alter a protocol payload, serialized fixture, or public message. + "[selectors.preferred-model]", 'strategy = "warm"', + # What: preserve the exact targets model b model a name preferred local literal fragment; why: native_catalog_text passes this fragment verbatim through 'targets = ["model-b", "model-a"]', 'name = "Preferred local model"', because changing it would alter a protocol payload, serialized fixture, or public messag. + 'targets = ["model-b", "model-a"]', 'name = "Preferred local model"', + # What: preserve the exact description reuses a ready target before literal fragment; why: native_catalog_text passes this fragment verbatim through 'description = "Reuses a ready target before the ordered cold fallback"', because changing it would alter a protocol payload, serialized fixture, or public messag. + 'description = "Reuses a ready target before the ordered cold fallback"', "", + # What: preserve the exact profiles coding description qualification routing profile literal fragment; why: native_catalog_text passes this fragment verbatim through "[profiles.coding]", 'description = "Qualification routing profile"', because changing it would alter a protocol payload, serialized fixture, or. + "[profiles.coding]", 'description = "Qualification routing profile"', + # What: preserve the exact profiles coding pins profile model preferred model literal fragment; why: native_catalog_text passes this fragment verbatim through "[profiles.coding.pins]", 'profile-model = "preferred-model"', because changing it would alter a protocol payload, serialized fixture, or public message. + "[profiles.coding.pins]", 'profile-model = "preferred-model"', + # What: preserve the exact disabled model literal fragment; why: native_catalog_text passes this fragment verbatim through 'disabled-model = ""', "", because changing it would alter a protocol payload, serialized fixture, or public message. + 'disabled-model = ""', "", + # What: complete the catalog.extend call with ordered positional inputs; why: native_catalog_text groups the supplied clauses as one catalog.extend call before its value is consumed. + )) + # What: gate on persistent a before extend and catalog; why: native_catalog_text admits extend and catalog only for this predicate and excludes the opposite state. + if persistent_a: + # What: call catalog.extend with router and groups and resident and members and model a; why: native_catalog_text invokes catalog.extend while performing router groups resident members model a swap false; the call advances that operation through its result or side effect. + catalog.extend(( + # What: preserve the exact router groups resident members model a swap false literal fragment; why: native_catalog_text passes this fragment verbatim through "[router.groups.resident]", 'members = ["model-a"]', "swap = false", because changing it would alter a protocol payload, serialized fixture, or publi. + "[router.groups.resident]", 'members = ["model-a"]', "swap = false", + # What: preserve the exact exclusive true persistent true literal fragment; why: native_catalog_text passes this fragment verbatim through "exclusive = true", "persistent = true", "", because changing it would alter a protocol payload, serialized fixture, or public message. + "exclusive = true", "persistent = true", "", + # What: complete the catalog.extend call with ordered positional inputs; why: native_catalog_text groups the supplied clauses as one catalog.extend call before its value is consumed. + )) + # What: iterate across model a and model b to perform profile lines and alias and ttl s and dumps and model; why: native_catalog_text repeats the body only while or for the loop header admits an iteration. + for alias, model in (("model-a", model_a), ("model-b", model_b)): + # What: compute profile lines from alias and ttl s and dumps and model; why: profile lines append group resident later reads profile lines, so native_catalog_text must retain the computed value under that name. + profile_lines = [ + # What: call json.dumps with model; why: native_catalog_text invokes json.dumps while performing check endpoint ready proxy http port; the call advances that operation through its result or side effect. + f"[models.{alias}]", f"model = {json.dumps(model)}", "port = 0", "ready_timeout_s = 600", + # What: apply the check endpoint ready proxy http port portion of profile lines; why: native_catalog_text uses this clause to evaluate profile lines as one grouped value. + 'check_endpoint = "/ready"', 'proxy = "http://127.0.0.1:${PORT}"', + # What: call json.dumps with alias; why: native_catalog_text invokes json.dumps while performing upstream timeout s; the call advances that operation through its result or side effect. + f"use_model_name = {json.dumps(alias)}", + # What: apply the upstream timeout s portion of profile lines; why: native_catalog_text uses this clause to evaluate profile lines as one grouped value. + "upstream_timeout_s = 659", + # What: apply the f ttl s ttl s f priority model a priority portion of profile lines; why: native_catalog_text uses this clause to evaluate profile lines as one grouped value. + f"ttl_s = {ttl_s}", f"priority = {model_a_priority if alias == 'model-a' else 0}", + # What: complete the profile_lines collection with alias and models and value and dumps and model and json and model and port and ready timeout s; why: native_catalog_text groups the supplied clauses as one profile_lines collection before its value is consumed. + ] + # What: gate on persistent a and alias before append and profile lines; why: native_catalog_text admits append and profile lines only for this predicate and excludes the opposite state. + if persistent_a and alias == "model-a": + # What: preserve the exact profile lines append group resident literal fragment; why: native_catalog_text passes this fragment verbatim through profile_lines.append('group = "resident"'), because changing it would alter a protocol payload, serialized fixture, or public message. + profile_lines.append('group = "resident"') + # What: gate on alias before append and profile lines; why: native_catalog_text admits append and profile lines only for this predicate and excludes the opposite state. + if alias == "model-a": + # What: preserve the exact profile lines append name qualification model a literal fragment; why: native_catalog_text passes this fragment verbatim through profile_lines.append('name = "Qualification model A"'), because changing it would alter a protocol payload, serialized fixture, or public message. + profile_lines.append('name = "Qualification model A"') + # What: preserve the exact profile lines append aliases compat model a literal fragment; why: native_catalog_text passes this fragment verbatim through profile_lines.append('aliases = ["compat/model-a"]'), because changing it would alter a protocol payload, serialized fixture, or public message. + profile_lines.append('aliases = ["compat/model-a"]') + # What: call profile_lines.extend with replace and alias and dumps and common args; why: native_catalog_text invokes profile_lines.extend while performing args json dumps common args replace model id alias; the call advances that operation through its result or side effect. + profile_lines.extend(( + # What: preserve the exact args json dumps common args replace model id alias literal fragment; why: native_catalog_text passes this fragment verbatim through "args = " + json.dumps(common_args).replace("${MODEL_ID}", alias), "", because changing it would alter a protocol payload, serialized fixture, or pu. + "args = " + json.dumps(common_args).replace("${MODEL_ID}", alias), "" + # What: complete the profile_lines.extend call with replace; why: native_catalog_text groups the supplied clauses as one profile_lines.extend call before its value is consumed. + )) + # What: gate on alias before extend and profile lines; why: native_catalog_text admits extend and profile lines only for this predicate and excludes the opposite state. + if alias == "model-a": + # What: call profile_lines.extend with models and model a and metadata and tier and qualification; why: native_catalog_text invokes profile_lines.extend while performing models model a metadata tier qualification; the call advances that operation through its result or side effect. + profile_lines.extend(( + # What: preserve the exact models model a metadata tier qualification literal fragment; why: native_catalog_text passes this fragment verbatim through "[models.model-a.metadata]", 'tier = "qualification"', because changing it would alter a protocol payload, serialized fixture, or public message. + "[models.model-a.metadata]", 'tier = "qualification"', + # What: preserve the exact type operator literal fragment; why: native_catalog_text passes this fragment verbatim through 'type = "operator"', "", because changing it would alter a protocol payload, serialized fixture, or public message. + 'type = "operator"', "", + # What: complete the profile_lines.extend call with ordered positional inputs; why: native_catalog_text groups the supplied clauses as one profile_lines.extend call before its value is consumed. + )) + # What: call catalog.extend with profile lines; why: native_catalog_text invokes catalog.extend while performing if invalid model is not; the call advances that operation through its result or side effect. + catalog.extend(profile_lines) + # What: gate on invalid model before extend and catalog and replace and dumps and invalid model; why: native_catalog_text admits extend and catalog and replace and dumps and invalid model only for this predicate and excludes the opposite state. + if invalid_model is not None: + # What: call catalog.extend with replace and dumps and invalid model and json; why: native_catalog_text invokes catalog.extend while performing models model invalid f model json dumps invalid model port; the call advances that operation through its result or side effect. + catalog.extend(( + # What: preserve the exact models model invalid f model json dumps invalid model port literal fragment; why: native_catalog_text passes this fragment verbatim through "[models.model-invalid]", f"model = {json.dumps(invalid_model)}", "port, because changing it would alter a protocol payload, serialized fixt. + "[models.model-invalid]", f"model = {json.dumps(invalid_model)}", "port = 0", + # What: preserve the exact ready timeout s check endpoint ready literal fragment; why: native_catalog_text passes this fragment verbatim through "ready_timeout_s = 15", 'check_endpoint = "/ready"', because changing it would alter a protocol payload, serialized fixture, or public message. + "ready_timeout_s = 15", 'check_endpoint = "/ready"', + # What: preserve the exact proxy http port ttl s literal fragment; why: native_catalog_text passes this fragment verbatim through 'proxy = "http://127.0.0.1:${PORT}"', "ttl_s = 0", because changing it would alter a protocol payload, serialized fixture, or public message. + 'proxy = "http://127.0.0.1:${PORT}"', "ttl_s = 0", + # What: preserve the exact args json dumps common args replace model id model invalid literal fragment; why: native_catalog_text passes this fragment verbatim through "args = " + json.dumps(common_args).replace("${MODEL_ID}", "model-invali, because changing it would alter a protocol payload, serialized fix. + "args = " + json.dumps(common_args).replace("${MODEL_ID}", "model-invalid"), "", + # What: complete the catalog.extend call with replace; why: native_catalog_text groups the supplied clauses as one catalog.extend call before its value is consumed. + )) + # What: return join and catalog and value from native_catalog_text; why: native_catalog_text exposes join and catalog and value so its caller can continue with the function\'s computed outcome. + return "\n".join(catalog) + + +# What: define main around the current object state; why: its direct callers call main for main and rely on this exact input and result contract. +def main() -> int: + # What: compute parser from argument parser and argparse and doc; why: parser add argument name required later reads parser, so main must retain the computed value under that name. + parser = argparse.ArgumentParser(description=__doc__) + # What: iterate across the computed value to perform add argument and parser and name; why: main repeats the body only while or for the loop header admits an iteration. + for name in ( + # What: apply the source python model a model b artifacts protected service portion of the enclosing predicate; why: this clause remains in main\'s enclosing expression so its grouping and evaluation order stay intact. + "source", "python", "model-a", "model-b", "artifacts", "protected-service", "protected-url", + # What: apply the expected hostname portion of the enclosing predicate; why: this clause remains in main\'s enclosing expression so its grouping and evaluation order stay intact. + "expected-hostname", + # What: complete the enclosing predicate collection with source and python and model a and model b; why: main groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. + ): + # What: register the parser add argument name required True command-line option; why: main validates this operator input before starting the qualification sequence. + parser.add_argument("--" + name, required=True) + # What: register the parser add argument allow maintenance action store true required True command-line option; why: main validates this operator input before starting the qualification sequence. + parser.add_argument("--allow-maintenance", action="store_true", required=True) + # What: preserve the exact parser add argument daemon port type int default literal fragment; why: main passes this fragment verbatim through parser.add_argument("--daemon-port", type=int, default=1964), because changing it would alter a protocol payload, serialized fixture, or public message. + # What: register protected-service ownership scope; why: the harness must stop and restore the exact manager that owns the workload. + parser.add_argument("--protected-service-scope", choices=("system", "user"), default="system") + # What: register an optional watchdog maintenance marker path; why: a protected-service timer must not race the harness by restarting the workload during an owned GPU window. + parser.add_argument("--protected-maintenance-marker") + # What: preserve the temporary daemon port option; why: callers still need a collision-free loopback control-plane endpoint. + parser.add_argument("--daemon-port", type=int, default=1964) + # What: compute args from parse args and parser; why: require expected hostname args expected hostname later reads args, so main must retain the computed value under that name. + args = parser.parse_args() + # What: gate on startswith and platform and sys before system exit; why: main admits system exit only for this predicate and excludes the opposite state. + if not sys.platform.startswith("linux"): + # What: raise SystemExit for the caller; why: main stops this rejected path before it can mutate state, dispatch work, or report success. + raise SystemExit("native maintenance qualification requires Linux process-group semantics") + # What: call require_expected_hostname with expected hostname and args; why: main invokes require_expected_hostname while performing artifacts path args artifacts; the call advances that operation through its result or side effect. + require_expected_hostname(args.expected_hostname) + + # What: compute artifacts from path and artifacts and args; why: artifacts mkdir parents exist ok later reads artifacts, so main must retain the computed value under that name. + artifacts = Path(args.artifacts) + # What: supply parents to artifacts.mkdir; why: main binds this true value to artifacts.mkdir's parents input. + artifacts.mkdir(parents=True, exist_ok=False) + # What: map the trials field as the fixture input; why: main carries trials through result into artifacts result json write text json dumps result indent 2. + result: dict = {"trials": [], "restored": False} + + # What: define save around the current object state; why: its direct callers call save for save and rely on this exact input and result contract. + def save() -> None: + # What: preserve the exact artifacts result json write text json dumps result indent literal fragment; why: save passes this fragment verbatim through (artifacts / "result.json").write_text(json.dumps(result, indent=2), enc, because changing it would alter a protocol payload, serialized fixture, or public mess. + (artifacts / "result.json").write_text(json.dumps(result, indent=2), encoding="utf-8") + + # What: compute service from sudo and n and systemctl; why: subprocess run service is active quiet args protected service check later reads service, so main must retain the computed value under that name. + # What: select the exact protected-service manager; why: maintenance and restoration must use the unit's real ownership scope. + service = protected_service_command(args.protected_service_scope) + # What: execute subprocess run service is active quiet args protected service check True; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + subprocess.run(service + ["is-active", "--quiet", args.protected_service], check=True) + # What: compute baseline raw and baseline from request json and protected url and args and health and 10; why: artifacts protected baseline health json write bytes baseline raw later reads baseline raw and baseline, so main must retain the computed value under that name. + baseline_raw, baseline = request_json(args.protected_url + "/health", timeout=10) + # What: preserve the exact artifacts protected baseline health json write bytes baseline raw literal fragment; why: main passes this fragment verbatim through (artifacts / "protected-baseline-health.json").write_bytes(baseline_raw), because changing it would alter a protocol payload, serialized fixture, or public. + (artifacts / "protected-baseline-health.json").write_bytes(baseline_raw) + # What: gate on get and baseline before runtime error; why: main admits runtime error only for this predicate and excludes the opposite state. + if baseline.get("status") != "ok": + # What: raise RuntimeError for the caller; why: main stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("protected service health baseline failed; no maintenance performed") + # What: compute and listing from request json and protected url and args and v1 and models; why: protected raw value canary args protected url protected model direct later reads and listing, so main must retain the computed value under that name. + _, listing = request_json(args.protected_url + "/v1/models", timeout=10) + # What: compute protected model from listing and id and 0 and data; why: protected raw value canary args protected url protected model direct later reads protected model, so main must retain the computed value under that name. + protected_model = listing["data"][0]["id"] + # What: compute protected raw and from canary and protected url and protected model and args and true; why: artifacts protected baseline response sse write bytes protected raw later reads protected raw and, so main must retain the computed value under that name. + protected_raw, _ = canary(args.protected_url, protected_model, direct=True) + # What: preserve the exact artifacts protected baseline response sse write bytes protected raw literal fragment; why: main passes this fragment verbatim through (artifacts / "protected-baseline-response.sse").write_bytes(protected_ra, because changing it would alter a protocol payload, serialized fixture, or publi. + (artifacts / "protected-baseline-response.sse").write_bytes(protected_raw) + + # What: compute env from copy and environ and os; why: env pythonpath str path args source python later reads env, so main must retain the computed value under that name. + env = os.environ.copy() + # What: compute env entry from str and path and source and args and python; why: env torch extensions dir str artifacts torch extensions later reads env entry, so main must retain the computed value under that name. + env["PYTHONPATH"] = str(Path(args.source) / "python") + # What: compute env entry from str and artifacts and torch extensions; why: env max jobs later reads env entry, so main must retain the computed value under that name. + env["TORCH_EXTENSIONS_DIR"] = str(artifacts / "torch-extensions") + # What: compute env entry from 2; why: cwd args source env env stdout log later reads env entry, so main must retain the computed value under that name. + env["MAX_JOBS"] = "2" + # What: compute catalog path from artifacts and models and toml; why: catalog path write text later reads catalog path, so main must retain the computed value under that name. + catalog_path = artifacts / "models.toml" + # What: compute invalid model from str and artifacts and intentionally missing model and gguf; why: args model a args model b invalid model invalid model api key native api key later reads invalid model, so main must retain the computed value under that name. + invalid_model = str(artifacts / "intentionally-missing-model.gguf") + # What: compute native api key from token urlsafe and secrets and 32; why: args model a args model b invalid model invalid model api key native api key later reads native api key, so main must retain the computed value under that name. + native_api_key = secrets.token_urlsafe(32) + # What: call catalog_path.write_text with native catalog text and model a and model b and args; why: main invokes catalog_path.write_text while performing native catalog text; the call advances that operation through its result or side effect. + catalog_path.write_text( + # What: call native_catalog_text with model a and args and model b and args; why: main invokes native_catalog_text while performing args model a args model b invalid model invalid model api key native api key; the call advances that operation through its result or side effect. + native_catalog_text( + # What: supply invalid model to native_catalog_text; why: main binds this invalid model value to native_catalog_text's invalid model input. + args.model_a, args.model_b, invalid_model=invalid_model, api_key=native_api_key + # What: complete the native_catalog_text call with invalid model and api key; why: main groups the supplied clauses as one native_catalog_text call before its value is consumed. + ), + # What: preserve the exact encoding utf 8 literal fragment; why: main passes this fragment verbatim through encoding="utf-8", because changing it would alter a protocol payload, serialized fixture, or public message. + encoding="utf-8", + # What: complete the catalog_path.write_text call with encoding; why: main groups the supplied clauses as one catalog_path.write_text call before its value is consumed. + ) + # What: enter the operation.open managed context before subprocess run; why: main releases this resource or lock after subprocess run on both success and failure paths. + with (artifacts / "kernel-preflight.log").open("wb") as log: + # What: call subprocess.run with python and args and c and from and freetoken; why: main invokes subprocess.run while performing args python c from freetoken kernel gguf import module; the call advances that operation through its result or side effect. + subprocess.run( + # What: preserve the exact args python c from freetoken kernel gguf import module literal fragment; why: main passes this fragment verbatim through [args.python, "-c", "from freetoken.kernel.gguf import _module. + [args.python, "-c", "from freetoken.kernel.gguf import _module; _module(); print('NATIVE_KERNEL_READY')"], + # What: supply cwd to subprocess.run; why: main binds this source and args value to subprocess.run's cwd input. + cwd=args.source, env=env, stdout=log, stderr=subprocess.STDOUT, check=True, timeout=600, + # What: complete the subprocess.run call with cwd and env and stdout and stderr and check; why: main groups the supplied clauses as one subprocess.run call before its value is consumed. + ) + + # What: compute daemon from the named fixture input; why: args python m freetoken cli daemon host later reads daemon, so main must retain the computed value under that name. + daemon: subprocess.Popen[bytes] | None = None + # What: compute detached engine from the named fixture input; why: detached engine old pid old port later reads detached engine, so main must retain the computed value under that name. + detached_engine: tuple[int, int] | None = None + # What: compute maintenance from false; why: maintenance later reads maintenance, so main must retain the computed value under that name. + maintenance = False + # What: resolve the optional watchdog marker path; why: lifecycle cleanup must track the exact marker that this harness may own. + maintenance_marker = Path(args.protected_maintenance_marker) if args.protected_maintenance_marker else None + # What: initialize marker ownership as false; why: cleanup must never remove a marker that predated this qualification run. + maintenance_marker_owned = False + # What: compute final engine port from the named fixture input; why: final engine port direct row hardware engine port later reads final engine port, so main must retain the computed value under that name. + final_engine_port: int | None = None + # What: compute base from daemon port and args and http; why: configure native auth base native api key later reads base, so main must retain the computed value under that name. + base = f"http://127.0.0.1:{args.daemon_port}" + # What: call configure_native_auth with base and native api key; why: main invokes configure_native_auth while performing def launch daemon log stop serve on exit bool subprocess popen; the call advances that operation through its result or side effect. + configure_native_auth(base, native_api_key) + + # What: define launch_daemon around log and stop serve on exit; why: its direct callers call launch_daemon for launch daemon and rely on this exact input and result contract. + def launch_daemon(log, *, stop_serve_on_exit: bool) -> subprocess.Popen[bytes]: + # What: compute command from python and args and str and daemon port; why: command append stop serve on exit later reads command, so launch_daemon must retain the computed value under that name. + command = [ + # What: apply the args python m freetoken cli daemon host portion of command; why: launch_daemon uses this clause to evaluate command as one grouped value. + args.python, "-m", "freetoken.cli", "daemon", "--host", "127.0.0.1", + # What: call str with daemon port and args; why: launch_daemon invokes str while performing catalog str catalog path catalog watch interval no oom; the call advances that operation through its result or side effect. + "--port", str(args.daemon_port), "--state-dir", str(artifacts / "daemon-state"), + # What: call str with catalog path; why: launch_daemon consumes the str return value while evaluating "--catalog", str(catalog_path), "--catalog-watch-interval", "0", "--no-o. + "--catalog", str(catalog_path), "--catalog-watch-interval", "0", "--no-oom", + # What: complete the command collection with python and args and m and freetoken and cli and daemon; why: launch_daemon groups the supplied clauses as one command collection before its value is consumed. + ] + # What: gate on stop serve on exit before append and command; why: launch_daemon admits append and command only for this predicate and excludes the opposite state. + if stop_serve_on_exit: + # What: preserve the exact command append stop serve on exit literal fragment; why: launch_daemon passes this fragment verbatim through command.append("--stop-serve-on-exit"), because changing it would alter a protocol payload, serialized fixture, or public message. + command.append("--stop-serve-on-exit") + # What: return popen and command and subprocess and source from launch_daemon; why: launch_daemon exposes popen and command and subprocess and source so its caller can continue with the function\'s computed outcome. + return subprocess.Popen( + # What: supply cwd to subprocess.Popen; why: launch_daemon binds this source and args value to subprocess.Popen's cwd input. + command, cwd=args.source, env=env, stdout=log, stderr=subprocess.STDOUT, + # What: supply stdin to subprocess.Popen; why: launch_daemon binds this devnull and subprocess value to subprocess.Popen's stdin input. + stdin=subprocess.DEVNULL, start_new_session=True, + # What: complete the subprocess.Popen call with cwd and env and stdout and stderr and stdin; why: launch_daemon groups the supplied clauses as one subprocess.Popen call before its value is consumed. + ) + + # What: establish the handler boundary for the protected operation; why: main routes failures to base exception while preserving cleanup and success flow. + try: + # What: enter the operation.open managed context before daemon launch daemon log stop serve on exit; why: main releases this resource or lock after daemon launch daemon log stop serve on exit on both success and failure paths. + with (artifacts / "daemon.log").open("wb") as log: + # What: compute daemon from launch daemon and log and false; why: stop process group daemon later reads daemon, so main must retain the computed value under that name. + daemon = launch_daemon(log, stop_serve_on_exit=False) + # What: preserve the exact wait json base router status seconds literal fragment; why: main passes this fragment verbatim through wait_json(base + "/router/status", seconds=30), because changing it would alter a protocol payload, serialized fixture, or public message. + wait_json(base + "/router/status", seconds=30) + # What: compute maintenance from true; why: if maintenance later reads maintenance, so main must retain the computed value under that name. + maintenance = True + # What: gate marker creation on an explicitly configured path; why: hosts without a watchdog marker preserve their existing lifecycle behavior. + if maintenance_marker is not None: + # What: fail if the marker already exists; why: another operator or process may own maintenance and must not be overridden. + if maintenance_marker.exists(): + # What: raise a lifecycle ownership error before stopping the service; why: qualification must fail closed when exclusive maintenance cannot be proven. + raise RuntimeError("protected-service maintenance marker already exists") + # What: create the marker with private run context; why: the target AMD host's health watchdog must suppress automatic restarts for this exact maintenance window. + maintenance_marker.write_text("FreeToken native-router qualification owns this maintenance window.\n", encoding="utf-8") + # What: record marker ownership after successful creation; why: only an owned marker may be removed during restoration. + maintenance_marker_owned = True + # What: execute subprocess run service stop args protected service check True timeout 90; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + subprocess.run(service + ["stop", args.protected_service], check=True, timeout=90) + + # Direct is intentionally measured against the native engine port after a router-owned load. + # What: map the name field as model a; why: main carries name through loaded raw and loaded into artifacts direct a load json write bytes loaded raw. + loaded_raw, loaded = request_json(base + "/router/load", {"name": "model-a"}, timeout=660) + # What: gate on get and isinstance and int and loaded before runtime error; why: main admits runtime error only for this predicate and excludes the opposite state. + if loaded.get("profile") != "model-a" or not isinstance(loaded.get("port"), int): + # What: raise RuntimeError for the caller; why: main stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("native management load did not return a concrete model-a target") + # What: wait for the router-owned engine's authoritative readiness endpoint; why: a successful lifecycle admission may precede model initialization and must not be mistaken for an inference-ready resident. + wait_json(f"http://127.0.0.1:{loaded['port']}/ready", seconds=600) + # What: compute activation count from validate routed trial and loaded and router and model a and 0; why: activation count validate routed trial later reads activation count, so main must retain the computed value under that name. + activation_count = validate_routed_trial( + # What: supply alias to validate_routed_trial; why: main binds this model a value to validate_routed_trial's alias input. + loaded["router"], alias="model-a", prior_activations=0, expected_delta=1 + # What: complete the validate_routed_trial call with alias and prior activations and expected delta; why: main groups the supplied clauses as one validate_routed_trial call before its value is consumed. + ) + # What: compute result entry from control plane canary and base and artifacts; why: result upstream model rewrite upstream model rewrite canary base artifacts later reads result entry, so main must retain the computed value under that name. + result["controlPlane"] = control_plane_canary(base, artifacts) + # What: compute result entry from upstream model rewrite canary and base and artifacts; why: result selector selector canary base artifacts later reads result entry, so main must retain the computed value under that name. + result["upstreamModelRewrite"] = upstream_model_rewrite_canary(base, artifacts) + # What: compute result entry from selector canary and base and artifacts; why: result routing profile routing profile canary base artifacts later reads result entry, so main must retain the computed value under that name. + result["selector"] = selector_canary(base, artifacts) + # What: compute result entry from routing profile canary and base and artifacts; why: result trials append direct row later reads result entry, so main must retain the computed value under that name. + result["routingProfile"] = routing_profile_canary(base, artifacts) + # What: compute direct raw and direct row from canary and loaded and model a and http and true; why: artifacts direct a sse write bytes direct raw later reads direct raw and direct row, so main must retain the computed value under that name. + direct_raw, direct_row = canary(f"http://127.0.0.1:{loaded['port']}", "model-a", direct=True) + # What: preserve the exact artifacts direct a sse write bytes direct raw literal fragment; why: main passes this fragment verbatim through (artifacts / "direct-a.sse").write_bytes(direct_raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / "direct-a.sse").write_bytes(direct_raw) + # What: preserve the exact artifacts direct a load json write bytes loaded raw literal fragment; why: main passes this fragment verbatim through (artifacts / "direct-a.load.json").write_bytes(loaded_raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / "direct-a.load.json").write_bytes(loaded_raw) + # What: compute direct row entry from loaded and router; why: direct row expected activation delta later reads direct row entry, so main must retain the computed value under that name. + direct_row["router"] = loaded["router"] + # What: compute direct row entry from 1; why: direct row hardware capture hardware base artifacts direct a later reads direct row entry, so main must retain the computed value under that name. + direct_row["expectedActivationDelta"] = 1 + # What: compute direct row entry from capture hardware and base and artifacts and direct a; why: final engine port direct row hardware engine port later reads direct row entry, so main must retain the computed value under that name. + direct_row["hardware"] = capture_hardware(base, artifacts, "direct-a") + # What: compute final engine port from direct row and port and engine and hardware; why: final engine port row hardware engine port later reads final engine port, so main must retain the computed value under that name. + final_engine_port = direct_row["hardware"]["engine"]["port"] + # What: preserve the exact result trials append direct row literal fragment; why: main passes this fragment verbatim through result["trials"].append(direct_row), because changing it would alter a protocol payload, serialized fixture, or public message. + result["trials"].append(direct_row) + + # What: compute cancel raw and cancellation from cancellation canary and base and model a; why: artifacts cancel a partial sse write bytes cancel raw later reads cancel raw and cancellation, so main must retain the computed value under that name. + cancel_raw, cancellation = cancellation_canary(base, "model-a") + # What: preserve the exact artifacts cancel a partial sse write bytes cancel raw literal fragment; why: main passes this fragment verbatim through (artifacts / "cancel-a.partial.sse").write_bytes(cancel_raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / "cancel-a.partial.sse").write_bytes(cancel_raw) + # What: compute result entry from cancellation; why: result concurrency concurrency later reads result entry, so main must retain the computed value under that name. + result["cancellation"] = cancellation + + # What: compute concurrent rows and concurrency from concurrent canaries and base and model a; why: for index concurrent raw value in enumerate later reads concurrent rows and concurrency, so main must retain the computed value under that name. + concurrent_rows, concurrency = concurrent_canaries(base, "model-a") + # What: iterate across enumerate and concurrent rows to perform write bytes and concurrent raw and artifacts and index; why: main repeats the body only while or for the loop header admits an iteration. + for index, (concurrent_raw, _) in enumerate(concurrent_rows): + # What: preserve the exact artifacts f concurrent a index sse write bytes literal fragment; why: main passes this fragment verbatim through (artifacts / f"concurrent-a-{index}.sse").write_bytes(concurrent_raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / f"concurrent-a-{index}.sse").write_bytes(concurrent_raw) + # What: compute result entry from concurrency; why: result trials append row later reads result entry, so main must retain the computed value under that name. + result["concurrency"] = concurrency + # What: call save with the declared inputs; why: main invokes save while performing for label alias expected delta in; the call advances that operation through its result or side effect. + save() + + # What: iterate across the computed value to perform raw and row and canary and base and alias; why: main repeats the body only while or for the loop header admits an iteration. + for label, alias, expected_delta in ( + # What: apply the warm a model a portion of the enclosing predicate; why: this clause remains in main\'s enclosing expression so its grouping and evaluation order stay intact. + ("warm-a", "model-a", 0), + # What: apply the cold b model b portion of the enclosing predicate; why: this clause remains in main\'s enclosing expression so its grouping and evaluation order stay intact. + ("cold-b", "model-b", 1), + # What: apply the alternating a model a portion of the enclosing predicate; why: this clause remains in main\'s enclosing expression so its grouping and evaluation order stay intact. + ("alternating-a", "model-a", 1), + # What: complete the enclosing predicate collection with warm a and model a and 0 and cold b and model b and 1 and alternating a and model a and 1; why: main groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. + ): + # What: compute raw and row from canary and base and alias and false; why: raw expected expected delta later reads raw and row, so main must retain the computed value under that name. + raw, row = canary(base, alias, direct=False) + # What: compute row entry from label; why: row loading feedback validate loading feedback later reads row entry, so main must retain the computed value under that name. + row["scenario"] = label + # What: compute row entry from validate loading feedback and raw and expected delta and 1; why: row router request json base router status later reads row entry, so main must retain the computed value under that name. + row["loadingFeedback"] = validate_loading_feedback( + # What: supply expected to validate_loading_feedback; why: main binds this expected delta and 1 value to validate_loading_feedback's expected input. + raw, expected=expected_delta == 1 + # What: complete the validate_loading_feedback call with expected; why: main groups the supplied clauses as one validate_loading_feedback call before its value is consumed. + ) + # What: compute row entry from request json and base and 1 and router and status; why: row router alias alias prior activations activation count later reads row entry, so main must retain the computed value under that name. + row["router"] = request_json(base + "/router/status")[1] + # What: compute activation count from validate routed trial and row and alias and activation count; why: row router alias alias prior activations activation count later reads activation count, so main must retain the computed value under that name. + activation_count = validate_routed_trial( + # What: supply alias to validate_routed_trial; why: main binds this alias value to validate_routed_trial's alias input. + row["router"], alias=alias, prior_activations=activation_count, + # What: supply expected delta to validate_routed_trial; why: main binds this expected delta value to validate_routed_trial's expected delta input. + expected_delta=expected_delta, + # What: complete the validate_routed_trial call with alias and prior activations and expected delta; why: main groups the supplied clauses as one validate_routed_trial call before its value is consumed. + ) + # What: compute row entry from expected delta; why: row hardware capture hardware base artifacts label later reads row entry, so main must retain the computed value under that name. + row["expectedActivationDelta"] = expected_delta + # What: preserve the exact artifacts f label sse write bytes raw literal fragment; why: main passes this fragment verbatim through (artifacts / f"{label}.sse").write_bytes(raw), because changing it would alter a protocol payload, serialized fixture, or public message. + (artifacts / f"{label}.sse").write_bytes(raw) + # What: preserve the exact artifacts f label metrics write bytes request bytes literal fragment; why: main passes this fragment verbatim through (artifacts / f"{label}.metrics").write_bytes(request_bytes(base + "/metr, because changing it would alter a protocol payload, serialized fixture, or public me. + (artifacts / f"{label}.metrics").write_bytes(request_bytes(base + "/metrics")) + # What: compute row entry from capture hardware and base and artifacts and label; why: final engine port row hardware engine port later reads row entry, so main must retain the computed value under that name. + row["hardware"] = capture_hardware(base, artifacts, label) + # What: compute final engine port from row and port and engine and hardware; why: final engine port result ttl port later reads final engine port, so main must retain the computed value under that name. + final_engine_port = row["hardware"]["engine"]["port"] + # What: preserve the exact result trials append row literal fragment; why: main passes this fragment verbatim through result["trials"].append(row), because changing it would alter a protocol payload, serialized fixture, or public message. + result["trials"].append(row) + # What: call save with the declared inputs; why: main invokes save while performing failure raw restored raw failed switch failed switch canary; the call advances that operation through its result or side effect. + save() + # What: evaluate and capture failure raw restored raw failed switch failed switch canary; why: the enclosing qualifier uses the captured result in its next validation or artifact step. + failure_raw, restored_raw, failed_switch = failed_switch_canary( + # What: apply the base model invalid model a portion of failure raw and restored raw and failed switch; why: main uses this clause to evaluate failure raw and restored raw and failed switch as one grouped value. + base, "model-invalid", "model-a" + # What: complete the failed_switch_canary call with base; why: main groups the supplied clauses as one failed_switch_canary call before its value is consumed. + ) + # What: preserve the exact artifacts failed switch response json write bytes failure raw literal fragment; why: main passes this fragment verbatim through (artifacts / "failed-switch-response.json").write_bytes(failure_raw), because changing it would alter a protocol payload, serialized fixture, or public. + (artifacts / "failed-switch-response.json").write_bytes(failure_raw) + # What: preserve the exact artifacts failed switch restored a sse write bytes restored raw literal fragment; why: main passes this fragment verbatim through (artifacts / "failed-switch-restored-a.sse").write_bytes(restored_raw), because changing it would alter a protocol payload, serialized fixture, or pub. + (artifacts / "failed-switch-restored-a.sse").write_bytes(restored_raw) + # What: compute result entry from failed switch; why: result re adoption later reads result entry, so main must retain the computed value under that name. + result["failedSwitch"] = failed_switch + + # What: compute before restart raw and before restart from request json and base and engine and status; why: main consumes before restart raw and before restart during artifacts re adoption before engine json write bytes before restart raw, so before restart raw and before restart value receives the comput. + before_restart_raw, before_restart = request_json(base + "/engine/status") + # What: compute old pid and old port from get and before restart and pid and port; why: or not isinstance old pid int or later reads old pid and old port, so main must retain the computed value under that name. + old_pid, old_port = before_restart.get("pid"), before_restart.get("port") + # What: gate on old pid and get and isinstance and int and old port before runtime error; why: main admits runtime error only for this predicate and excludes the opposite state. + if ( + # What: call before_restart.get with running; why: main invokes before_restart.get while performing or not isinstance old pid int or; the call advances that operation through its result or side effect. + not before_restart.get("running") + # What: call isinstance with old pid and int; why: main invokes isinstance while performing or not isinstance old port int or; the call advances that operation through its result or side effect. + or not isinstance(old_pid, int) or old_pid <= 0 + # What: call isinstance with old port and int; why: main consumes the isinstance return value while evaluating or not isinstance(old_port, int) or not 1 <= old_port <= 65535. + or not isinstance(old_port, int) or not 1 <= old_port <= 65535 + # What: complete the enclosing predicate with if not before restart get running or not isinstance old pid int; why: main groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RuntimeError for the caller; why: main stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("pre-restart engine identity is invalid") + # What: preserve the exact artifacts re adoption before engine json write bytes before restart raw literal fragme; why: main passes this fragment verbatim through (artifacts / "re-adoption-before-engine.json").write_bytes(before_restar, because changing it would alter a protocol payload, serialized fixture. + (artifacts / "re-adoption-before-engine.json").write_bytes(before_restart_raw) + # What: compute detached engine from old pid and old port; why: detached engine later reads detached engine, so main must retain the computed value under that name. + detached_engine = (old_pid, old_port) + # What: call catalog_path.write_text with native catalog text and model a and model b and args; why: main invokes catalog_path.write_text while performing native catalog text; the call advances that operation through its result or side effect. + catalog_path.write_text( + # What: call native_catalog_text with model a and args and model b and args; why: main invokes native_catalog_text while performing args model a args model b invalid model invalid model; the call advances that operation through its result or side effect. + native_catalog_text( + # What: supply invalid model to native_catalog_text; why: main binds this invalid model value to native_catalog_text's invalid model input. + args.model_a, args.model_b, invalid_model=invalid_model, + # What: supply api key to native_catalog_text; why: main binds this native api key value to native_catalog_text's api key input. + api_key=native_api_key, startup=True, + # What: complete the native_catalog_text call with invalid model and api key and startup; why: main groups the supplied clauses as one native_catalog_text call before its value is consumed. + ), + # What: preserve the exact encoding utf 8 literal fragment; why: main passes this fragment verbatim through encoding="utf-8", because changing it would alter a protocol payload, serialized fixture, or public message. + encoding="utf-8", + # What: complete the catalog_path.write_text call with encoding; why: main groups the supplied clauses as one catalog_path.write_text call before its value is consumed. + ) + # What: call stop_process_group with daemon; why: main invokes stop_process_group while performing daemon; the call advances that operation through its result or side effect. + stop_process_group(daemon) + # What: compute daemon from the named fixture input; why: daemon launch daemon log stop serve on exit later reads daemon, so main must retain the computed value under that name. + daemon = None + # What: call require_listener_open with old port; why: main invokes require_listener_open while performing daemon launch daemon log stop serve on exit; the call advances that operation through its result or side effect. + require_listener_open(old_port) + # What: compute daemon from launch daemon and log and true; why: if daemon is not later reads daemon, so main must retain the computed value under that name. + daemon = launch_daemon(log, stop_serve_on_exit=True) + # What: compute adopted router from wait json and base and router and status and 30; why: identity validate re adoption before restart adopted engine adopted router later reads adopted router, so main must retain the computed value under that name. + adopted_router = wait_json(base + "/router/status", seconds=30) + # What: compute adopted raw and adopted engine from request json and base and engine and status; why: artifacts re adoption after engine json write bytes adopted raw later reads adopted raw and adopted engine, so main must retain the computed value under that name. + adopted_raw, adopted_engine = request_json(base + "/engine/status") + # What: preserve the exact artifacts re adoption after engine json write bytes adopted raw literal fragment; why: main passes this fragment verbatim through (artifacts / "re-adoption-after-engine.json").write_bytes(adopted_raw), because changing it would alter a protocol payload, serialized fixture, or pub. + (artifacts / "re-adoption-after-engine.json").write_bytes(adopted_raw) + # What: compute identity from validate re adoption and before restart and adopted engine and adopted router; why: identity later reads identity, so main must retain the computed value under that name. + identity = validate_re_adoption(before_restart, adopted_engine, adopted_router) + # What: gate on get and adopted router before runtime error; why: main admits runtime error only for this predicate and excludes the opposite state. + if adopted_router.get("activeRoutingProfile") != "coding": + # What: raise RuntimeError for the caller; why: main stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("startup routing profile was not activated") + # What: compute readopted raw and readopted completion from canary and base and model a and false; why: artifacts re adoption restored a sse write bytes readopted raw later reads readopted raw and readopted completion, so main must retain the computed value under that name. + readopted_raw, readopted_completion = canary(base, "model-a", direct=False) + # What: preserve the exact artifacts re adoption restored a sse write bytes readopted raw literal fragment; why: main passes this fragment verbatim through (artifacts / "re-adoption-restored-a.sse").write_bytes(readopted_raw), because changing it would alter a protocol payload, serialized fixture, or publi. + (artifacts / "re-adoption-restored-a.sse").write_bytes(readopted_raw) + # What: gate on get and request json and base before runtime error; why: main admits runtime error only for this predicate and excludes the opposite state. + if request_json(base + "/router/status")[1].get("activations") != 0: + # What: raise RuntimeError for the caller; why: main stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("routed request replaced the re-adopted engine") + # What: compute result entry from identity and get and readopted completion and startup preload reused resident and startup routing profile; why: result conflicting request conflict later reads result entry, so main must retain the computed value under that name. + result["reAdoption"] = { + # What: apply the identity portion of result entry; why: main uses this clause to evaluate result entry as one grouped value. + **identity, + # What: map the startup preload reused resident field as true; why: main carries startup preload reused resident through result entry into result conflicting request conflict. + "startupPreloadReusedResident": True, + # What: map the startup routing profile field as coding; why: main carries startup routing profile through result entry into result conflicting request conflict. + "startupRoutingProfile": "coding", + # What: map the completion passed field as get and readopted completion and true and passed; why: main carries completion passed through result entry into result conflicting request conflict. + "completionPassed": readopted_completion.get("passed") is True, + # What: map the passed field as get and readopted completion and true and passed; why: main carries passed through result entry into result conflicting request conflict. + "passed": readopted_completion.get("passed") is True, + # What: complete the result entry mapping with startup preload reused resident and startup routing profile and completion passed and passed; why: main groups the supplied clauses as one result entry mapping before its value is consumed. + } + # What: compute detached engine from the named fixture input; why: if detached engine is later reads detached engine, so main must retain the computed value under that name. + detached_engine = None + # What: evaluate and capture conflict a raw conflict b raw conflict restored raw conflict conflicting request canary; why: the enclosing qualifier uses the captured result in its next validation or artifact step. + conflict_a_raw, conflict_b_raw, conflict_restored_raw, conflict = conflicting_request_canary( + # What: apply the base model a model b portion of conflict a raw and conflict b raw and conflict restored raw and conflict; why: main uses this clause to evaluate conflict a raw and conflict b raw and conflict restored raw and conflict as one grouped value. + base, "model-a", "model-b" + # What: complete the conflicting_request_canary call with base; why: main groups the supplied clauses as one conflicting_request_canary call before its value is consumed. + ) + # What: preserve the exact artifacts conflict active a partial sse write bytes conflict a raw literal fragment; why: main passes this fragment verbatim through (artifacts / "conflict-active-a.partial.sse").write_bytes(conflict_a_raw, because changing it would alter a protocol payload, serialized fixture, o. + (artifacts / "conflict-active-a.partial.sse").write_bytes(conflict_a_raw) + # What: preserve the exact artifacts conflict waiting b sse write bytes conflict b raw literal fragment; why: main passes this fragment verbatim through (artifacts / "conflict-waiting-b.sse").write_bytes(conflict_b_raw), because changing it would alter a protocol payload, serialized fixture, or public mess. + (artifacts / "conflict-waiting-b.sse").write_bytes(conflict_b_raw) + # What: preserve the exact artifacts conflict restored a sse write bytes conflict restored raw literal fragment; why: main passes this fragment verbatim through (artifacts / "conflict-restored-a.sse").write_bytes(conflict_restored_ra, because changing it would alter a protocol payload, serialized fixture. + (artifacts / "conflict-restored-a.sse").write_bytes(conflict_restored_raw) + # What: compute result entry from conflict; why: result reload conflict reload conflict canary later reads result entry, so main must retain the computed value under that name. + result["conflictingRequest"] = conflict + # What: compute result entry from reload conflict canary and base and catalog path and model a; why: result persistent capacity persistent later reads result entry, so main must retain the computed value under that name. + result["reloadConflict"] = reload_conflict_canary( + # What: supply api key to reload_conflict_canary; why: main binds this native api key value to reload_conflict_canary's api key input. + base, catalog_path, args.model_a, args.model_b, api_key=native_api_key + # What: complete the reload_conflict_canary call with api key; why: main groups the supplied clauses as one reload_conflict_canary call before its value is consumed. + ) + # What: compute persistent raw and persistent from persistent capacity canary and base and catalog path and model; why: main consumes persistent raw and persistent during artifacts persistent capacity rejection json write bytes persistent raw, so persistent raw and persistent value receives the computed va. + persistent_raw, persistent = persistent_capacity_canary( + # What: supply api key to persistent_capacity_canary; why: main binds this native api key value to persistent_capacity_canary's api key input. + base, catalog_path, args.model_a, args.model_b, api_key=native_api_key + # What: complete the persistent_capacity_canary call with api key; why: main groups the supplied clauses as one persistent_capacity_canary call before its value is consumed. + ) + # What: preserve the exact artifacts persistent capacity rejection json write bytes persistent raw literal fragme; why: main passes this fragment verbatim through (artifacts / "persistent-capacity-rejection.json").write_bytes(persisten, because changing it would alter a protocol payload, serialized fixture. + (artifacts / "persistent-capacity-rejection.json").write_bytes(persistent_raw) + # What: compute result entry from persistent; why: result ttl ttl eviction canary later reads result entry, so main must retain the computed value under that name. + result["persistentCapacity"] = persistent + # What: compute result entry from ttl eviction canary and base and catalog path and model a; why: final engine port result ttl port later reads result entry, so main must retain the computed value under that name. + result["ttl"] = ttl_eviction_canary( + # What: supply api key to ttl_eviction_canary; why: main binds this native api key value to ttl_eviction_canary's api key input. + base, catalog_path, args.model_a, args.model_b, api_key=native_api_key + # What: complete the ttl_eviction_canary call with api key; why: main groups the supplied clauses as one ttl_eviction_canary call before its value is consumed. + ) + # What: compute final engine port from result and port and ttl; why: if final engine port is not later reads final engine port, so main must retain the computed value under that name. + final_engine_port = result["ttl"]["port"] + # What: call save with the declared inputs; why: main invokes save while performing result passed; the call advances that operation through its result or side effect. + save() + # What: compute result entry from all and len and get and result; why: len result trials later reads result entry, so main must retain the computed value under that name. + result["passed"] = ( + # What: call len with result and trials; why: main invokes len while performing and all x passed for x; the call advances that operation through its result or side effect. + len(result["trials"]) == 4 + # What: call all with x and result and passed and trials; why: main invokes all while performing and result get cancellation get passed is; the call advances that operation through its result or side effect. + and all(x["passed"] for x in result["trials"]) + # What: call operation.get with passed; why: main invokes operation.get while performing and result get concurrency get passed is; the call advances that operation through its result or side effect. + and result.get("cancellation", {}).get("passed") is True + # What: call operation.get with passed; why: main invokes operation.get while performing and result get ttl get passed is; the call advances that operation through its result or side effect. + and result.get("concurrency", {}).get("passed") is True + # What: call operation.get with passed; why: main invokes operation.get while performing and result get reload conflict get passed is; the call advances that operation through its result or side effect. + and result.get("ttl", {}).get("passed") is True + # What: call operation.get with passed; why: main invokes operation.get while performing and result get failed switch get passed is; the call advances that operation through its result or side effect. + and result.get("reloadConflict", {}).get("passed") is True + # What: call operation.get with passed; why: main invokes operation.get while performing and result get re adoption get passed is; the call advances that operation through its result or side effect. + and result.get("failedSwitch", {}).get("passed") is True + # What: call operation.get with passed; why: main invokes operation.get while performing and result get persistent capacity get passed is; the call advances that operation through its result or side effect. + and result.get("reAdoption", {}).get("passed") is True + # What: call operation.get with passed; why: main invokes operation.get while performing and result get conflicting request get passed is; the call advances that operation through its result or side effect. + and result.get("persistentCapacity", {}).get("passed") is True + # What: call operation.get with passed; why: main invokes operation.get while performing and result get control plane get passed is; the call advances that operation through its result or side effect. + and result.get("conflictingRequest", {}).get("passed") is True + # What: call operation.get with passed; why: main invokes operation.get while performing and result get selector get passed is; the call advances that operation through its result or side effect. + and result.get("controlPlane", {}).get("passed") is True + # What: call operation.get with passed; why: main invokes operation.get while performing and result get routing profile get passed is; the call advances that operation through its result or side effect. + and result.get("selector", {}).get("passed") is True + # What: call operation.get with passed; why: main consumes the operation.get return value while evaluating and result.get("routingProfile", {}).get("passed") is True. + and result.get("routingProfile", {}).get("passed") is True + # What: complete the result entry expression with result passed len result trials equals 4 and all; why: main groups the supplied clauses as one result entry expression before its value is consumed. + ) + # What: handle base exception by result error repr exc; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException as exc: # noqa: BLE001 -- cleanup must record interrupts as qualification failures. + # What: compute result entry from repr and exc; why: result cleanup error repr exc later reads result entry, so main must retain the computed value under that name. + result["error"] = repr(exc) + # What: run if daemon is not on every exit path; why: main performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: gate on daemon before request json and oserror and value error and httperror and base; why: main admits request json and oserror and value error and httperror and base only for this predicate and excludes the opposite state. + if daemon is not None: + # What: capture any daemon-owned engine before shutting down its control plane; why: early failures previously lost the only exact PID and leaked a memory-consuming model process. + cleanup_engine = running_engine_identity(base) + # What: establish the handler boundary for the protected operation; why: main routes failures to oserror and value error and httperror and error and urllib while preserving cleanup and success flow. + try: + # What: preserve the exact request json base shutdown timeout literal fragment; why: main passes this fragment verbatim through request_json(base + "/shutdown", {}, timeout=45), because changing it would alter a protocol payload, serialized fixture, or public message. + request_json(base + "/shutdown", {}, timeout=45) + # What: handle oserror and value error and httperror and error and urllib by pass; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except (OSError, ValueError, urllib.error.HTTPError): + # What: ignore the anticipated exception handled by this branch; why: launch_daemon continues its retry or cleanup path instead of re-raising that transient failure. + pass + # What: establish the handler boundary for the protected operation; why: main routes failures to oserror and timeout expired and subprocess and runtime error while preserving cleanup and success flow. + try: + # What: call stop_process_group with daemon; why: main invokes stop_process_group while performing if daemon poll is; the call advances that operation through its result or side effect. + stop_process_group(daemon) + # What: stop the captured test-owned engine after daemon termination; why: the first daemon intentionally supports re-adoption and therefore does not automatically stop serve on every early failure. + if cleanup_engine is not None: + # What: terminate and verify the exact captured process group and listener; why: subsequent qualification and protected-service restoration require all test-owned unified memory to be released. + stop_detached_engine(*cleanup_engine) + # What: gate on poll and daemon before runtime error; why: main admits runtime error only for this predicate and excludes the opposite state. + if daemon.poll() is None: + # What: raise RuntimeError for the caller; why: main stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("temporary daemon process did not exit") + # What: gate on final engine port before require listener closed and final engine port; why: main admits require listener closed and final engine port only for this predicate and excludes the opposite state. + if final_engine_port is not None: + # What: call require_listener_closed with final engine port; why: main invokes require_listener_closed while performing except oserror subprocess timeout expired as exc; the call advances that operation through its result or side effect. + require_listener_closed(final_engine_port) + # What: handle oserror and timeout expired and subprocess by result cleanup error repr exc; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except (OSError, subprocess.TimeoutExpired) as exc: + # What: compute result entry from repr and exc; why: result cleanup error repr exc later reads result entry, so main must retain the computed value under that name. + result["cleanupError"] = repr(exc) + # What: handle runtime error by if detached engine is; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except RuntimeError as exc: + # What: gate on detached engine before result and repr and exc; why: main admits result and repr and exc only for this predicate and excludes the opposite state. + if detached_engine is None: + # What: compute result entry from repr and exc; why: result cleanup error repr detached exc later reads result entry, so main must retain the computed value under that name. + result["cleanupError"] = repr(exc) + # What: select the remaining branch that performs try; why: main covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: establish the handler boundary for the protected operation; why: main routes failures to oserror and runtime error while preserving cleanup and success flow. + try: + # What: call stop_detached_engine with detached engine; why: main invokes stop_detached_engine while performing detached engine; the call advances that operation through its result or side effect. + stop_detached_engine(*detached_engine) + # What: compute detached engine from the named fixture input; why: elif detached engine is not later reads detached engine, so main must retain the computed value under that name. + detached_engine = None + # What: handle oserror and runtime error by result cleanup error repr detached exc; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except (OSError, RuntimeError) as detached_exc: + # What: compute result entry from repr and detached exc; why: result cleanup error repr exc later reads result entry, so main must retain the computed value under that name. + result["cleanupError"] = repr(detached_exc) + # What: gate on detached engine before stop detached engine and oserror and runtime error and detached engine and result; why: main admits stop detached engine and oserror and runtime error and detached engine and result only for this predicate and excludes the opposite state. + elif detached_engine is not None: + # What: establish the handler boundary for the protected operation; why: main routes failures to oserror and runtime error while preserving cleanup and success flow. + try: + # What: call stop_detached_engine with detached engine; why: main invokes stop_detached_engine while performing except oserror runtime error as exc; the call advances that operation through its result or side effect. + stop_detached_engine(*detached_engine) + # What: handle oserror and runtime error by result cleanup error repr exc; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except (OSError, RuntimeError) as exc: + # What: compute result entry from repr and exc; why: result restored later reads result entry, so main must retain the computed value under that name. + result["cleanupError"] = repr(exc) + # What: gate on maintenance before base exception and run and wait json and restored raw and value; why: main admits base exception and run and wait json and restored raw and value only for this predicate and excludes the opposite state. + if maintenance: + # What: establish the handler boundary for the protected operation; why: main routes failures to base exception while preserving cleanup and success flow. + try: + # What: execute subprocess run service start args protected service check True timeout 180; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + subprocess.run(service + ["start", args.protected_service], check=True, timeout=180) + # What: preserve the exact wait json args protected url health seconds literal fragment; why: main passes this fragment verbatim through wait_json(args.protected_url + "/health", seconds=300), because changing it would alter a protocol payload, serialized fixture, or public message. + wait_json(args.protected_url + "/health", seconds=300) + # What: compute restored raw and from canary and protected url and protected model and args and true; why: artifacts protected restored response sse write bytes restored raw later reads restored raw and, so main must retain the computed value under that name. + restored_raw, _ = canary(args.protected_url, protected_model, direct=True) + # What: preserve the exact artifacts protected restored response sse write bytes restored raw literal fragment; why: main passes this fragment verbatim through (artifacts / "protected-restored-response.sse").write_bytes(restored_raw, because changing it would alter a protocol payload, serialized fixtur. + (artifacts / "protected-restored-response.sse").write_bytes(restored_raw) + # What: remove the owned watchdog marker after authoritative health and inference; why: normal automatic protection should resume only after restoration is proven. + if maintenance_marker_owned and maintenance_marker is not None: + # What: unlink the exact owned marker; why: a completed maintenance window must not leave watchdog recovery disabled. + maintenance_marker.unlink(missing_ok=True) + # What: clear marker ownership after removal; why: later cleanup must not repeat or misreport the action. + maintenance_marker_owned = False + # What: compute result entry from true; why: result restore error repr exc later reads result entry, so main must retain the computed value under that name. + result["restored"] = True + # What: handle base exception by result restore error repr exc; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException as exc: # noqa: BLE001 -- cleanup must record interrupts as qualification failures. + # What: compute result entry from repr and exc; why: return if result get passed and result later reads result entry, so main must retain the computed value under that name. + result["restoreError"] = repr(exc) + # What: remove an owned marker after a failed explicit restoration attempt; why: the existing watchdog must regain permission to recover the protected service. + if maintenance_marker_owned and maintenance_marker is not None: + # What: unlink the exact owned marker without masking the restoration error; why: recovery enablement is safer than preserving a stale maintenance lock. + maintenance_marker.unlink(missing_ok=True) + # What: clear marker ownership after emergency release; why: saved state must reflect that watchdog suppression no longer remains. + maintenance_marker_owned = False + # What: call save with the declared inputs; why: main invokes save while performing return if result get passed and result; the call advances that operation through its result or side effect. + save() + # What: return get and result and 0 and 1 and passed from main; why: main exposes get and result and 0 and 1 and passed so its caller can continue with the function\'s computed outcome. + return 0 if result.get("passed") and result["restored"] and "cleanupError" not in result else 1 + + +# What: gate on name before system exit and main; why: qualify_native_router admits system exit and main only for this predicate and excludes the opposite state. +if __name__ == "__main__": + # What: raise SystemExit for the caller; why: qualify_native_router stops this rejected path before it can mutate state, dispatch work, or report success. + raise SystemExit(main()) diff --git a/docs/amd-rocm-gfx1151.md b/docs/amd-rocm-gfx1151.md new file mode 100644 index 0000000000..6ee84d17dc --- /dev/null +++ b/docs/amd-rocm-gfx1151.md @@ -0,0 +1,145 @@ +# FreeToken AMD ROCm on Radeon 8060S `gfx1151` + +## Purpose + +This branch ports the FreeToken serving runtime to native AMD ROCm and HIP on +the AMD Ryzen AI Max+ 395 with Radeon 8060S (`gfx1151`). The port preserves +the NVIDIA implementation as a separate runtime path. It does not use Vulkan +or a CPU-only runner as a substitute for native GPU execution. + +The evaluated deployment system serves the same local API +surface as upstream FreeToken, including OpenAI-compatible endpoints, while +using HIP-compiled extensions and AMD Triton kernels. + +## Scope and parity contract + +The port is complete only when the target model can load and serve through +`ft serve`, return a coherent streamed and non-streamed OpenAI-compatible +response, and exercise the applicable FreeToken cache and MoE paths. The +initial full-model validation set is: + +1. `Qwen/Qwen3.6-35B-A3B`, FreeToken's primary consumer-hardware MoE + benchmark model. +2. The current Gemma 4 MoE GGUF accepted by FreeToken's native Gemma loader. + +The project records correctness, stability, API behavior, GPU memory, host +memory, prefill throughput, decode throughput, TTFT, temperature, clocks, and +throttling. NVIDIA GPU tokens per second are context, not an AMD acceptance +threshold: the evaluated platform uses a shared-memory APU rather than discrete VRAM and +PCIe. + +## What this branch changes + +The code is deliberately gated at the narrowest possible boundary so CUDA +behavior stays unchanged. + +- `setup.py` detects a ROCm PyTorch build and links the two native extensions + to `libamdhip64` instead of `libcudart`. +- `kernel/csrc/hip_compat.h` maps the small CUDA Runtime API subset used by + FreeToken's pinned-memory and CPU MoE extensions to HIP equivalents. +- CUDA JIT compilation removes NVCC-only flags on HIP and replaces CUDA-only + launch behavior with compatible HIP launch behavior. +- Triton paths avoid NVIDIA PTX inline assembly, Hopper Programmatic Dependent + Launch controls, and CUDA tile assumptions when PyTorch reports HIP. +- CUDA-only optional package probes are suppressed on HIP. The pure Triton + implementations remain the portable GPU fast path. +- NVIDIA SM feature gates reject ROCm before numerical capability comparison. + This matters because PyTorch presents HIP devices under `torch.cuda` for + compatibility, and `gfx1151` must never be interpreted as a new NVIDIA SM. + +## Clean installation + +Do not install into system Python, an existing llama.cpp environment, or the +existing vLLM environment. The reference layout is intentionally isolated: + +```text +$PROJECT_ROOT/ + source/ this Git checkout + .venv/ Python 3.12, ROCm PyTorch, AMD Triton, FreeToken + artifacts/ commands, environment manifests, tests, logs, telemetry + models/ optional links to read-only local model storage +``` + +The exact PyTorch ROCm wheel must be selected after validating its compatible +Triton build on the target system. FreeToken's upstream CUDA package set must not be +installed on AMD: `flashinfer`, `sglang-kernel`, CUDA-indexed Torch wheels, and +the CUDA kernel-cache wheel are NVIDIA binaries. + +The initial build command is run from `source` only after the isolated Python +environment has a working HIP PyTorch import: + +```bash +python -m pip install -e . --no-build-isolation --no-deps +``` + +Use `hipcc --version`, `rocminfo`, and a small PyTorch HIP allocation before +the FreeToken build. Record outputs in `artifacts/environment/`, with secrets +and access tokens removed. + +## Persistent GGUF HIP JIT cache + +The native Gemma GGUF extension is compiled once per combination of FreeToken +source, PyTorch and HIP version, compiler flags, Python ABI, and GPU target. +`torch.utils.cpp_extension` reuses the resulting shared object on later +process starts. Normal serving must not delete that cache. + +The default cache is `$HOME/.cache/torch_extensions/`. For a deliberate, +portable installation-specific location, set this before every `ft serve` +launch and keep the directory across reboots and service restarts: + +```bash +export TORCH_EXTENSIONS_DIR="$PROJECT_ROOT/cache/torch_extensions" +mkdir -p "$TORCH_EXTENSIONS_DIR" +``` + +After an intentional FreeToken source or ROCm toolchain update, one rebuild is +expected. Deleting this directory is a recovery action only. It was cleared +during the original port investigation to force revised HIP sources to build; +that development step is not part of normal operation. + +## Required validation sequence + +1. Verify the host's `gfx1151` device, HIP runtime, PyTorch HIP build, and + AMD Triton version. +2. Build and import `_pinned_tensor` and `_cpu_moe` from the isolated + environment. +3. Run the ROCm gate unit tests plus the relevant CPU and Triton tests. +4. Run Qwen3.6-35B-A3B through `ft serve` on a non-conflicting local port. +5. Test `/v1/models`, non-streaming `/v1/chat/completions`, and streamed + `/v1/chat/completions` with fixed requests. +6. Run `ft bench bw` on the target system. Treat its recommendation as a measured + candidate, then verify it with full serving workloads. +7. Repeat the same API and stability checks for the supported Gemma 4 MoE + GGUF. +8. Save raw command output, service logs, request responses, profiler output, + and hardware telemetry under `artifacts/`. + +No llama-swap service, model configuration, or existing port is modified by +these commands. Service packaging happens only after the full validation set +passes. + +## Provenance + +This branch incorporates the focused current-main ROCm work from FreeToken +pull request #241, preserving its commits and authorship. It adds explicit +`gfx1151` safety coverage and project-specific validation documentation. +Upstream review should receive a focused pull request containing code plus +tests. Evaluated-system environment reports and benchmark artifacts belong in this +fork unless the upstream maintainers request them. + +The completed 2026-08-28 native HIP validation, exact evaluated-system environment, +API evidence, command shapes, and known limitations are documented in +[`host-identity canary-rocm-validation-2026-08-28.md`](host-identity canary-rocm-validation-2026-08-28.md). +The post-repair Gemma vision, Qwen API, matched runner, and clean-memory +endurance evidence is documented separately in +[`host-identity canary-rocm-validation-2026-08-30.md`](host-identity canary-rocm-validation-2026-08-30.md). + +## Public reproduction interface + +The original host-specific scripts intentionally preserve a local protected service and use +host-specific model locations. They are not the public entry point. Independent +users should begin with [`reproducibility.md`](reproducibility.md) and its +parameterized `scripts/reproduce/collect_host_manifest.sh` collector. The +collector requires a native HIP PyTorch device, writes a new non-sensitive +artifact directory, redacts the hostname by default, and never starts or stops +a model server or changes host state. diff --git a/docs/expert-route-group-0.json b/docs/expert-route-group-0.json new file mode 100644 index 0000000000..bf03dc64c1 --- /dev/null +++ b/docs/expert-route-group-0.json @@ -0,0 +1,777 @@ +{ + "scope": "isolated real-shape routed expert transfer only", + "checkpoint": "checkpoint", + "device": "cuda:0", + "layers": [ + 0, + 1 + ], + "experts": [ + 0, + 1, + 2, + 3, + 4, + 5 + ], + "repeats": 5, + "selected_tensor_count": 72, + "selected_bytes": 160432128, + "selected_mib": 153.0, + "tensors": [ + { + "name": "layers.0.ffn.experts.0.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.0.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.0.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.0.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.0.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.0.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.1.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.1.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.1.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.1.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.1.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.1.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.2.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.2.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.2.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.2.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.2.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.2.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.3.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.3.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.3.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.3.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.3.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.3.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.4.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.4.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.4.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.4.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.4.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.4.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.5.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.5.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.5.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.5.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.5.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.5.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.0.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.0.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.0.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.0.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.0.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.0.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.1.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.1.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.1.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.1.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.1.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.1.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.2.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.2.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.2.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.2.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.2.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.2.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.3.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.3.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.3.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.3.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.3.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.3.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.4.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.4.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.4.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.4.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.4.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.4.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.5.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.5.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.5.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.5.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.5.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.5.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + } + ], + "round_trips": [ + { + "h2d_seconds": 0.024474173900671303, + "d2h_seconds": 0.0023964029969647527, + "h2d_gib_per_second": 6.104968572438791, + "d2h_gib_per_second": 62.349305475433624 + }, + { + "h2d_seconds": 0.0019258600659668446, + "d2h_seconds": 0.0023145959712564945, + "h2d_gib_per_second": 77.58303167524753, + "d2h_gib_per_second": 64.55297786545854 + }, + { + "h2d_seconds": 0.001921451068483293, + "d2h_seconds": 0.002306715934537351, + "h2d_gib_per_second": 77.76105514773307, + "d2h_gib_per_second": 64.77349909579021 + }, + { + "h2d_seconds": 0.0019197110086679459, + "d2h_seconds": 0.002307686023414135, + "h2d_gib_per_second": 77.83153913550552, + "d2h_gib_per_second": 64.74627006621442 + }, + { + "h2d_seconds": 0.001919100061058998, + "d2h_seconds": 0.002303007058799267, + "h2d_gib_per_second": 77.85631689134037, + "d2h_gib_per_second": 64.87781352172709 + } + ], + "protected_service_touched": false, + "full_model_serving_claim": false +} diff --git a/docs/expert-route-group-16.json b/docs/expert-route-group-16.json new file mode 100644 index 0000000000..bd31e78eea --- /dev/null +++ b/docs/expert-route-group-16.json @@ -0,0 +1,777 @@ +{ + "scope": "isolated real-shape routed expert transfer only", + "checkpoint": "checkpoint", + "device": "cuda:0", + "layers": [ + 0, + 1 + ], + "experts": [ + 16, + 17, + 18, + 19, + 20, + 21 + ], + "repeats": 5, + "selected_tensor_count": 72, + "selected_bytes": 160432128, + "selected_mib": 153.0, + "tensors": [ + { + "name": "layers.0.ffn.experts.16.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.16.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.16.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.16.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.16.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.16.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.17.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.17.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.17.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.17.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.17.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.17.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.18.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.18.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.18.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.18.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.18.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.18.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.19.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.19.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.19.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.19.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.19.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.19.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.20.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.20.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.20.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.20.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.20.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.20.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.21.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.21.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.21.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.21.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.21.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.21.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.16.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.16.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.16.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.16.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.16.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.16.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.17.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.17.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.17.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.17.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.17.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.17.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.18.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.18.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.18.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.18.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.18.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.18.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.19.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.19.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.19.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.19.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.19.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.19.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.20.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.20.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.20.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.20.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.20.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.20.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.21.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.21.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.21.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.21.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.21.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.21.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + } + ], + "round_trips": [ + { + "h2d_seconds": 0.024694555904716253, + "d2h_seconds": 0.0023958830861374736, + "h2d_gib_per_second": 6.0504859077649735, + "d2h_gib_per_second": 62.36283538395778 + }, + { + "h2d_seconds": 0.001904760953038931, + "d2h_seconds": 0.002307957038283348, + "h2d_gib_per_second": 78.44242200661395, + "d2h_gib_per_second": 64.73866715089886 + }, + { + "h2d_seconds": 0.0019060210324823856, + "d2h_seconds": 0.0023082559928297997, + "h2d_gib_per_second": 78.39056335354516, + "d2h_gib_per_second": 64.73028250078374 + }, + { + "h2d_seconds": 0.0019077310571447015, + "d2h_seconds": 0.002315965946763754, + "h2d_gib_per_second": 78.32029674226085, + "d2h_gib_per_second": 64.51479250322559 + }, + { + "h2d_seconds": 0.0019013010896742344, + "d2h_seconds": 0.002302416949532926, + "h2d_gib_per_second": 78.58516639550254, + "d2h_gib_per_second": 64.89444169975837 + } + ], + "protected_service_touched": false, + "full_model_serving_claim": false +} diff --git a/docs/expert-route-group-32.json b/docs/expert-route-group-32.json new file mode 100644 index 0000000000..cc56089c0e --- /dev/null +++ b/docs/expert-route-group-32.json @@ -0,0 +1,777 @@ +{ + "scope": "isolated real-shape routed expert transfer only", + "checkpoint": "checkpoint", + "device": "cuda:0", + "layers": [ + 0, + 1 + ], + "experts": [ + 32, + 33, + 34, + 35, + 36, + 37 + ], + "repeats": 5, + "selected_tensor_count": 72, + "selected_bytes": 160432128, + "selected_mib": 153.0, + "tensors": [ + { + "name": "layers.0.ffn.experts.32.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.32.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.32.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.32.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.32.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.32.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.33.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.33.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.33.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.33.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.33.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.33.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.34.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.34.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.34.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.34.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.34.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.34.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.35.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.35.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.35.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.35.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.35.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.35.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.36.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.36.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.36.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.36.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.36.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.36.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.37.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.37.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.37.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.37.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.37.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.37.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.32.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.32.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.32.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.32.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.32.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.32.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.33.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.33.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.33.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.33.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.33.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.33.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.34.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.34.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.34.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.34.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.34.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.34.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.35.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.35.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.35.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.35.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.35.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.35.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.36.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.36.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.36.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.36.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.36.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.36.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.37.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.37.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.37.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.37.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.37.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.37.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + } + ], + "round_trips": [ + { + "h2d_seconds": 0.02506138291209936, + "d2h_seconds": 0.0024102929746732116, + "h2d_gib_per_second": 5.961924089506829, + "d2h_gib_per_second": 61.99000041489048 + }, + { + "h2d_seconds": 0.001906190998852253, + "d2h_seconds": 0.002302735927514732, + "h2d_gib_per_second": 78.3835736240306, + "d2h_gib_per_second": 64.88545243711803 + }, + { + "h2d_seconds": 0.0018984719645231962, + "d2h_seconds": 0.002297736005857587, + "h2d_gib_per_second": 78.70227493063115, + "d2h_gib_per_second": 65.02664454014769 + }, + { + "h2d_seconds": 0.001903461990877986, + "d2h_seconds": 0.002298266044817865, + "h2d_gib_per_second": 78.4959527513768, + "d2h_gib_per_second": 65.01164773194955 + }, + { + "h2d_seconds": 0.0018978809239342809, + "d2h_seconds": 0.0022996170446276665, + "h2d_gib_per_second": 78.72678449724165, + "d2h_gib_per_second": 64.97345410143792 + } + ], + "protected_service_touched": false, + "full_model_serving_claim": false +} diff --git a/docs/expert-route-group-64.json b/docs/expert-route-group-64.json new file mode 100644 index 0000000000..c0c31ea1a5 --- /dev/null +++ b/docs/expert-route-group-64.json @@ -0,0 +1,777 @@ +{ + "scope": "isolated real-shape routed expert transfer only", + "checkpoint": "checkpoint", + "device": "cuda:0", + "layers": [ + 0, + 1 + ], + "experts": [ + 64, + 65, + 66, + 67, + 68, + 69 + ], + "repeats": 5, + "selected_tensor_count": 72, + "selected_bytes": 160432128, + "selected_mib": 153.0, + "tensors": [ + { + "name": "layers.0.ffn.experts.64.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.64.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.64.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.64.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.64.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.64.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.65.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.65.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.65.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.65.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.65.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.65.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.66.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.66.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.66.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.66.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.66.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.66.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.67.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.67.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.67.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.67.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.67.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.67.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.68.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.68.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.68.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.68.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.68.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.68.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.69.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.69.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.69.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.69.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.69.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.69.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.64.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.64.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.64.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.64.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.64.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.64.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.65.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.65.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.65.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.65.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.65.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.65.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.66.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.66.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.66.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.66.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.66.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.66.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.67.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.67.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.67.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.67.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.67.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.67.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.68.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.68.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.68.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.68.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.68.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.68.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.69.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.69.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.69.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.69.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.69.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.69.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + } + ], + "round_trips": [ + { + "h2d_seconds": 0.024525672080926597, + "d2h_seconds": 0.00239807297475636, + "h2d_gib_per_second": 6.092149565034673, + "d2h_gib_per_second": 62.30588646501894 + }, + { + "h2d_seconds": 0.0019016820006072521, + "d2h_seconds": 0.002295907004736364, + "h2d_gib_per_second": 78.56942562020808, + "d2h_gib_per_second": 65.07844707636886 + }, + { + "h2d_seconds": 0.0019015909638255835, + "d2h_seconds": 0.00230049598030746, + "h2d_gib_per_second": 78.5731870535458, + "d2h_gib_per_second": 64.94863011237729 + }, + { + "h2d_seconds": 0.001903570955619216, + "d2h_seconds": 0.0023081169929355383, + "h2d_gib_per_second": 78.4914594640875, + "d2h_gib_per_second": 64.73418070111357 + }, + { + "h2d_seconds": 0.0019029710674658418, + "d2h_seconds": 0.002301126020029187, + "h2d_gib_per_second": 78.51620292838844, + "d2h_gib_per_second": 64.93084741969275 + } + ], + "protected_service_touched": false, + "full_model_serving_claim": false +} diff --git a/docs/freetoken-paper-benchmark-spec.md b/docs/freetoken-paper-benchmark-spec.md new file mode 100644 index 0000000000..e64455c34b --- /dev/null +++ b/docs/freetoken-paper-benchmark-spec.md @@ -0,0 +1,99 @@ +# FreeToken paper benchmark specification and AMD coverage + +This document transcribes the benchmark scope stated in the supplied FreeToken +paper and maps each requirement to the evidence currently available for the +GMKtek EVO-X2 Strix Halo port. It is a planning and evidence index. It does +not treat a paper-inspired workload as an exact reproduction unless the model, +fixture, protocol, and measurement definition are all known to match. + +## Models named by the paper + +| Model | Paper precision and role | AMD status | +|---|---|---| +| Qwen3.6-35B-A3B | BF16; primary model across the four workloads | Bounded Qwen validation and ROCm 10 llama.cpp comparison complete. Exact paper harness parity is not established. | +| DeepSeek-V4-Flash | 284B parameters, 13B active; native MXFP4 routed experts; large-model demonstration | Not reproduced. The exact checkpoint is not present in the current capacity inventory. | +| GLM-5.2 | 753B parameters, approximately 40B active; NVFP4 routed experts; workstation-tier demonstration | Not reproduced. The required 433 GB checkpoint and workstation-class memory are outside the current test inventory. | + +The paper also states that FreeToken supports more than 20 MoE models, but the +evaluation section identifies the three models above as the representative +benchmarks. A support claim is not equivalent to a completed benchmark result. + +## Paper hardware tiers + +The paper reports six discrete-GPU systems: + +| System | GPU and VRAM | PCIe | Measured host-to-device bandwidth | +|---|---|---|---:| +| 5090 | RTX 5090, 32 GB | PCIe 5.0 x16 | 52.7 GB/s | +| 4090 | RTX 4090, 24 GB | PCIe 4.0 x16 | 25.1 GB/s | +| 3090 | RTX 3090, 24 GB | PCIe 4.0 x16 | 25.3 GB/s | +| 5090 desktop | RTX 5090, 32 GB | PCIe 5.0 x16 | 49.0 GB/s | +| 4060 laptop | RTX 4060 Laptop, 8 GB | PCIe 4.0 x8 | 11.8 GB/s | +| PRO 6000 | RTX PRO 6000 Blackwell, 96 GB | PCIe 5.0 x16 | 51.5 GB/s | + +The GMKtek EVO-X2 is not one of these systems. It uses an integrated Radeon +8060S Strix Halo GPU with unified memory rather than a discrete NVIDIA card +with a separately reported VRAM pool. Its results therefore need a separate +AMD platform label and must not be presented as a direct replication of an +RTX 4060, RTX 5090, or RTX PRO 6000 result. + +## Paper workloads + +The evaluation defines four scenarios: + +1. **W1 math reasoning:** AIME competition problems, long chain-of-thought, + no tools, single-turn, decode-dominated. +2. **W2 coding agent:** A SWE-bench repository issue solved through OpenCode, + with real tool execution over three scripted user turns. +3. **W3 native-protocol coding agent:** The same issue driven through Claude + Code using the Anthropic-compatible endpoint. The harness starts concurrent + subagents and grows sessions to approximately 56,000 to 65,000 tokens. +4. **W4 email and calendar agent:** Thirteen fixed user turns over a mailbox + kit through OpenClaw, with an approximately 24,500-token system-context + floor. The paper disables OpenClaw's 120-second idle watchdog for measurement. + +The paper requires the coding runs to produce the reference gold patch and the +W4 run to complete all thirteen turns. Our existing AIME, tool, long-context, +and state-retention tests are useful bounded controls, but they are not exact +W2, W3, or W4 reproductions because the original repository fixtures and agent +clients are not all available in the current evidence set. + +## Paper metrics and reported claims + +The primary metrics are per-request mean decode throughput and per-request +mean TTFT. The paper separately discusses tail TTFT because availability +timeouts matter for agents. It reports FreeToken at approximately 77 to 83 +decode tokens per second on Qwen3.6 and 22 to 25 decode tokens per second on +DeepSeek-V4-Flash on the RTX 5090 setup. + +The paper's prefill analysis also reports an 8,192-token Qwen prefill chunk +completing in approximately 1.19 to 1.22 seconds with pipelined full-layer +loading, and approximately 6,700 tokens per second at 16,000 tokens. These +figures are mechanism-analysis results, not a replacement for the four +agent-workload measurements. + +## Current AMD evidence and gaps + +| Requirement | Current evidence | Classification | +|---|---|---| +| Native ROCm/HIP execution | FreeToken AMD port runs on Strix Halo | Proven | +| Qwen functional quality | Deterministic Qwen matrices and 1,440-session endurance pass | Proven for tested Qwen workload | +| Qwen single-request speed parity | FreeToken approximately 28 decode TPS versus ROCm 10 llama.cpp approximately 47 TPS in the matched control | Gap remains | +| Qwen aggregate concurrency | One four-request control reached approximately 94.8 aggregate decode TPS for both runtimes | Workload-specific parity, not universal parity | +| Gemma 4 bounded operation | Text, vision, concurrency, long-context, and endurance suites pass | Proven for tested Gemma workload | +| Exact W1 to W4 paper reproduction | Fixtures, protocol, and scoring are not all identical | Incomplete | +| DeepSeek-V4-Flash 284B | No checkpoint or measured run in the current inventory | Incomplete | +| GLM-5.2 753B | No checkpoint or workstation-class capacity in the current inventory | Incomplete | + +## Next test gates + +1. Obtain or reconstruct the exact W2, W3, and W4 fixtures and acceptance + criteria before calling those workloads reproduced. +2. Qualify the exact DeepSeek-V4-Flash checkpoint and MXFP4 format only after + a read-only capacity calculation confirms that the test is safe on the + available unified-memory system. +3. Keep Qwen kernel optimization separate from paper-replication claims. Every + candidate must pass deterministic quality, long-context, concurrency, and + recovery gates before it can be compared on TPS. +4. Report AMD results beside, not as replacements for, the paper's discrete-GPU + results unless the model, workload, and measurement protocol are identical. diff --git a/docs/freetoken-swap-completion-audit.md b/docs/freetoken-swap-completion-audit.md new file mode 100644 index 0000000000..dc983080c4 --- /dev/null +++ b/docs/freetoken-swap-completion-audit.md @@ -0,0 +1,163 @@ +# FreeToken swap completion audit + +This audit preserves the full integration goal. A draft PR and passing CPU tests +do not establish that every lifecycle behavior is qualified on real models. It +distinguishes historical evidence from current-branch evidence: neither is +silently promoted to proof for a later native-router implementation. + + +## Final qualification update — 2026-09-23 + +The current native branch has now passed the private GMKtek EVO-X2 two-model +qualification, protected-workload restoration, production migration, and a real +unattended reboot/startup test. The exact source is commit `56e0232`, published +only in draft PR 4. Focused verification reports 183 passing tests; the complete +repository run reports 1947 passed, 81 skipped, and 21 failures that reproduce +unchanged at the parent commit on the same environment. Raw operational evidence +remains private. + +See [the final qualification record](freetoken-swap-final-qualification-2026-09-23.md). +Statements below that describe current-engine evidence as missing are retained as +historical audit snapshots and are superseded by this final record. + +## Historical combined-source verification + +- Swap source: `64dcc683d4e767fb4af8b7088ebb58564b1b7535`. +- AMD model-repair source: `de23ad6a9e74aecc72b9f6b9e81b8c3376ff2e60`. +- Git's clean merge-tree result: `c3c0ae54a09857b98bba83cfc75b91264e6eeb43`. +- The combined tree was archived into an isolated temporary directory on + GMKtek EVO-X2. Neither branch nor the live runtime was replaced by that tree. +- Historical Linux validation: 114 daemon, privacy, benchmark, and reproducibility tests + passed, including the real child-process recovery tests. No skips. +- Combined-tree Qwen validation: 21 grouped-output, SSM, and config tests passed. +- The protected llama.cpp service remained active throughout those CPU checks. + This archived combined tree is not the current `freetoken-swap` branch. + +Reproduce the combined-tree CPU suites from the extracted source, with its +`python` directory on `PYTHONPATH` and the required test dependencies installed: + +```bash +python -m pytest tests/daemon \ + tests/benchmarks/test_public_document_privacy.py \ + tests/benchmarks/test_gmk_evo_x2_benchmark.py \ + tests/reproduce/test_collect_host_manifest.py -q +python -m pytest tests/models/test_qwen36_gdn_grouped_output.py \ + tests/models/test_qwen35_gguf_ssm_a.py \ + tests/models/test_qwen35_gguf_config.py -q +``` + +## Current checkout verification + +- Read-only comparison reference: `mostlygeek/llama-swap` + `41ec321b6216d838488b2a7d936274ed227c0c5e`, whose `LICENSE.md` says MIT. +- Local deterministic verification at `39c3aaabd6fefd7dd462e85e8dfd2ba03be849ab` + on the current Windows checkout: 345 daemon + tests passed and 7 Linux-only tests were skipped. This proves CPU/HTTP + behavior only; it does not substitute for real-model evidence. +- GitHub-hosted Ubuntu verification at + `39c3aaabd6fefd7dd462e85e8dfd2ba03be849ab` (Actions run `34941311939`) + reported 352 tests with zero failures, errors, or skips. This includes the + fail-closed maintenance-host and measured-memory gates, AMD SMI parsing, + queued-disconnect ownership regression, and capability-metadata parser and + listing coverage, model display/metadata collision precedence, ordered + upstream-model/request-filter, per-profile upstream-timeout, safe + direct-upstream static-suffix cold-load guard, startup + preload/profile, and generated-alias coverage, and + pin/warm selector and runtime routing-profile parsing, routing, listing, + metadata, management-isolation, reload-reset, safe readiness/proxy-target, + and qualifier gates. + It also covers bounded body-free activity, authenticated aggregate and + capture retrieval APIs, capture-disabled defaults, serialized-byte and + per-response bounds, credential-header redaction, binary fidelity, eviction, + and exact cold-loading downstream SSE capture. Body-free activity recovery, + bounded corruption handling, atomic compaction, path-free persistence health, + memory-only capture restart behavior, hashed non-credential session grouping, + and explicit-on-click UI capture retrieval are also covered. + The bounded periodic-performance tests cover one-hour eviction, strict + RFC3339 filtering, authentication, disabled 503 behavior, generic probe + failure health, reload generation cleanup, and path/PID omission. + It also executes the + disposable process-group, readiness rollback, + re-adoption, dynamic-port, routed SSE, and cleanup tests that Windows skips. + It is current-branch Linux process evidence, not current-engine or GPU-model + qualification. +- No current-branch maintenance-window benchmark artifact has been published. + Raw paths, prompts, responses, logs, and host data must remain private. +- Current isolated combined-tree CPU verification used swap head + `39c3aaabd6fefd7dd462e85e8dfd2ba03be849ab` and draft AMD compatibility + head `c0534c6f38162cb2ddfd0193cd9bf1031613dde1`. Git produced the clean + synthetic tree `74a4f3b1649442d9d8c24576751d29f30e218d04` without checking out or + changing either branch. On Windows, 377 daemon/privacy/benchmark-contract/ + reproducibility tests passed with 7 expected Linux skips, and all 21 + grouped-output, SSM, and GGUF configuration tests passed in a disposable + Python 3.13 / torch 2.13 CPU environment. This proves source compatibility + only; no protected service, model, GPU, or runtime was inspected or changed. + +## Requirement evidence and gaps + +| Requirement | Evidence | Status | +| --- | --- | --- | +| Official source, license, and provenance | Read-only llama-swap reference pinned to `41ec321b6216d838488b2a7d936274ed227c0c5e`, MIT license; research report and configuration example | Documented and reverified locally | +| Model catalog and lifecycle controls | Validated TOML catalog, collision-safe slash-namespaced and colon-variant alternate IDs, ordered upstream-model/strip/hard/soft/by-ID JSON filters with protected routing identity, runtime and startup pin profiles, singleton native startup preload, pin/warm virtual selectors with listing metadata, unlisted model entries, safe configured readiness paths and manager-owned loopback proxy prefixes, authenticated profile endpoints, native process manager, longest-prefix direct-upstream resolution, and exact explicit/dynamic/omitted-default-port re-adoption. Spillover is rejected as incompatible with one-resident capacity. | Implemented and CPU/HTTP tested; profile/selector/readiness-target/upstream-model/startup live canaries remain required | +| Automatic model routing | Native `freetoken-swap` model-ID admission, readiness-gated activation, request-preserving proxying, cancellation, TTL eviction, reload, and deterministic HTTP tests; prior direct llama-swap runs remain comparison evidence only | Implemented and CPU/HTTP tested; current native real-engine qualification remains required | +| Readiness and API compatibility | Separate `/ready`, uncached generation-aware default health checks, safe profile-configured readiness paths, exact owned-port proxy targets with optional path prefixes, ordinary and SSE completions, side-effect-free sanitized browser preflight, authenticated model-list CORS, exact `/models` listing alias, public model entries with atomic loaded/activating/unloaded status, collision-safe display/JSON metadata, and declarative text/tool/context capability metadata matching the pinned listing fields | CPU/HTTP tested; current native real-engine evidence required | +| Streaming cold-load feedback | Global/per-profile safe configuration; atomic post-concurrency cold admission; reasoning and queue-position SSE; upstream continuation; in-band terminal errors; strict warm/route/stream bypass; explicit cancellation and disconnect cleanup | Deterministic HTTP and hosted Linux disposable-child gates passed; current GMKtek native execution required | +| Concurrency and unloading | Race-safe global/per-profile reservations, default and configured limits, immediate 429, canonical/alternate sharing, same-model and conflicting-model admission, concurrent cold dynamic binding, and idle eviction are deterministically tested | Current native real-engine verification required | +| Rollback protections | Launch/readiness recovery, newer lifecycle intent, accounting preservation, and current-branch hosted Linux real-child rollback/process-group cleanup passed; historical invalid-GGUF evidence is retained separately | Current-engine real-model recovery execution remains required | +| Client cancellation | Native opaque router request IDs, atomic duplicate-ID rejection before admission/upstream work, disconnect-aware admission, queued/connecting/active request list, explicit cancel endpoint across every owned phase, orphan socket close, lease release, and cancellation metrics. Failed, disconnected, or cancelled admission and failed upstream connection release ownership safely. | Deterministic HTTP tested; current native same-instance GPU verification required | +| Authentication and observability | Bearer, Basic-password, and `X-Api-Key` inference authentication with precedence and local termination; separate `X-FT-Token` lifecycle control; catalog-key-protected `/models` compatibility alias; configured aliases and profiles; Prometheus metrics; bounded router-log SSE; exact-origin qualification credentials | Deterministic HTTP tested; current native GMKtek control-plane execution required | +| Model compatibility | Mixed-format Qwen/GDN repair, tokenizer checks, exact-model contracts, prior live completion evidence, and 21 current isolated combined-tree model tests | Source-compatible at the recorded heads; current-engine real-model qualification remains required | +| Production protection | Isolated test paths, explicit maintenance gate, exact operator-supplied hostname required before artifacts or service inspection, historical restore/completion checks, no interruption during combined-tree checks | Maintained and fail-closed; no current protected workload was touched | +| Privacy | Generic GMKtek EVO-X2 label, sanitized public metadata and examples, privacy regressions, regenerated reviewed PDF | Current publication changes sanitized; historical copies not erased | +| FreeToken-only publication | Public GitHub recheck on 2026-09-15: draft PR 1 targets `main` from `feat/freetoken-swap`, and its public PR ref matched the branch head at recheck; draft PR 2 targets `amd-rocm-gfx1151` from `fix/qwen36-swap-compat` | Submitted, draft, not merged | + +The PRs target different base branches: PR 1 targets `main`; PR 2 targets +`amd-rocm-gfx1151`. Their current draft state and branch relationships were +rechecked on their public GitHub pages; current mergeability was not reverified, +and no authenticated mutation was attempted. The clean combined tree is +compatibility evidence, not an instruction to merge either PR or change the +repository's release strategy. PR 1 now has a secret-free GitHub-hosted Ubuntu +daemon check; PR 2 has no hosted check at this audit. The local and hosted test +results are separate evidence. PR 1's public description remains historical and +is not the authoritative record of current-branch qualification. + +## Historical maintenance-window evidence + +The following records describe an earlier approved window, not current-branch +completion proof: + +1. GPU stream cancellation reached terminal idle on the same backend, without + a normal-completion increment. Post-disconnect A-to-B-to-A streaming, + concurrency, and TTL unloading passed. +2. Native daemon recovery passed after the real loader rejected an invalid + GGUF fixture. The restored Qwen3.6 model reached readiness and generated the + expected answer. The failed switch correctly remained HTTP 503. +3. Both phases restored and health-checked the protected service, including a + verified completion. Final process/listener checks found no test runtime + remaining. The accounting gap for the crashed loader is explicitly degraded. + +The approved historical window is closed. No permanent production activation, +merge, or upstream submission was performed. Long-context quality, broad model +compatibility, direct-router automatic rollback, and long-duration endurance +remain explicitly unclaimed limitations. + +## Historical completion gates (superseded by the final qualification record) + +At the time of this earlier audit, the native router was **not complete** until an approved GMKtek EVO-X2 +maintenance window runs the current branch's +`benchmarks/swap/qualify_native_router.py`, retains its raw artifacts privately, +and records sanitized direct, warm-routed, cold-routed, alternating-model, +resident-target warm-selector, +runtime-profile activation/composition/clear, +router-cancellation, same-model concurrency, conflicting-model queue/drain, +failed-switch rollback/accounting, +same-process re-adoption, active-reload-conflict, capacity-safe persistent residency, +TTL-eviction, unauthenticated 401, authenticated model/profile inventory, +Prometheus, bounded router-log, and positive available periodic owned-process +RAM/VRAM results without PID/model/path fields. It must also run Linux real-child tests on +the current branch, then restore and health-check the protected workload. No +merge, permanent service activation, or publication of raw artifacts is +authorized by this audit. + +See [integration behavior](freetoken-swap.md) and +[source research and live-test limitations](freetoken-swap-research.md). diff --git a/docs/freetoken-swap-final-qualification-2026-09-23.md b/docs/freetoken-swap-final-qualification-2026-09-23.md new file mode 100644 index 0000000000..c8a9a897fd --- /dev/null +++ b/docs/freetoken-swap-final-qualification-2026-09-23.md @@ -0,0 +1,73 @@ +# FreeToken native swap final qualification — 2026-09-23 + +This record supersedes earlier statements that current-engine GMKtek EVO-X2 evidence was still missing. Raw paths, credentials, prompts, responses, process identifiers, and host-specific artifacts remain private. + +## Qualified source + +- FreeToken source commit: `56e0232` (`feat(daemon): qualify native model swapping on AMD`). +- Draft review: `dbourdea/FreeToken` PR 4, from `codex/freetoken-swap-amd` to `main`. +- Read-only reference: `mostlygeek/llama-swap` commit `41ec321b6216d838488b2a7d936274ed227c0c5e`, MIT licensed. +- No llama-swap or llama.cpp source was modified or submitted. + +## Deterministic and repository evidence + +- Changed-file Ruff checks passed. +- The focused daemon, router, metrics, qualification, and Qwen GGUF configuration suites passed: **183 passed**. +- The complete repository run reported **1947 passed, 81 skipped, and 21 failed**. +- Every one of the 21 failures reproduced at the parent commit on the same environment. They are baseline platform/dependency failures outside the changed daemon, router, metrics, qualification, and Qwen 3.5/3.6 GGUF files; they are not presented as a green full-suite result. +- Python compilation and `git diff --check` passed before commit. + +## Private live two-model matrix + +The final private qualification result recorded `passed: true`, `restored: true`, no primary error, no cleanup error, and no restoration error. The same run behaviorally proved: + +| Capability | Sanitized result | +| --- | --- | +| Direct, warm, cold, and A-B-A routing | Passed with distinct measurements and correct model identity. | +| Aliases, configured readiness, upstream model rewrite, selectors, and startup profile | Passed. | +| OpenAI-compatible ordinary and SSE proxying | Passed, including terminal stream handling and usage-backed performance collection. | +| Authentication | Unauthenticated inference and management were rejected; Bearer, Basic-password, and `X-Api-Key` succeeded. | +| Same-model concurrency and conflicting-model drain | Passed without overlapping owned engines or killing an active request. | +| Explicit cancellation | Passed with terminal abort, lease release, and cancellation accounting on the same daemon instance. | +| Failed switch and rollback | Passed with a visible failure, durable accounting evidence, restored prior model, and a successful restored completion. | +| Restart and re-adoption | Passed with exact persisted process identity and no duplicate runner. | +| Reload conflict | Passed; active lifecycle redefinition was rejected atomically. | +| Capacity-safe persistence and TTL eviction | Passed under the declared one-resident capacity policy. | +| Metrics, logs, activity, and hardware telemetry | Passed; owned-process RAM and AMD VRAM/GTT were positive and source-labelled. | +| Cleanup and protected-workload restoration | Passed with no qualification listeners or engines left behind. | + +## Production migration evidence + +The legacy model runner and its model-specific watchdog timer were disabled, and the model-specific cache cron entry was removed. Their prior units and crontab were backed up before modification. The generic thermal watchdog and unrelated backup workflow were preserved. + +One user-level `freetoken-swap.service` now owns the native daemon and exactly one model engine. The service: + +- is enabled under the lingering user `default.target`; +- binds the stable LAN control/inference port while engine ports remain loopback-only and dynamically allocated; +- applies the validated power profile before launch; +- uses a private extension cache and a mode-600 catalog/API key; +- preloads the qualified GPT-OSS model because it starts reliably inside the bounded boot gate; +- keeps the qualified Qwen GGUF available for routed on-demand swapping; +- stops its owned engine with the daemon and has no second model supervisor. + +A real reboot changed the boot identifier and then produced all of the following without a login or manual action: + +- the service became enabled and active; +- the startup profile became active; +- `model-b` was readiness-gated resident with exact identity matching; +- unauthenticated model listing returned 401; +- authenticated health returned 200 with `engineRunning: true`; +- an authenticated deterministic completion returned exactly `4` from `model-b`; +- owned-process telemetry reported approximately 14.5 GB of AMD VRAM/GTT and positive process RAM; +- the stable port was reachable from another LAN computer; +- the retired service and timer remained disabled and inactive. + +The first reboot exposed and corrected a systemd ordering cycle caused by requiring a watchdog that itself ordered after `default.target`. The corrected unit requests the watchdog without ordering the model service after it. A second real reboot proved the fix. + +## Compatibility disposition + +Applicable pinned-reference behavior is implemented and behaviorally evidenced for FreeToken's text-generation backend. Unsupported embeddings, rerank, image, speech, transcription, SDAPI, and ComfyUI routes remain intentionally inapplicable because FreeToken has no matching backend. MCP, Tailcat, and remote peer transport remain explicit future product expansions, not fabricated parity. Multi-resident matrix solving remains inapplicable under the declared one-engine ownership policy; the implemented equivalent is exclusive capacity, protected persistence, FIFO/priority admission, and deterministic eviction. + +## Release disposition + +The source is published only as a **draft** FreeToken pull request. It is not merged. Raw operational artifacts remain private. This evidence closes the current-engine, GPU-model, restoration, and unattended-startup gates for the qualified GMKtek EVO-X2 configuration; it does not claim broad model-family quality, long-context quality, or multi-host peer support. \ No newline at end of file diff --git a/docs/freetoken-swap-native-qualification.md b/docs/freetoken-swap-native-qualification.md new file mode 100644 index 0000000000..756b5acb41 --- /dev/null +++ b/docs/freetoken-swap-native-qualification.md @@ -0,0 +1,157 @@ +# Native freetoken-swap qualification runbook + +This runbook qualifies the FreeToken-owned router. It does not qualify the +separate llama-swap integration and it does not authorize production activation +or a merge. Use only an approved maintenance window on **GMKtek EVO-X2**. + + +## Completed qualification record + +The approved private execution for source commit `56e0232` passed the full +matrix with clean cleanup and restoration. A later authorized production +migration also passed authenticated completion and real unattended reboot/startup +proof. Raw artifacts remain private. See +[the sanitized final record](freetoken-swap-final-qualification-2026-09-23.md). +The remaining instructions are retained as the reproducible runbook. + +## Preconditions + +- An approved authentication method for the qualification host is available. +- The protected workload, its exact restoration procedure, and a deterministic + health request are recorded privately before any change. +- The current FreeToken feature branch is checked out in an isolated path. +- Model artifacts, tokenizer, native extension, and FreeToken runtime are + verified known-good for one short deterministic completion before swap tests. +- The native catalog has two aliases with distinct model paths, loopback ports, + bounded context settings, and no unreviewed `drop_fields` policy. +- `git status --short` is clean except for intentional qualification-only files. + +Do not copy private paths, addresses, hardware identifiers, prompts, raw model +output, API keys, or request headers into the public PR. + +## Baseline capture + +Capture privately, before starting the daemon: + +1. Protected workload health and one deterministic completion. +2. Current listener and process inventory for the selected temporary ports. +3. Host memory, accelerator-visible memory, and available system memory as + separate values. +4. FreeToken branch commit, catalog hash, model file hashes, Python version, + ROCm version, and runtime package versions. + +Abort before loading if the protected workload is unhealthy or the measured +capacity is lower than the documented gate. + +## Native router matrix + +Run each test through the daemon's stable URL, never by calling the engine port +directly. Keep request and response content private. Record status codes, +model alias, elapsed time, first-byte time, final duration, usage-derived completion tokens/second, router metrics, engine metrics, accounting receipt IDs, and cleanup result. + +| Test | Required observation | Pass condition | +| --- | --- | --- | +| Cold A | First request to alias A | Ready engine, valid ordinary completion, router activation increments | +| Warm A | Repeat alias A | Same engine PID, no activation increment, completion succeeds | +| Warm selector | Request the temporary warm selector while A is resident | Request is rewritten to A, completion succeeds, and activation remains unchanged | +| Runtime profile | Activate the temporary profile, request its pin through the warm selector, then clear it | Virtual pin is listed only while active, disabled pin stays omitted, completion uses resident A with zero activation, and the profile is cleared before later trials | +| Configured readiness target | Start and recheck temporary profiles through their configured `/ready` path | Control inventory reports `/ready`; activation and stable daemon readiness succeed without leaving the exact manager-owned port | +| Upstream model-name rewrite | Request temporary alias `compat/model-a` while A is resident and configured with `use_model_name = "model-a"` | The streamed response reports upstream model `model-a`, routing remains resident on A, active requests return to zero, and activation count is unchanged | +| Model-list display metadata | Inspect canonical A and `compat/model-a` through both public listing aliases and authenticated router inventory | Display name and operator metadata are present on both public IDs; canonical/alias identity metadata overwrites the conflicting operator `type`; inventory remains sanitized and consistent | +| Per-profile upstream timeout | Inspect the temporary profile inventory and complete routed ordinary/SSE canaries | Inventory reports the profile override and routed requests complete through that acquired-profile connection policy rather than the distinct global fallback | +| Startup preload and profile | After maintenance begins, restart the temporary daemon with alias A configured for preload and `coding` as startup profile while the exact A child remains available for re-adoption | Alias canonicalizes to A, startup profile is active, preload reuses the exact adopted PID/port with zero activation delta, and a routed completion succeeds | +| Cold B | Request alias B after A is idle | A receives durable stop receipt, B becomes ready, completion succeeds | +| A to B to A | Three routed requests | Each expected alias returns, no overlapping owned children, every replacement is ready | +| SSE | Stream an alias request | First event and terminal event arrive, final lease count is zero | +| Explicit cancel | Send a long stream with `X-FT-Request-ID`, call router cancel | Same engine reaches zero active leases, cancellation counter increments, no normal completion is credited | +| Same-model concurrency | Two A requests | Both complete, one resident engine, no unintended switch | +| Conflicting request | Keep A stream active then request B | B waits or receives documented capacity response, A is not killed mid-stream | +| TTL | Allow a nonpersistent idle profile to reach its TTL | Engine stops through accounting path, listener closes, eviction increments | +| Bad replacement | Select an intentionally invalid disposable fixture | HTTP failure is visible, previous engine recovery is attempted only when applicable, failed receipt is degraded rather than fabricated | +| Reload | Replace catalog with a valid idle change, then an invalid or active-profile redefinition | Valid change applies atomically; invalid and active redefinitions are refused without altering live ownership | +| Authentication and control plane | Probe inference, both public model-list paths, namespaced direct upstream, and management without credentials, then exercise Bearer, Basic-password, and `X-Api-Key` before inspecting aliases, profiles, metrics, and router-log SSE | Unauthenticated inference, `/v1/models`, `/models`, and management return 401; both listings are equivalent apart from request timestamps; the temporary `compat/model-a` alias reaches resident A's `/v1/stats`; all three key forms expose exact resident A; authenticated inventory is consistent; metrics and a bounded `management_loaded` event are available | + +## Separate performance evidence + +`benchmarks/swap/qualify_native_router.py` is the opt-in Linux harness for +collecting the four required comparisons in one approved maintenance window. +It starts a private native daemon with a private state directory and extension +cache and a validated dynamic-port TOML catalog. It generates a fresh private +router API key for the run and scopes that credential to the exact temporary +daemon origin; the protected service and direct engine comparison never receive +it. Before performance trials, it requires unauthenticated `/router/status` and +`/v1/models` and `/models` requests to return 401, verifies their normalized +listing equivalence plus Bearer, Basic-password, and `X-Api-Key`, then +authenticates alias, selector, model, profile, configured readiness target, +Prometheus, and bounded router-log SSE checks. Their raw responses and the key-bearing +catalog remain private. The harness then records private raw artifacts for: a direct request to the +router-owned engine port, warm-selector and runtime-profile composition canaries, a warm routed request, a cold routed swap to the +other model, and an alternating routed swap back. It requires streamed OpenAI usage, +then records first-byte time, final duration, completion tokens, and usage-derived +decode tokens/second at the client. Before the comparison sequence it also opens a +long routed stream, explicitly cancels its opaque request ID, and fails unless the +router returns to idle, increments cancellation telemetry, emits no normal terminal +completion credit, and the retained private partial SSE lacks `[DONE]`. It then runs +two simultaneous same-alias streams and fails unless both complete with zero +activation delta and one unchanged resident profile. It stores the corresponding +`/router/status` snapshot and Prometheus `/metrics` response for each routed +comparison, and fails if activation counters do not prove the advertised +warm/cold/alternating state. It then switches to a deliberately missing private +model fixture and requires HTTP 503, a successful rollback launch, restored exact +identity, an activation-failure increment, and a valid completion from the restored +model. The harness compares the private accounting outbox before and after that +failure and requires at least one new valid receipt ID, publishing only the count. +It also writes a private active-profile priority change and requires the router to +reject it with HTTP 409 while retaining exact active identity. +Before that reload check, the harness gracefully terminates the first daemon while +leaving its test-owned engine detached, starts a replacement daemon against the same +private state, and requires the manager's adopted flag, engine PID, engine port, and +router profile identity to match. A routed completion must succeed with zero router +activations before the replacement daemon becomes the final cleanup owner. +It then holds a long model-A stream while requesting model B, requires B to be +visibly queued with A still exact and active, cancels A, and requires exactly one +activation into B followed by one activation restoring A. The cancelled A prefix +must not contain a normal terminal marker. +The harness then reloads a singleton persistent group while no model is resident, +loads that profile, and requires a conflicting load to return HTTP 409 while the +same PID remains exact and persistent. Explicit unload must release the slot and +allow the conflicting profile to activate. +Finally, it explicitly unloads its temporary resident, atomically reloads the private +catalog with a two-second idle TTL, verifies TTL-driven eviction and listener closure, +and leaves no temporary engine for daemon cleanup. +The direct comparison retains its router-owned load receipt and activation +snapshot privately as well. +It also saves the authenticated-local `/router/hardware` process and memory +snapshot for every comparison privately. Each loaded-model snapshot must contain +positive Linux process-tree PSS and per-process GPU-memory values with explicit +available/source markers; an unavailable probe or compatibility zero fails the +run. The published result must remain a sanitized aggregate. + +The harness requires `--allow-maintenance`, the exact operating-system hostname +in `--expected-hostname`, a new empty `--artifacts` directory, two known-good +model paths, and the protected service's private restore endpoint. A hostname +mismatch fails before artifact creation, service inspection, or maintenance and +does not disclose either hostname. It then verifies the protected baseline, prebuilds kernels, +stops the protected service only after the native daemon is reachable, and +always attempts daemon cleanup and protected-workload restoration. Do not run +it on Windows or substitute a direct engine URL for the routed cases. Publish +only sanitized aggregate timings and explicit pass/fail results; raw responses, +paths, daemon logs, catalog, and host data remain private. +The harness also requires the final temporary engine listener to be closed +before it can report success. + +## Restoration and acceptance + +After testing: + +1. Stop temporary daemon and all test engines through their owned lifecycle. +2. Verify temporary listeners are closed and no test worker process remains. +3. Restore the protected workload exactly as captured. +4. Verify its health and deterministic completion. +5. Preserve raw logs and request content privately. Publish only sanitized + aggregate timings, pass or fail outcomes, anonymous hardware label, commit + hashes, and explicit limitations. + +The native route is not qualified by an open port, a green `/health`, or a +single start. Qualification requires the relevant end-to-end matrix evidence +above and protected-workload restoration. diff --git a/docs/freetoken-swap-parity-matrix.md b/docs/freetoken-swap-parity-matrix.md new file mode 100644 index 0000000000..7f0452f66b --- /dev/null +++ b/docs/freetoken-swap-parity-matrix.md @@ -0,0 +1,132 @@ +# freetoken-swap parity matrix + +This is the implementation acceptance contract for native `freetoken-swap`. +It is based on the read-only official llama-swap reference at commit +`41ec321b6216d838488b2a7d936274ed227c0c5e`, MIT licensed. It does not copy +that project's code or authorize changes outside FreeToken. + +Status labels: + +- **Native**: implemented by FreeToken and behaviorally tested. +- **Integrated only**: available only when an unmodified llama-swap binary + supervises FreeToken. This is not native parity. +- **Missing**: applicable, not yet implemented. +- **Deferred**: a potentially applicable product expansion that is not part of + the current one-engine contract. It remains an open difference, not parity. +- **Inapplicable**: a stated backend-modality or one-engine architectural + constraint makes the behavior impossible or misleading. The constraint is + named explicitly rather than compatibility being claimed. + +The field- and route-level classifications behind this matrix are recorded in +[the pinned-source inventory](freetoken-swap-source-inventory.md). + +## Final live-evidence status — 2026-09-23 + +Commit `56e0232` passed the private GMKtek EVO-X2 native two-model matrix with +clean restoration and cleanup, followed by deployment under one native service +and a real unattended reboot/startup proof. Direct, warm, cold, A-B-A, selector, +profile, authentication, SSE, concurrency, cancellation, failed-switch rollback, +re-adoption, reload conflict, persistence-capacity, TTL, metrics, logs, positive +owned-process AMD memory, and deterministic completion gates passed. See +[the final qualification record](freetoken-swap-final-qualification-2026-09-23.md). +Any table wording below that says this current live execution is still required +is historical pre-gate wording and is superseded by this status section. + +## Pinned-source inventory + +The following is a read-only source inventory, obtained with `git show` and +`git ls-tree` from the pinned commit rather than from the damaged local working +copy. It makes the scope of the comparison auditable without vendoring any +llama-swap code. + +| Reference source at `41ec321…` | Observed responsibility | Native classification and evidence | +| --- | --- | --- | +| `internal/server/server.go` (`modelPostJSONRoutes`, `modelPostFormRoutes`, `modelGetRoutes`, `routes`), `internal/server/api.go` (`handleListModels`), `internal/swaputil/http.go` (`FindModelInPath`, `EscapedPathSuffix`) | Model-dispatched OpenAI, Anthropic, embeddings, rerank, audio, images, SDAPI, ComfyUI and upstream routes; public model records and status; slash-namespaced longest-prefix upstream dispatch with escaped suffix preservation; list, health, unload, running, logs, metrics, UI, API group, browser CORS | Native text-generation routes, guarded namespaced passthrough, browser preflight/model-list CORS and atomic public loaded/activating/unloaded model status, plus a local management UI, are implemented and HTTP-tested. Embedding, rerank, image, speech, transcription, SDAPI and ComfyUI are **inapplicable** because FreeToken exposes no matching backend route. MCP and Tailcat remain explicitly deferred product surfaces. | +| `internal/config/{config,model_config,commands,filters,macros,selectors,profile,upstream,performance,peer,tailcat}.go`, `internal/server/{api,filters,selector,profiles}.go`, `docs/kb/guides/{api-integration/filters-and-request-rewriting,routing/profiles-and-selectors,model-runtime/capabilities-and-model-listings}.md` | YAML schema, command/macro expansion, request rewriting, runtime pin profiles, selectors, display and capability metadata, peers, hardware/performance policy, and global/per-model `sendLoadingState` | Native allowlisted TOML parser rejects commands/macros and unsafe owned options; aliases, dynamic ports, readiness, TTL, groups, priorities, keys, upstream timeout, safe ordered strip/hard/soft/by-ID JSON filters, runtime pin profiles, pin/warm selectors, model display/JSON metadata, global/per-model loading feedback and atomic reload are behavior-tested. Text input/output, tool-calling, and context declarations render the pinned listing fields but do not enable behavior. Unsupported backend modality and reranker claims fail closed. Spillover is inapplicable to one-resident capacity; macro, peer and Tailcat policy remain deferred or inapplicable rather than emulated unsafely. | +| `internal/router/{router,base,loading,group,matrix,matrix_solver,peer}.go`, `internal/router/scheduler/fifo.go` | Loading, queueing, group/matrix and peer routing | Native single-owner FIFO/priority coordinator, exclusive one-resident capacity, persistent-group protection, leases, eviction and cancellation are tested. Multi-resident matrix solving and peers are deferred: the declared one-engine supervisor cannot prove safe concurrent residency. | +| `internal/process/{process,process_command,runtime_*,treecleanup_*}.go` | Child launch, process identity, stop/reap/tree cleanup | Native `ServeManager` owns the child, durable state, exact identity/re-adoption, process-group cleanup, drain/abort accounting and rollback. On daemon reconstruction, the routing coordinator binds one unambiguous catalog profile to an exact explicit, dynamic, or omitted-default-port adopted identity; ambiguous or argument-mismatched identities fail closed. Deterministic and Linux actual-child recovery tests cover this boundary. | +| `internal/server/{auth,profiles,inflight,log,metrics,metrics_middleware,api,apigroup}.go`, `internal/logmon/*`, `internal/perf/*`, `internal/store/*` | API-key auth, profiles, inflight cancellation, log streams, Prometheus/activity/performance and persistence | Native Bearer, Basic-password, `X-Api-Key`, and dedicated control authentication, profiles, opaque cancellation, bounded engine/router logs, Prometheus lifecycle/queue/transport signals, durable accounting/activity, and a memory-only one-hour owned-process performance history are implemented. Token throughput, memory and extended performance evidence remain bounded live-test gates. | +| `internal/server/{ui,apimcp,captures,tailcat}.go`, `ui/*`, `internal/mcptools/*`, `internal/tailcat/*` | Browser UI, embedded MCP, captures and Tailcat | Native local management UI, restart-durable body-free activity, hashed session grouping, and opt-in bounded/redacted memory-only captures are implemented. MCP and Tailcat are **deferred** product expansions. | +| `internal/**/*_test.go`, `docs/kb/guides/**/*` | Reference behavioral tests and operator documentation | Native tests live in `tests/daemon`; the qualification runbook and completion audit separate deterministic, Linux and approved maintenance-window evidence. | + +| Pinned llama-swap capability | Current FreeToken state | Required native parity evidence | +| --- | --- | --- | +| Model catalog and aliases | Native TOML catalog with collision-safe slash-namespaced and colon-variant canonical/alternate model IDs, unlisted model entries, runtime pin profiles, pin/warm virtual selectors, global/per-model concurrency, validated model, port, args, readiness path, manager-owned loopback proxy target, optional upstream model-name override, unload, and global/per-profile upstream response timeouts. Model IDs use safe nonempty ASCII segments with a 128-character total cap; groups and each dotted filter-path segment retain their narrower grammar. `port = 0` requests a concrete kernel-selected loopback port for each activation. Model/profile/selector lookup, priority ticketing, head-of-queue port binding, and concurrency reservation are atomic with reload, which rejects admission/lifecycle races. | Deterministic tests prove namespaced/variant canonical and alternate routing, unsafe empty/traversal-like segment rejection, alternate-ID canonical residency, optional alias listing, hidden-model routing/list omission, alias unload, profile/selector validation, safe custom readiness and proxy-prefix targets, upstream model-name and timeout validation/inventory, remote/explicit-port/query/fragment/traversal rejection, concurrent cold dynamic-target sharing, allocation-failure cleanup, dynamic-port residency stability, atomic lookup/port binding, queued-request profile snapshot, reload profile reset, and a fresh target after a swap; a Linux real-child test exercises fresh dynamic ports across eviction/reactivation. TLS and pooled-connection timeout knobs are inapplicable to fresh plain-HTTP loopback targets. | +| Virtual model selectors | **Native, applicable subset.** `pin` selects the first ordered local target. `warm` chooses the first exact ready target, then the first activating target, else the first target. The virtual ID is rewritten before target alias filters. Public listing status follows pinned strategy semantics and carries optional name, description, and JSON-compatible metadata with router-owned keys protected. Selector IDs are not direct-upstream or unload IDs. `spillover` is **inapplicable** because its concurrent reservation distribution requires multi-resident or peer capacity, which conflicts with the one-child supervisor contract. | Deterministic parser, routing, concurrent activation, rewrite/filter order, event identity, direct-upstream rejection, hidden/listing status, and metadata tests pass. The private current-engine harness lists the selector and must prove a warm selector reuses resident A with zero activation delta; execution remains required. | +| Start, stop, switch, PID identity, re-adoption | Native manager is the sole process owner. Routed transitions, HTTP and OS/lifespan daemon exit, and legacy manual engine controls use the same coordinator; manual claims fail while routing owns or admits work. Explicit, dynamic, and omitted ports are matched to exact persisted targets, with omitted ports bound only to the configured default. | Deterministic tests prove exact explicit/dynamic/omitted-default-port re-adoption, ambiguity and argument mismatch rejection, recovered identity after failed readiness, matching-token release, routed-lease conflict rejection, stop preemption with stale-token protection, routed admission waiting behind a blocked or client-disconnected manual start, failed-readiness rollback completing after client cancellation, shutdown rejecting queued/new admission while draining active leases and all manual transaction tokens, and drain-before-detach with idempotent exit handling. The complete suite, including disposable actual-child/process-group tests, passed twice on hosted Ubuntu at `5ee1e26`; current-engine evidence remains required. | +| Readiness and diagnostic health | Native `/ready` atomically checks exact resident identity and an uncached profile-configured engine path behind the admission barrier; `/health` retains generation-aware status/maintenance semantics and diagnostic daemon `/health` remains liveness. Non-health readiness paths use HTTP-success semantics but cannot leave the owned loopback port. | Deterministic tests prove default health-state handling, custom-path dispatch, no cold-load, stale model/args/port rejection, maintenance-state rejection, active-target reload refusal, and that a conflicting swap cannot begin during a successful readiness probe. The private qualifier configures the real engine `/ready` path; current execution remains required. | +| Automatic OpenAI model-ID routing | Native single-engine coordinator with priority-aware admission and health-gated activation | Deterministic HTTP coverage plus hosted Linux disposable-child routing passed. GMKtek EVO-X2 real-engine evidence remains required. | +| OpenAI model list, completion and chat completion forwarding | Native catalog-key-protected `GET /v1/models` and pinned `GET /models` alias return identical visible canonical IDs and, by policy, alternate IDs; unlisted profiles and aliases are omitted. Public records carry standard ownership/timestamp fields, optional descriptions, and atomic loaded/unloaded status: launch intent alone remains unloaded, while an exact manager-owned child in readiness-gated activation is loaded; canonical and alternate IDs share status. Request-byte-preserving proxy includes SSE body forwarding. The backend's stateless response-resource lookup/cancel routes preserve its authenticated `invalid_request_error` 404 without arbitrary model activation. | Deterministic tests cover exact alias payload/CORS/key protection, unloaded, pre-ownership launch intent, exact activating, resident, stale-identity and activation-failure recovery status; canonical/alternate listing and routing without local model-path or argument disclosure; hidden routable profiles; every model-bearing supported text endpoint; stateless response-resource compatibility without admission; request bytes; SSE bytes; upstream error status/body/safe headers; and lease release. Direct, cold, warm, cancellation, and performance evidence remains required. | +| Model display and capability metadata | Native profiles accept display `name`, JSON-compatible nested `metadata`, and declarative text `capabilities` for `in`/`out`, `tools`, and nonnegative `context`. Canonical and listed alternate records share display values and render pinned `architecture`, `capabilities.function_calling`, `supported_parameters`, `context_length`, `context_window`, and `meta.n_ctx` fields. Operator metadata is nested under `meta.freetoken`; router-owned canonical/alias identity wins collisions, and capability-owned keys are filtered when capabilities are declared. Metadata does not change routing or enable inference features. | Deterministic catalog and HTTP tests prove nested/list/scalar JSON validation, display propagation, exact canonical/alias identity, collision precedence, capability rendering, empty-capability behavior, no model-path disclosure, malformed-type rejection, and fail-closed rejection of unsupported image/audio/video or reranker claims. The private control-plane gate verifies display and metadata consistency across both public-list aliases and router inventory; current GMKtek execution remains required. Operators remain responsible for advertising tools only when the selected model and template actually support them. | +| Browser CORS compatibility | Native global `OPTIONS` preflight returns the pinned 204 compatibility headers without entering routing or lifecycle work; requested header names are token-sanitized. Authenticated `/v1/models` and `/models` reflect `Origin`. | Deterministic HTTP tests prove unknown-path preflight, default and sanitized requested headers, zero manager calls, retained 401 on unauthenticated model listings, and origin reflection after bearer authentication. | +| Optional streaming cold-load feedback | **Implemented, applicable.** Native global configuration with a nullable per-profile override applies only to strictly streaming `/v1/chat/completions` when the exact target is not readiness-gated resident. An atomic post-concurrency reservation signal commits HTTP 200 only for admitted cold work, emits reasoning and queue-position SSE, then continues the real upstream stream; post-commit activation/connect failures are framed in-band with `[DONE]`. | Deterministic tests prove queued cold and warm behavior, global/override precedence, strict route/stream eligibility, unchanged disabled-path status/body/headers, preserved upstream SSE, pre-admission 429 JSON, activation/connect failure framing, explicit cancellation, client-disconnect cleanup, reservation/lease ownership and metrics. The hosted Linux disposable-process router test passed with loading feedback before the real child's terminal SSE. The private native harness requires loading frames on cold-B/A-B-A trials and their absence on warm-A; current GMKtek execution remains required. | +| OpenAI Responses endpoint | Native `POST /v1/responses` uses the same admission and proxy contract. FreeToken's stateless response lookup/cancel stubs are authenticated compatibility routes that preserve the engine's `invalid_request_error` 404 without model admission. | Deterministic tests cover the model-bearing routed endpoint and exact no-admission lookup/cancel errors. Add routed response-object and cancellation proof only if FreeToken gains a stateful backend. | +| Reference versionless and llama.cpp-native text aliases | The pinned reference routes `/v/chat/completions`, `/v/responses`, `/v/completions`, `/v/messages`, `/v/messages/count_tokens`, `/completion`, and `/infill`. FreeToken's engine registers none of these aliases; its text contract is the `/v1/*` surface above plus model-less legacy `/generate`. | Intentionally inapplicable while the backend lacks those routes; do not advertise fabricated compatibility. A custom or future backend route remains reachable only through explicit `/upstream/{profile}/...` selection until it becomes a FreeToken-supported model-bearing endpoint. | +| Anthropic Messages and token-count routing | Native routes use the same admission and proxy contract | Deterministic HTTP tests cover both Messages and token-count routing; add live failure proof. | +| FreeToken legacy `POST /generate` | The request schema has no model identifier, so an automatic route at the stable daemon URL is intentionally inapplicable: choosing a model would require an unsafe implicit default. Profile-qualified `POST /upstream/{profile}/generate` remains available through unified admission. | Deterministic HTTP proof rejects ambiguous top-level `/generate` and preserves the explicit passthrough method, body, SSE response, and lease. | +| Unknown-model status and direct upstream access | Native stable `unknown_model` error envelope and `/upstream/{model-id}/...` passthrough through the same lease. The longest configured canonical/alternate ID wins when IDs contain slashes; encoded model separators and the downstream escaped path/query are preserved. A safe bounded suffix policy defaults to pinned static extensions and returns 409 before reservation or activation when the exact model is unloaded. | Deterministic HTTP tests prove the identical 404 error type across all five routed text endpoints, namespaced longest-prefix and encoded-alias routing, exact escaped slash/query forwarding, bare-root and GET passthrough, cold static rejection with zero lifecycle/upstream work, warm static forwarding, policy validation, and rejection of unsafe direct `prepare-stop`. The private native harness requires a namespaced alias `/v1/stats` passthrough; GMKtek execution remains required. | +| FIFO, priority, concurrency, exclusive group routing | Native priority-aware FIFO queue, pinned default per-profile concurrency cap of 10, optional per-profile/global overrides, immediate 429 rejection with `Retry-After`, and one-engine exclusive admission. Reservations cover active, queued, and activating requests and alternate IDs share their canonical cap. The TOML parser rejects coexistence flags it cannot honor while admitting singleton persistent protected slots. | Deterministic tests cover default/override/global limits, alternate-ID sharing, immediate rejection before queue/upstream work, request-ID cleanup, released-slot reuse, duplicate-release protection, priority-before-earlier-low-priority queueing, accepted/rejected group policy, and capacity protection. The private native harness holds A, proves B queues without disturbing A, cancels A, and requires ordered B then A activation; current GMKtek execution remains required. | +| Matrix or equivalent capacity policy and eviction costs | **Native equivalent policy:** the sole `ServeManager` child is the one resident slot; status exposes its exact identity, group, availability, queue, and eviction decisions. | Deterministic tests and the private maintenance harness cover exclusive transitions and persistent-slot protection. Multi-resident matrix solving and memory-ranked victim selection are **inapplicable under one-engine ownership** because there is never a choice among co-resident victims; they become deferred requirements only if FreeToken adds multi-engine ownership. | +| Persistent resident models and startup preload | Native persistent group protects the sole resident slot until explicit unload. A validated singleton `preload_model` canonicalizes aliases and acquires through the same native lifecycle during app startup; optional `startup_routing_profile` activates a validated pin map before serving. Selectors, unknown targets, and multiple preloads are rejected under one-resident capacity. | Deterministic parser and lifespan tests prove canonicalization, unknown-target rejection, native readiness-gated preload, zero residual lease, and startup profile activation. The private harness enables startup only after protected maintenance begins and requires preload to reuse the exact re-adopted A PID/port with zero activation delta. Multi-resident preload is inapplicable with the current one-engine supervisor; GMKtek execution remains required. | +| TTL and unload timeout | Native timer schedules idle-only eviction; authenticated `POST /router/unload` uses the profile or global graceful-stop timeout and the existing accounting transaction | Deterministic lease/TTL and explicit-unload tests cover no eviction while leased, profile timeout selection, and durable manager cleanup; real-engine endurance remains separately bounded. | +| Load/unload management API and running-model list | Native router status, configured plus resident `/router/models`, authenticated lifecycle profiles at `/router/profiles`, `POST /router/load`, and `POST /router/unload` through the same lifecycle coordinator. The daemon CLI reads `/router/profiles`; `/models` is reserved for pinned public-list compatibility. A named body unloads that profile; no body unloads all residents (the current resident under one-engine capacity). | Deterministic HTTP tests prove CLI/control authentication, no profile-path disclosure through `/models`, named mismatch preservation, named unload, and no-body unload-all. Load-all and multi-resident management are inapplicable to the explicit one-engine capacity policy. | +| Runtime routing profiles | **Native:** validated `[profiles..pins]` atomically replaces a set of client model IDs before selectors, aliases, and target filters. Empty targets disable pins. Profile pins compose with selectors, rewrite longest direct-upstream prefixes, add non-shadowing virtual IDs to public listings, start cleared unless `startup_routing_profile` is configured, and reset on catalog reload. Authenticated `PUT /router/profiles/active` and CLI verbs activate or clear the map; concrete lifecycle load/unload ignores it. | Deterministic parser, startup, API, CLI, disabled-pin, shadow, profile→selector, alias-filter, escaped direct-upstream, listing, event, queued-snapshot, management-isolation, and reload-reset tests pass. The private harness must exercise both runtime and restart-time activation while reusing resident A with zero activation; current GMKtek execution remains required. | +| API keys | Native router keys accept case-insensitive Bearer, Basic-password, or `X-Api-Key` for inference-compatible routes (including both model-list paths) and, absent a separate daemon token, management; explicit Authorization wins over fallback. `X-FT-Token` remains the dedicated control-plane override, does not bypass catalog-key-protected inference listings, and all local credentials are terminated before proxying. | Deterministic authorization tests cover the separated listing/control domains, every key form, malformed-Basic fallback, anti-bypass precedence, Anthropic routing without credential forwarding, 401 challenge, atomic catalog-driven key rotation, and qualification credential isolation. The private live harness gates all three forms, requires unauthenticated inference and management to return 401, and never sends the key to the protected service or direct engine; GMKtek execution remains required. | +| Logs and bounded streaming logs | Native, separate bounded router event ring at authenticated `GET /router/logs?since=` with the same replay/resume/SSE contract as engine logs | Deterministic tests prove admission/completion events, privacy-safe payloads, bounded ring behavior, management authorization, and multi-frame bounded qualification capture. The live harness requires an authenticated `management_loaded` event; GMKtek execution remains required. | +| Prometheus and activity/performance metrics | Native `/metrics` exposes bounded router admission, active/reserved/queued requests, queue wait, active identity, activation time, failure, cancellation, eviction, normal-terminal-stream, last-TTFT, last-duration, response-byte, and proxy-byte-rate signals; router-cancelled streams are not credited as normal terminal completions. Authenticated `/api/performance` and `/router/performance` retain at most one hour of owned engine process-tree RAM/VRAM samples, support strict RFC3339 `after`, and preserve unavailable/source markers without fabricating adapter-wide sensors. | Deterministic tests cover sampler lifecycle generations, one-hour eviction, filtering, auth, disabled 503, failure isolation, and path/PID omission. `benchmarks/swap/qualify_native_router.py` requires an authenticated positive available periodic RAM/VRAM sample with source labels and no PID/model/path fields, plus router metrics, and collects private direct/warm/cold/alternating first-byte, duration, streamed-usage-derived completion-token-rate, process, and memory evidence. It still requires an approved Linux GMKtek EVO-X2 execution. | +| Inflight cancellation API | Native router issues or accepts opaque `X-FT-Request-ID` values, atomically reserves them before admission, lists IDs throughout queued/connecting/active ownership, removes disconnected waiters from the admission queue, and provides `POST /router/requests/{id}/cancel` | Deterministic tests prove duplicate IDs cannot create a second admission or upstream request; operator or disconnect cancellation removes queued work before a later swap; connecting cancellation closes eventual sockets and releases leases; failure paths release ownership; and active cancellation closes the socket and is not credited as normal completion. Cancellation telemetry is counted once per accepted cancellation. Same-instance real-engine terminal-abort proof remains required. | +| Parameter filters and configuration hooks | Native `use_model_name`, `drop_fields`, `set_fields`, and `set_fields_by_id` follow the pinned outbound-model/strip/global/by-ID order. The optional override changes the upstream JSON `model` without changing requested routing identity or by-ID selection. Hard values override clients; `?` values fill only absent paths; explicit null/zero/false remain present. By-ID tables automatically create collision-checked aliases. The top-level `model` field is otherwise protected. Policy comes from the exact admitted profile; JSON direct-upstream requests share it, while non-JSON and empty policies remain byte-exact. | Deterministic parser, transform, HTTP, cold-loading, alias-collision, protected-field, active-reload, direct-upstream, and qualification-canary tests cover the applicable data-only behavior. The private harness requires an alias response to report the configured upstream name with unchanged residency; GMKtek execution remains required. Lifecycle shell hooks are intentionally inapplicable because native `ServeManager` owns argument-vector launch, accounting, drain, rollback, and cleanup without a shell. | +| Configuration watch/reload | Native authenticated `POST /router/reload` and default cross-platform local catalog polling re-parse and atomically validate the catalog. Watch status and sanitized results are observable. | Deterministic tests cover manual valid replacement, invalid-file rejection, active-profile scheduling/effective-lifecycle redefinition refusal, watcher valid replacement and watcher rejection. Real-engine reload evidence remains required. | +| UI, hardware, captures, MCP, Tailcat | Native dependency-free `/ui/` management shell, authenticated `/router/hardware`, bounded performance history, restart-durable body-free activity/stat APIs, hashed session grouping, and opt-in memory-bounded redacted captures fetched by UI only on selection. Byte fields pair availability/source markers so unavailable probes cannot masquerade as zero. MCP and Tailcat are **Deferred** product expansions. | Deterministic tests prove API protection, performance bounds/filtering/privacy, fsynced restart recovery/compaction/failure health, hashed session identity, capture-disabled default, credential redaction, binary fidelity, overflow/cancellation refusal, eviction, aggregation, UI secret isolation, AMD SMI summing, and explicit unavailable memory. The private live gate requires positive measured RAM and VRAM; raw captures remain private. | +| Embedding, rerank, image, speech, transcription, ComfyUI, SDAPI routes | Inapplicable today where FreeToken has no matching server route | Document absent FreeToken backend capability and reject safely. Do not mimic endpoint success | +| Accounting, drain/abort barrier, rollback | Native automatic routing delegates every stop/switch to `ServeManager`; readiness and launch failures retain its recovery result, including through `POST /router/load` | Deterministic routing and management-API tests prove recovery evidence and restored exact identity. The private native harness now requires a failed disposable real-model switch, rollback launch, new durable outbox receipt, failure-counter increment, and restored completion; current-branch Linux and GMKtek execution remain required. | + +## Native real-process gate + +`tests/daemon/test_real_process_recovery.py` now includes a Linux-only native +router test that starts a disposable HTTP child through `ServeManager`, waits +for real `/health` readiness, routes an SSE request through the daemon, then +stops the child and verifies pidfile cleanup. A second Linux-only test persists +a live disposable child as prior-daemon state, re-adopts it into a new manager, +binds the exact catalog profile in a new routing coordinator, routes SSE without +calling the spawn function, and verifies cleanup by the new owner. It is skipped +on Windows. The complete 352-test daemon suite, including these tests, passed +with no skips in GitHub-hosted Ubuntu run `34941311939` for commit `39c3aaab`. +This closes the current-branch +disposable Linux process gate only; it does not qualify the current FreeToken +engine, GPU models, or the GMKtek maintenance matrix. + +Git also produced clean synthetic combined tree +`74a4f3b1649442d9d8c24576751d29f30e218d04` from swap head `39c3aaab` and +draft AMD compatibility head `c0534c6f`. In an isolated Windows export, 377 +daemon/privacy/benchmark-contract/reproducibility tests passed with 7 expected +Linux skips, followed by 21/21 grouped-output, SSM, and GGUF configuration +tests in a disposable CPU torch environment. This is source-tree compatibility +evidence only; it is not current-engine, GPU-model, or restoration proof. + +## Architecture gate + +The target is one FreeToken-owned router and lifecycle supervisor. It must not +delegate automatic routing to llama-swap while retaining safety only in the +manual daemon. The existing llama-swap integration remains a compatibility and +comparison reference until native request routing reaches the acceptance gates. + +## Remaining acceptance sequence + +1. In an approved GMKtek EVO-X2 maintenance window, run the private native + qualification harness through direct, warm, cold, alternating, cancellation, + same-model concurrency, conflicting-model drain, failed-switch recovery, + daemon re-adoption, reload-conflict, persistent-capacity, TTL, unauthenticated + rejection, and authenticated management/metrics/router-log gates. Supply the + exact operating-system hostname explicitly; the harness fails before artifacts + or service inspection if it does not match. +2. Restore and health-check the protected workload, retain raw evidence privately, + and publish only sanitized aggregate observations in the final audit. +3. Re-run deterministic, hosted Linux, and combined-tree compatibility suites at + the final PR head and keep the PR draft until all applicable evidence is linked. + +Every row moves to Native only after deterministic tests and relevant live +evidence are linked here. No endpoint name alone establishes parity. + +The privacy-safe native live acceptance matrix is maintained in +[native qualification runbook](freetoken-swap-native-qualification.md). diff --git a/docs/freetoken-swap-research.md b/docs/freetoken-swap-research.md new file mode 100644 index 0000000000..946e1fa594 --- /dev/null +++ b/docs/freetoken-swap-research.md @@ -0,0 +1,137 @@ +# FreeToken swap compatibility and model repair + +## Findings + +Automatic model swapping is feasible without modifying llama.cpp or rewriting the llama-swap router. llama-swap already accepts OpenAI-compatible inference servers. FreeToken needs a compatible readiness contract, qualified model loaders, explicit resource limits, and a documented choice of process supervisor. The initial native daemon catalog was only a manual control plane. The current FreeToken branch adds a native model-ID router, lease-based admission, byte-preserving proxying, guarded upstream passthrough, cancellation, idle eviction, atomic catalog reload, and a deliberately explicit single-resident-model capacity policy. This is deterministic implementation evidence, not yet real-engine parity proof.[1][2] + +Two independent defect classes explain the unsuccessful initial attempts. First, the supervisor could report success incorrectly or time out before the model's readiness budget expired. Second, the AMD model loader and tokenizer did not support the exact resident GGUF layouts. A model catalog cannot repair a tensor format mismatch, and an HTTP listener cannot establish backend readiness. These defects need separate acceptance gates. + +The implementation target remains FreeToken GitHub. The reference source is mostlygeek/llama-swap, a separate MIT-licensed project, not a component in the llama.cpp repository. The inspected reference revision is `41ec321b6216d838488b2a7d936274ed227c0c5e`. No third-party source is copied into this implementation and no upstream llama.cpp change is proposed.[1] + +## How swapping should work + +The intended client contract is a stable proxy URL. A request names an allowlisted model alias in its JSON `model` field. The supervisor selects that configuration, starts the corresponding backend when necessary, waits for it to accept work, and forwards the request. Subsequent requests reuse the resident backend. A request for a different model causes the routing policy to decide which process must leave memory.[1] + +llama-swap's default routing is one model at a time. Its group router can explicitly make a group exclusive and require swapping among its members. Concurrent groups and the matrix router are additional capabilities, not evidence that a particular shared-memory machine has capacity to run multiple models safely. For initial GMKtek EVO-X2 qualification, an exclusive single-model policy is the appropriate starting point.[3] + +There are three distinct time budgets. The readiness timeout limits how long a new backend may take to become usable. The idle TTL determines when an unused backend may be evicted. The unload timeout limits graceful process termination after eviction has begun. Increasing one does not increase the others. A short TTL can cause expensive repeated cold loads, so the example retains one model indefinitely and shows a five-minute idle TTL for the other.[4] + +The assigned backend port must match the proxy target. The example passes `${PORT}` to FreeToken and explicitly proxies to `127.0.0.1:${PORT}`. It passes `${MODEL_ID}` as FreeToken's served model name. Where a backend expects a different name, the native profile's validated `use_model_name` rewrites only the outbound JSON model before strip/global/by-ID filters while preserving the requested routing identity. Catalog entries must point to already available, compatible model artifacts. The example does not download or qualify weights.[2] + +Streaming is part of the acceptance contract. A proxy that buffers all generated output before replying is not equivalent to an SSE-capable model router. Tests must also cover client cancellation, admission while a model changes, concurrent requests for one model, and conflicting requests for two models. An apparently healthy proxy can still have a broken inference path; liveness and successful routing are separate measurements. + +## Readiness incompatibility and repair + +llama-swap polls a configured endpoint and accepts HTTP 200 as readiness. FreeToken's existing `/health` deliberately returns a diagnostic JSON document even while loading or in an error state. Consequently, pointing llama-swap at FreeToken's default `/health` can release requests before the backend is usable.[2][5] + +The added `/ready` endpoint preserves `/health` compatibility. It returns HTTP 200 only when the health document reports `status=ok` and `maintenance=serving`. Loading, failure, and maintenance produce HTTP 503. The example sets `checkEndpoint: /ready`. `/v1/models` is not a substitute for this gate because listing a configured model does not prove that its weights and execution backend are ready. + +The manual daemon profile path also had a stale-cache hazard. Its general health probe caches by port, but successive engines can reuse a port. A readiness check now bypasses that cache and rechecks the managed PID after the HTTP request. It also uses the launch port captured for the transaction instead of resolving a potentially changed current port afterward. This reduces false success during replacement, but it is not a request lease or a complete proof against PID reuse and unrelated port ownership. + +Readiness failure returns HTTP 503 from profile operations, and the CLI returns nonzero for unsuccessful readiness responses, including responses from older servers that still use HTTP 200. The default profile transport budget is 1920 seconds, covering replacement and recovery readiness windows plus lifecycle overhead. A user-specified timeout still takes precedence. Native `switch-profile` now attempts previous-engine recovery after readiness failure, guarded by a one-use lifecycle epoch so newer operator actions win. Initial starts without a previous engine remain managed for inspection. Recovery retains the accounting safeguards and reports launch and readiness independently. + +The catalog validation also rejects the `--model-path` alias and abbreviations of supervisor-owned model and port options. FreeToken uses argparse, whose default abbreviation behavior makes checking only the exact strings `--model` and `--port` insufficient. Validation remains torch-free and accepts argument vectors rather than catalog-supplied shell commands.[6] + +## Model loading defects + +The GGUF label Q4_K_M describes a quantization recipe, not a guarantee that every tensor is Q4_K. GGUF tensor descriptors carry their individual types. Qwen hybrid attention has independent QKV, gate, and output projections. Their packed storage cannot be concatenated blindly when the quantization block formats differ.[7][8] + +The exact Qwen3.6 27B candidate contains Q6_K GDN QKV weights alongside Q4_K gate weights. The old loader attempted a packed concatenation and encountered incompatible row widths. The repair uses separate native GGUF linear operators, then combines their floating-point activations in the existing GDN computation. It does not expand the complete model to full precision. + +GDN output ordering requires another repair. Quantized output blocks can span more than one value head. Moving a fraction of a block as though it were an independent head also moves or misassociates shared quantization metadata. The repaired dense path keeps the packed output weights intact and applies the inverse head-group permutation to activations before the output projection. A permutation regression test is necessary in addition to shape checks. + +The first exact-file Qwen3.6 CPU/meta contract passed after applying the earlier candidate repair to an isolated AMD checkout. The same candidate did not pass Qwen3.8: its first GDN gate was Q8_0, while the candidate assumed Q4_K. This is direct evidence that a repair hardcoded for one quantization recipe should not be advertised as general Qwen support. The next iteration derives projection types from each tensor's descriptor and keeps the legacy MoE path separate. + +Dense `qwen35` also needs a tokenizer converter mapping. The candidate maps it to the compatible `qwen3` converter key rather than allowing a `qwen35` dictionary lookup to fail. A real tokenizer round-trip and chat-template test remain necessary because a successful architecture lookup alone does not establish special-token behavior. + +Dense checkpoints contain no routed experts. Their expert-only loading phase must be a no-op, while the separate MoE expert-cache contract must remain intact. Initial dense model qualification uses the fused/non-MoE execution selection. This should not be generalized to MoE checkpoints, which need their actual expert-residency configuration. + +## Architecture decision + +There are two valid operating modes, with different guarantees. In direct integration mode, a pinned llama-swap binary owns FreeToken processes and supplies automatic routing, streaming proxying, idle eviction, and its existing model-management interfaces. FreeToken supplies `/ready` and inference. The YAML example describes this mode. Do not simultaneously give those processes to `ft daemon`. + +In native daemon mode, FreeToken owns process groups, durable state, final accounting receipts, automatic inference routing, stream-aware admission, idle eviction, guarded passthrough, and explicit router cancellation. The router serializes unsafe replacements through the existing manager instead of double-supervising an engine. It presently supports one resident engine, so persistent groups reserve that slot and multi-resident matrix solving remains unimplemented. Calling this mode complete llama-swap parity would still overstate the evidence until real-engine, timing, and broader endpoint tests pass. + +The native mode now uses one FreeToken-owned routing and lifecycle layer with explicit leases, durable receipt semantics, rollback, and exact re-adoption. It is not a separate catalog layered over another supervisor. Direct integration with the pinned llama-swap binary remains a distinct comparison mode only: never run it against a process owned by `ft daemon`, and do not use its historical results as evidence for the current native implementation. The direct example does not promise the daemon's durable accounting outbox. + +Model support and runtime support must be pinned separately. This swap branch is based on the FreeToken fork's main branch, whereas the repaired Qwen loader targets its AMD branch. A model repair PR must target the AMD base rather than silently importing unrelated runtime and benchmark history into the control-plane PR. Combining branches for qualification is a local integration step, not proof that upstream FreeToken already supports the candidate. + +## Qualification and operating limits + +Validation proceeds from cheapest and safest checks to expensive serving tests. First parse the catalog and confirm the exact model artifact and architecture. Next validate all loader keys, shapes, and dtypes against a meta-device model. Then test tokenizer behavior and the relevant tensor-order transformations. Only after these gates should an isolated GPU process be started. + +The initial GPU profile should use an explicit small sequence and token budget, such as 4096 tokens, with a bounded prefill size. FreeToken's relevant flag is `--max-seq-len-override`, not llama.cpp's `--ctx-size`. A large automatically derived cache can turn a model compatibility check into an uncontrolled capacity experiment. Successful short-context qualification does not establish 64K support. + +At the current safety check, GMKtek EVO-X2 had an active llama.cpp process and approximately 23 GiB of available system memory. That process was left untouched. System `MemAvailable`, GPU-visible UMA, and current accelerator allocations are distinct measurements. A historical GPU-memory value cannot authorize a new load, and model file size alone cannot establish fit after runtime overhead, KV cache, staging, and other workloads are included. + +The next real-model gate is one deterministic completion, repeated after a cold reload, with model identity and raw output retained privately. After that, test A-to-B-to-A routing, ordinary and streamed responses, cancellation, same-model concurrency, conflicting-model admission, idle eviction, forced backend failure, and shutdown. Record peak memory, swap activity, load time, time to first token, and final process cleanup. Stop a failed quality or memory-safety trial without promoting it to production. + +CPU contract tests and mocked HTTP tests are valuable regression evidence, but they are not proof of GPU numerical correctness, backend graph readiness, or live swap throughput. Any release checklist must retain those distinctions. The subsequently approved maintenance-window results below supersede the initial restriction on stopping the protected service. No permanent production activation was performed. + +## Completed live iterations + +The repaired Qwen3.6 and Qwen3.8 files both passed their exact CPU/meta tensor contracts and tokenizer text round-trips. Twenty-one model tests passed, including a matrix of independently typed QKV/gate projections. The daemon suite passed 52 tests with two platform skips; the AMD benchmark/privacy suite passed 27 tests. + +The first live startup problem was a qualification-command error: `python -m freetoken` is the legacy direct-server entrypoint and rejects the `serve` subcommand. The corrected invocation is `python -m freetoken.cli serve`. Another attempt stalled behind an abandoned shared PyTorch extension-cache lock. The solution was a private `TORCH_EXTENSIONS_DIR` and native-kernel preflight before stopping the protected service. The shared cache was left untouched. + +Two complete A-to-B-to-A passes then succeeded through the pinned, unmodified llama-swap binary. The extended pass returned the deterministic answer `4` for Qwen3.6, Qwen3.8, then Qwen3.6 in 35.99, 44.17, and 33.17 seconds, including loading or switching. It also passed two concurrent requests for the same model, concurrent requests for different models, explicit streamed usage blocks, and five-second idle eviction. These are bounded functional controls, not broad quality benchmarks or isolated decode-throughput measurements. + +Both ordinary and SSE responses were checked, including `[DONE]`. Adding `stream_options: {"include_usage": true}` eliminated the missing-usage metrics issue without changing llama-swap. The original stream was valid JSON but lacked the usage block that its metrics parser requires. The final proxy/backend log contained no recorded traceback or streaming-metrics error. + +Every maintenance trial restored the protected service and verified a deterministic completion. After the final pass, the service manager reported it active and running, and the test listeners were closed. Raw artifacts remain private under the logical sets `freetoken-swap-live-20260910-d` and `freetoken-swap-live-20260910-e`. FreeToken PR #1 contains the control-plane integration and PR #2 contains the AMD model repair and anonymization. + +Remaining limits are explicit: no claim of long-context qualification, comprehensive tool-calling quality, direct-supervisor rollback, or long-duration reliability is made. The direct integration does not acquire the daemon's durable accounting guarantees. Semaphore-cleanup warnings remain a follow-up investigation even though the final worker-process and port cleanup checks passed. The additional bounded cancellation and native real-model recovery results below supersede those earlier unqualified gates. + +### Approved live cancellation and native recovery + +The subsequent approved window passed the cancellation harness using the same pinned llama-swap binary and repaired runtime. Qwen3.6, Qwen3.8, and Qwen3.6 returned `4` in 35.01, 36.95, and 33.29 seconds, including loading/switching. The cancellation request produced first content after 0.368 seconds. After disconnect, the same engine instance reached zero active requests in an observed 0.254 seconds, while completed requests stayed at one. No engine restart or natural completion was accepted as cancellation. Post-disconnect A-to-B-to-A streaming, same-model concurrency, conflicting-model concurrency, and idle eviction passed again. + +The native daemon was then tested against the real Qwen3.6 runtime through its profile API. A private invalid GGUF fixture caused the real loader to raise `GGUF magic invalid`; the switch returned HTTP 503. Automatic rollback restored the previous Qwen3.6 model, reached readiness, and returned `4` in a streamed completion. Accounting preserved the previous engine's sealed receipt (27 prompt tokens, 2 completion tokens, complete drain). The failed loader's separate crash receipt was marked degraded with unknown token totals, rather than inventing zero usage. + +Both phases restored the original llama.cpp service and verified generation. Final read-only checks found no test listeners or remaining FreeToken multiprocessing workers. Private logical artifact sets are `freetoken-swap-live-20260910-f` and `freetoken-native-recovery-20260910-a`. These results qualify the documented bounded workflows, not every model, failure mode, context size, or extended workload. + +### Native recovery regression suite + +The additional Linux real-process suite passes both normal SIGTERM and SIGTERM-resistant child cases on GMKtek EVO-X2, without loading models or interrupting the protected workload. It uses isolated loopback HTTP test children and verifies previous-engine readiness recovery, restored arguments and pidfile, two durable replacement receipts, process-group worker cleanup, and a closed listening port. This strengthens OS lifecycle evidence but is not GPU model-failure qualification. + +The current native-router Windows daemon suite passes 313 tests with 7 expected Linux-only skips. Coverage exercises replacement launch failure, recovery launch failure, readiness error and timeout, recovery readiness failure, accounting failure preservation, replacement exit and persisted-state cleanup, one-use recovery tickets, automatic canonical, alternate, pin/warm virtual, runtime/startup-profile-pinned model-ID routing, singleton alias-canonicalized startup preload through native lifecycle, profile-to-selector composition, disabled and shadowing pins, explicit upstream model-name rewrite with preserved by-ID routing identity, model display/JSON metadata propagation and collision precedence, global/per-profile upstream socket timeouts, profile and selector rewrite/filter ordering, strategy-specific listing status, safe configurable readiness paths and manager-owned loopback proxy prefixes, HTTP-success readiness without body retention, remote/explicit-port/query/fragment/traversal target rejection at both parser and connector boundaries, active-target reload refusal, direct-upstream longest-prefix profile rewrites, atomic profile selection/listing snapshots and reload clearing, hidden-profile list policy, exact `/models` public-list alias and separate profile-control authentication, atomic public pre-ownership/unloaded/activating/resident/stale model status without path disclosure, global/per-profile concurrency reservations and immediate rejection, concurrent cold dynamic-target sharing, global/per-profile cold-load feedback after admission with queue reasoning SSE, warm and disabled-path preservation, in-band activation errors, explicit cancellation and disconnect cleanup, sanitized side-effect-free browser preflight and authenticated model-list CORS, Bearer/Basic-password/`X-Api-Key` extraction and anti-bypass precedence with local credential termination, atomic readiness, disconnect-safe shared manual/routed lifecycle exclusion and rollback completion, coordinated HTTP and OS/lifespan daemon shutdown, drain-before-detach including preempted manual transactions, immediate shutdown admission closure under lifecycle-pool contention, queued/connecting/active cancellation ownership, guarded longest-prefix passthrough with escaped path/query preservation, authenticated stateless response-resource compatibility without model admission, race-safe atomic reload and dynamic-port binding, strict filters and namespaced-ID validation, exact explicit/dynamic/omitted-default-port re-adoption, capacity protection, invalidation by newer lifecycle operations, exact-origin qualification credentials, unauthenticated control/inference rejection, authenticated alias/selector/profile/readiness/upstream-model/upstream-timeout/model-metadata/metrics/router-log evidence gates, and privacy-safe exact-host maintenance gating before side effects. These are controlled CPU and loopback-HTTP tests, not new real-model measurements. + +The latest complete daemon suite passed on GitHub-hosted Ubuntu at commit +`a3fc0ddbf6c929164aa925d50c2270c1ca90c318`: run `34933392153` reported +320 passed with no failures, errors, or skips. It executes the actual disposable +Linux child, process-group escalation, readiness rollback, exact re-adoption, +dynamic-port reactivation, routed SSE, cleanup cases skipped on Windows, and +the deterministic pin/warm selector, runtime routing-profile, upstream model-name, +model display/metadata, and safe readiness/proxy-target gates. This run +establishes current-branch Linux process behavior only; current-engine and +GMKtek GPU-model qualification remain separate gates. + +The pinned optional cold-load feedback behavior is now implemented through an +atomic reservation callback after concurrency admission. Strictly streaming chat +requests for a nonresident target receive queue/load progress as reasoning SSE; +warm and disabled requests retain the ordinary proxy, immediate concurrency +rejection remains HTTP 429 JSON, and post-commit activation or connection failure +is framed in-band before `[DONE]`. Deterministic tests cover explicit cancellation, +client disconnect, reservation and lease cleanup, and byte-preserving bypasses. +The private native harness now requires loading frames during cold-B and +alternating-A trials and rejects them during warm-A, but that gate has not yet +been executed on the current branch during an approved GMKtek maintenance window. + +## Privacy and publication + +Public material identifies the primary test computer as GMKtek EVO-X2. Personal home paths use `/home/operator` or equivalent placeholders, and LAN addresses use documentation-only example addresses. Raw logs remain private because they may contain personal paths, hostnames, device identifiers, and request content. Redaction must not make an example address appear to be a working deployment address. + +Privacy review preserves license notices, upstream authorship, and repository URLs needed for provenance. Working-tree sanitation does not remove identifiers from historical Git objects, forks, cached PR revisions, or previously generated PDFs. History rewriting and regenerated publication artifacts require a separate, verified pass; they must not be reported as completed merely because current Markdown has been sanitized. + +## Sources + +Sources were inspected on 2026-09-10. Local implementation and test observations above refer to the candidate branches, not to claims made by upstream maintainers. + +1. mostlygeek/llama-swap contributors. [Repository and feature overview](https://github.com/mostlygeek/llama-swap/tree/41ec321b6216d838488b2a7d936274ed227c0c5e), pinned revision; MIT license in `LICENSE.md`. +2. mostlygeek/llama-swap contributors. [Writing the cmd for a model](https://github.com/mostlygeek/llama-swap/blob/41ec321b6216d838488b2a7d936274ed227c0c5e/docs/kb/guides/model-runtime/writing-cmd.md), updated 2026-08-25. Port assignment, readiness, and model-name rewriting. +3. mostlygeek/llama-swap contributors. [Running several models at once with groups and matrix](https://github.com/mostlygeek/llama-swap/blob/41ec321b6216d838488b2a7d936274ed227c0c5e/docs/kb/guides/routing/groups-and-matrix.md), updated 2026-08-25. +4. mostlygeek/llama-swap contributors. [Automatic model unloading with ttl](https://github.com/mostlygeek/llama-swap/blob/41ec321b6216d838488b2a7d936274ed227c0c5e/docs/kb/guides/model-runtime/ttl-and-unloading.md), updated 2026-08-25. +5. FreeToken contributors. [Control API](../python/freetoken/server/control_api.py), `build_health` and `register_control_routes` in the candidate checkout. +6. FreeToken contributors. [Server argument parser](../python/freetoken/server/args.py), model aliases and parser construction in the candidate checkout. +7. ggml-org/llama.cpp contributors. [Qwen35 model implementation](https://github.com/ggml-org/llama.cpp/blob/master/src/models/qwen35.cpp). Independent attention/gate model structure; moving upstream reference, not a candidate qualification result. +8. ggml-org/llama.cpp contributors. [Quantization recipes discussion](https://github.com/ggml-org/llama.cpp/discussions/20522). Primary maintainer discussion of per-tensor quantization choices; not evidence that every file using the same recipe has identical layouts. diff --git a/docs/freetoken-swap-source-inventory.md b/docs/freetoken-swap-source-inventory.md new file mode 100644 index 0000000000..c5986b59a5 --- /dev/null +++ b/docs/freetoken-swap-source-inventory.md @@ -0,0 +1,161 @@ +# freetoken-swap pinned-source inventory + +This document decomposes the parity contract in +[`freetoken-swap-parity-matrix.md`](freetoken-swap-parity-matrix.md). The source +of truth is the read-only `mostlygeek/llama-swap` commit +`41ec321b6216d838488b2a7d936274ed227c0c5e`. Source was inspected with +`git show` and `git ls-tree`; no reference code is vendored. The pinned +`LICENSE.md` is the MIT License, copyright 2024 Benson Wong. + +Classifications describe behavior, not matching names: + +- **Native** — implemented in FreeToken and covered by deterministic tests. +- **Equivalent** — a different native contract provides the applicable + behavior and is covered by deterministic tests. +- **Missing** — applicable behavior that is not implemented yet. A missing + live-only proof is called out separately from missing implementation. +- **Deferred** — potentially applicable expansion that is not part of the + current one-engine product contract. It remains an open difference, not + parity. +- **Inapplicable** — impossible or misleading under a stated FreeToken backend + or one-engine architectural constraint. A future capability change reopens + the item. + +The current-engine and GMKtek EVO-X2 maintenance qualification remains pending; +therefore **Native** never implies that the live gate is complete. + +## Configuration schema + +### Global fields + +Pinned sources: `internal/config/config.go` (`Config`, `GroupConfig`, +`HookOnStartup`, `ProfileConfig`, `RoutingConfig`), +`internal/config/performance.go`, `internal/config/upstream.go`, +`internal/config/peer.go`, and `internal/config/tailcat.go`. + +| Pinned field or block | Classification | FreeToken behavior and evidence location | +| --- | --- | --- | +| `models`, `apiKeys`, `globalTTL`, `unloadTimeout`, `globalConcurrencyLimit`, `includeAliasesInList`, `sendLoadingState` | **Native** | Allowlisted TOML equivalents in `python/freetoken/daemon/catalog.py`; routing/auth/list/loading tests in `tests/daemon/test_catalog.py` and `test_router.py`. | +| `startPort` | **Equivalent** | Each profile accepts an explicit port; `port = 0` asks the kernel for a loopback port at activation. Allocation and reactivation are tested. | +| `routing.scheduler.use=fifo` and FIFO priorities | **Native** | One priority-aware FIFO coordinator in `python/freetoken/daemon/router.py`; unsupported schedulers fail validation. | +| group `members`, `swap`, `exclusive`, `persistent` | **Native applicable subset** | Membership and persistent protection are native. Configuration that requests coexistence outside the sole resident slot is rejected rather than weakened. | +| matrix `vars`, `sets`, `evict_costs` | **Inapplicable today** | `ServeManager` owns exactly one child, so there is no co-resident set or victim choice to solve. This reopens if FreeToken gains multi-engine ownership. | +| startup `hooks.on_startup.preload` and `profile` | **Native applicable subset** | `router.preload_model` allows one concrete model/alias and `router.startup_routing_profile` selects a validated pin map. Multiple preloads/selectors are rejected under one-resident capacity. | +| runtime `profiles` descriptions and pin maps, including disabled pins | **Native** | `[profiles..pins]` and authenticated activation API; parser, routing, reload, and lifespan tests. | +| `selectors` (`pin`, `warm`, `spillover`) | **Native applicable subset** | `pin` and `warm` are native. `spillover` is **inapplicable today** because it requires simultaneous reservations across local residents or peers. | +| global `macros` | **Inapplicable by safety contract** | FreeToken accepts argument vectors and explicit typed fields; arbitrary command/proxy/environment interpolation is rejected to prevent shell and target injection. | +| `peers` and peer credentials/filters/timeouts | **Deferred** | The current contract is one local FreeToken-owned engine. No distributed peer transport is claimed. | +| `upstream.ignorePaths` | **Native safe subset** | `router.upstream_no_activation_suffixes` defaults to the pinned static extensions, returns 409 before reservation/activation/upstream I/O while the exact local model is unloaded, and proxies normally when resident. A bounded validated suffix list replaces arbitrary regex to avoid a regex execution surface. | +| `healthCheckTimeout` | **Native** | Per-profile `ready_timeout_s` bounds readiness; the checked path is configurable. | +| request-log level/time/stdio fields | **Equivalent** | Native daemon logging and bounded rings have their own process-level controls; these are not hot catalog policy. Router events deliberately omit bodies, headers, query strings, and secrets. | +| `metricsMaxInMemory` | **Equivalent** | Native router logs and metric state are bounded; Prometheus counters are aggregate rather than a queryable in-memory activity table. | +| `captureBuffer` | **Native safe equivalent** | `router.capture_buffer_mb` is opt-in and defaults to zero. Captures are credential-redacted, serialized-byte-budgeted, per-response capped, binary-safe, memory-only, and retrieved by activity ID. | +| `store.path` | **Native safe equivalent** | The daemon-owned state directory contains lifecycle/accounting state and bounded body-free `activity.jsonl`. Rows are fsynced, streamed on recovery, strictly validated, and atomically compacted; persistence health is exposed without its path. Captures remain memory-only. | +| `ui.activity.session_id` | **Native privacy-preserving equivalent** | Validated non-credential header names select the first nonempty value, but only a stable truncated SHA-256 label is stored and shown. Matching is case-insensitive and raw identifiers/general headers are not persisted. | +| `performance.disabled`, `performance.every` | **Native privacy-preserving equivalent** | Validated `router.performance_disabled` and `performance_every_s` (5–3600 seconds) control an app-owned sampler retaining at most one hour in memory. It samples only the owned engine process-tree RAM/VRAM probe. | +| `tailcat` | **Deferred** | No Tailcat network dependency or remote-listener product contract exists. Local auth and route allowlisting do not claim Tailcat interoperability. | + +### Per-model fields + +Pinned source: `internal/config/model_config.go` (`ModelConfig`, +`TimeoutsConfig`, `CompatConfig`, `ModelCapConfig`) and +`internal/config/filters.go` (`Filters`). + +| Pinned field | Classification | FreeToken behavior and evidence location | +| --- | --- | --- | +| `cmd` | **Equivalent, safer** | `model` plus `args` constructs an allowlisted `ft serve` argument vector without a shell. Unknown daemon-owned options are rejected. | +| `cmdStop` | **Inapplicable by ownership contract** | `ServeManager` performs drain/abort, exact-identity signalling, process-group cleanup, accounting, and rollback; arbitrary stop commands would create a second lifecycle authority. | +| `env` | **Inapplicable by safety contract** | Per-profile environment injection is rejected. The daemon inherits its controlled service environment. | +| `proxy`, `checkEndpoint` | **Native applicable subset** | Exact plain-HTTP loopback `${PORT}` target with optional fixed path prefix and a safe readiness path. Remote targets, credentials, fragments, traversal, and ambiguous port templates fail closed. | +| `aliases`, `unlisted`, `useModelName` | **Native** | Collision-safe canonicalization/listing and outbound JSON model rewrite with client routing identity retained. | +| `ttl`, `unloadTimeout` | **Native** | Per-profile override plus global default, idle-only eviction, and manager-owned graceful stop. | +| `name`, `description`, `metadata` | **Native** | JSON-compatible metadata with router-owned identity/capability precedence; no local model paths or args leak through public listings. | +| `concurrencyLimit` | **Native** | Canonical and alternate IDs share one reservation cap; admission rejects before lifecycle/upstream work. | +| `filters.stripParams`, `setParams`, `setParamsByID` | **Native** | `drop_fields`, hard/soft `set_fields`, and `set_fields_by_id` preserve the pinned strip/global/by-ID order and protect `model`. Nested safe JSON paths extend the flat pinned behavior. | +| per-model `macros` | **Inapplicable by safety contract** | Explicit typed fields replace arbitrary interpolation. | +| `sendLoadingState` | **Native** | Nullable per-profile override of the global setting for admitted cold streaming chat requests. | +| timeout `connect`, `responseHeader` | **Equivalent** | One per-profile/global upstream socket deadline bounds connect and response reads for ordinary and SSE requests. It is deliberately simpler than independent phase timers. | +| timeout `keepalive`, `idleConn` | **Inapplicable today** | Native proxy calls use fresh manager-owned loopback HTTP connections, not a reusable idle pool. | +| timeout `tlsHandshake` | **Inapplicable today** | Valid proxy targets are plain HTTP loopback only. | +| timeout `expectContinue` | **Inapplicable today** | The supported small JSON text routes are forwarded over a fresh local connection without an Expect/Continue policy surface. | +| `compat.ignoreWebsockets` | **Inapplicable today** | Neither the current FreeToken engine nor the stable router registers a websocket inference route. | +| capabilities `in`, `out`, `tools`, `context` | **Native declarative subset** | Text/tool/context listing metadata is native and does not enable inference features. Unsupported image/audio/video declarations fail closed. | +| capability `reranker` | **Inapplicable today** | FreeToken has no rerank backend route, so advertising it is rejected. | +| copied `healthCheckTimeout` | **Native** | Resolved directly from each profile's `ready_timeout_s`. | + +## HTTP and management routes + +Pinned route source: `internal/server/server.go` (`modelPostJSONRoutes`, +`modelPostFormRoutes`, `modelGetRoutes`, `routes`, `ServeTailcatHTTP`). Native +registrations are in `python/freetoken/daemon/app.py`. + +| Pinned route family | Classification | Native behavior or boundary | +| --- | --- | --- | +| `POST /v1/chat/completions`, `/v1/completions`, `/v1/responses`, `/v1/messages`, `/v1/messages/count_tokens` | **Native** | Automatic model-ID inference, lifecycle acquisition, filtering, byte/SSE forwarding, cancellation and accounting share one coordinator. | +| `/v/*` versionless aliases, `/completion`, `/infill` | **Inapplicable today** | The current engine does not register these model-bearing aliases. Explicit profile-qualified upstream access remains available. | +| embeddings and rerank/reranking families | **Inapplicable today** | No matching FreeToken backend modality/route. | +| audio speech/voices/transcriptions and generic audio task route | **Inapplicable today** | No matching FreeToken backend modality/route. | +| image generations/edits, SDAPI, `/props`, ComfyUI | **Inapplicable today** | No matching FreeToken backend modality/route. | +| `GET /v1/models`, `/models` | **Native** | Authenticated canonical/optional-alias listing with atomic loaded state, display metadata and CORS. | +| `/logs` and `/logs/stream*` | **Equivalent** | Bounded engine and router streams use `/engine/logs` and `/router/logs?since=` with replay/resume behavior. | +| `/health`, `/wol-health` | **Native / inapplicable split** | `/health` is native liveness; `/ready` is stricter readiness. Wake-on-LAN health is not part of the local service contract. | +| root redirect, favicon, `/ui/` | **Equivalent** | Dependency-free local management UI is native; matching static asset names are not a parity requirement. | +| `/metrics` | **Native** | Authenticated Prometheus lifecycle, queue, cancellation, activation, timing, bytes and throughput signals. | +| `/unload`, `/running` | **Equivalent** | `/router/unload`, `/router/status`, and `/router/models`; one/all unload and configured/resident views are tested. | +| `/upstream/{model}/{path...}` | **Native** | Same admission/lifecycle lease, longest slash-namespaced ID, escaped suffix/query preservation, safe credential termination, and a pre-admission static-suffix guard that returns 409 rather than cold-loading. | +| `/api/models/unload*`, `/api/profiles`, `/api/profiles/active` | **Equivalent** | Native router management APIs implement the behavior under one-resident capacity. | +| `/api/inflight/{id}/cancel` | **Equivalent** | Opaque request reservation/list/cancel API covers queued, connecting, and active requests. | +| `/api/events` | **Equivalent** | Bounded resumable router event stream; route templates and lifecycle facts only. | +| `/api/metrics/activity`, `/api/metrics/stats` | **Native bounded equivalent** | Authenticated `/router/activity` supports newest-first bounded pagination and model filtering; `/router/activity/stats` reports counts, errors, cancellation, bytes, and average duration. Rows are body-free. | +| `/api/performance` | **Native privacy-preserving equivalent** | Authenticated pinned and native route aliases return a bounded one-hour `sys_stats` history with strict RFC3339 `after` filtering. Rows declare engine-process-tree scope and RAM/VRAM availability/source; `gpu_stats` stays empty rather than fabricating adapter-wide sensors. Disabled monitoring returns the pinned 503 `{enabled:false}` contract. | +| `/api/version` | **Equivalent** | `ft --version` and package version provide build identity; no duplicate router JSON endpoint is required for lifecycle behavior. | +| `/api/hardware` | **Native** | `/router/hardware` reports process-tree RAM and explicit available/source GPU memory. | +| `/api/captures/{id}` | **Native safe equivalent** | Authenticated opt-in retrieval by activity ID with pinned and custom credential-header redaction, Base64 bodies, one-MiB response cap, total serialized-byte budget, and no capture for cancellation/overflow. | +| `/api/mcp` | **Deferred** | The pinned endpoint exposes llama-swap's embedded docs/tools. FreeToken has no equivalent agent-tool product contract. | +| `/api/tailcat` and Tailcat listener restrictions | **Deferred** | No Tailcat listener or client protocol is claimed. | + +## Lifecycle and routing internals + +| Pinned source subsystem | Classification | FreeToken implementation | +| --- | --- | --- | +| `internal/router/{base,router,loading}.go` | **Native** | `RoutingCoordinator` owns admission, loading feedback, leases, readiness, eviction and routing snapshots. | +| `internal/router/group.go` and FIFO scheduler | **Native applicable subset** | Exclusive one-slot routing, persistent protection, priority and FIFO are deterministic-tested. | +| `internal/router/{matrix,matrix_solver}.go` | **Inapplicable today** | No multi-resident placement or victim set exists under one child. | +| `internal/router/peer.go` | **Deferred** | No remote peer transport in the local one-engine contract. | +| `internal/process/*` | **Native** | `ServeManager`, `osproc.py`, `pidfile.py`, and accounting state provide launch, exact identity, stop/reap/tree cleanup, rollback and re-adoption. | +| server profile/selector/filter middleware | **Native** | Order is routing profile, selector, alias, target filtering, admission and proxy; queued requests retain their admitted policy snapshot. | +| global and model concurrency middleware | **Native** | Reservations include queued/activating/active work and release once on every terminal path. | + +## Authentication, observability, UI, and persistence + +| Pinned behavior | Classification | FreeToken implementation or gap | +| --- | --- | --- | +| API-key middleware | **Native** | Case-insensitive Bearer, Basic password, and `X-Api-Key`; dedicated `X-FT-Token` control override; upstream credential stripping and rotation tests. | +| Inflight ownership/cancellation | **Native** | Opaque IDs reserve before admission and cancel queued, connecting, or active work. | +| Bounded logs and SSE resume | **Native** | Separate engine and privacy-safe router rings. | +| Prometheus metrics | **Native** | Lifecycle, queue, activation, cancellation, eviction, terminal stream, TTFT, duration and byte signals. | +| Bounded activity/performance stores | **Native** | Body-free inference activity survives restart in a bounded fsynced/compacted store. Performance history is intentionally memory-only and retains at most one hour, matching the pinned ring behavior. | +| Redacted bounded request/response captures | **Native** | Disabled by default; opt-in memory budget, sensitive-header redaction, binary-safe bodies, overflow/cancellation refusal, authenticated retrieval, and deterministic tests. Captures must never become public evidence artifacts. | +| Embedded management UI | **Native applicable subset** | Status/models/profiles/requests/logs/metrics/hardware/performance plus body-free activity and explicit-on-click capture views are local and dependency-free. The initial HTML embeds no operational data. | +| Hardware snapshot | **Native, extended** | Current process-tree RAM plus NVIDIA/AMD per-process VRAM, with unavailable distinct from zero. | +| Embedded documentation MCP | **Deferred** | No FreeToken MCP contract. This does not affect inference or lifecycle parity. | +| Tailcat remote access | **Deferred** | No FreeToken Tailcat contract. This does not imply generic remote access is safe. | + +## Tests and evidence classes + +Pinned source tests span `internal/**/*_test.go`, router/process/config/server +tests, and UI tests. Native deterministic coverage lives in `tests/daemon`: + +| Evidence class | Current state | +| --- | --- | +| Catalog validation, routing, HTTP/auth/SSE, filters, profiles/selectors, loading state, cancellation, TTL, reload, metrics/logs, process/accounting and startup hooks | **Native deterministic evidence present.** | +| Disposable actual-child process, process-group cleanup, re-adoption and routed SSE on Linux | **Native hosted-Linux evidence present** at the exact PR lineage recorded in the parity matrix. | +| Combined-tree/current engine compatibility | Historical synthetic-tree suites passed as recorded. Current source commit `56e0232` then passed 183 focused tests; the complete run reported 1947 passed, 81 skipped, and 21 parent-reproduced baseline failures. | +| GMKtek EVO-X2 direct/warm/cold/A-B-A/concurrency/cancellation/failure/rollback/re-adoption/reload/TTL/auth/metrics/logs/restoration | **Native live evidence present** for commit `56e0232`; the private result passed with clean cleanup and restoration. See the final qualification record. | +| Bounded activity/stat and opt-in capture APIs | **Native deterministic implementation and tests present.** Body-free rows survive app reconstruction; captures remain memory-only by policy. UI fetches captures only on explicit selection. | +| Periodic performance history | **Native deterministic implementation and tests present.** One-hour eviction, filtering, privacy, auth, disabled behavior, probe failure isolation, and sampler generation cleanup are covered. | + +## Open applicable implementation gaps + +No protocol-agnostic implementation gap is currently identified by this pinned +source inventory. The current-engine, GPU-model, protected-restoration, and unattended-startup gates are closed for the qualified GMKtek EVO-X2 configuration at commit `56e0232`. Unsupported backend modalities and explicitly deferred product expansions remain classified above rather than being presented as parity. diff --git a/docs/freetoken-swap.md b/docs/freetoken-swap.md new file mode 100644 index 0000000000..4c271e4fd4 --- /dev/null +++ b/docs/freetoken-swap.md @@ -0,0 +1,409 @@ +# freetoken-swap: native, safe model routing + +`freetoken-swap` is the native `ft daemon` routing mode. A client sends a +supported FreeToken OpenAI- or Anthropic-compatible request to the daemon's +stable URL with an allowlisted catalog alias in JSON `model`. The daemon alone +admits the request, starts or reuses one `ft serve` child, waits for its +generation-aware readiness, and proxies ordinary and SSE bytes unchanged. Its +lease stays active until the response closes, so another model cannot replace a +stream in flight. The same owner performs accounting, graceful drain/abort, +process-identity checks, cleanup, rollback, and re-adoption; **do not** put +llama-swap or another supervisor in front of the same FreeToken child. + +Legacy `/engine/start`, `/engine/stop`, `/engine/switch`, and profile variants +remain available only when the router does not own or admit work. They reserve +the same lifecycle barrier for their complete transaction, so a routed request +waits rather than racing a manual process operation. A manual stop may supersede +a manual operation blocked in readiness; its newer manager intent invalidates +stale rollback, and the older token cannot clear the stop's barrier. If the +manual HTTP client disconnects, the barrier remains held until the complete +executor-backed lifecycle transaction, including required rollback, terminates. +Daemon shutdown uses the same coordinator: it closes admission, wakes queued +requests with a stable shutdown error, drains active leases and lifecycle work, +then permanently stops the manager-owned child. A failed stop reopens admission; +a successful stop requests daemon exit even if the initiating client disconnects. +OS- and lifespan-triggered exit also quiesces this coordinator. The default +detach policy drains ownership and leaves the exact persisted child available +for re-adoption; `--stop-serve-on-exit` drains and permanently stops it instead. +Catalog reload binds profile lookup, priority ticketing, and dynamic-port +selection atomically. Reload is rejected while admission or lifecycle work is +pending, so an activating or queued request cannot change definitions mid-flight. + +The read-only, pinned llama-swap source remains a compatibility reference and +an optional separate deployment mode, not a runtime dependency. That direct +mode cannot gain this daemon's accounting guarantees. See the +[parity matrix](freetoken-swap-parity-matrix.md) for the source-backed +capability classification and [research](freetoken-swap-research.md) for +bounded qualification evidence and limits. + +The catalog is TOML and is optional. Start the daemon with `--catalog` or set `FREETOKEN_SWAP_CATALOG`: + +```toml +[router] +send_loading_state = true +preload_model = "qwen-coder-compatible" +startup_routing_profile = "coding" +# Matching direct-upstream assets return 409 instead of cold-loading. This +# suffix-only safe subset defaults to js/json/css/png/gif/jpg/jpeg/ico/txt. +upstream_no_activation_suffixes = [".js", ".json", ".css", ".png"] +# Activity metadata is always bounded and body-free. Captures are disabled by +# default; enabling them retains redacted bodies in memory only. +activity_max_entries = 1000 +capture_buffer_mb = 0 +activity_session_headers = ["X-Session-ID", "X-Litellm-Session-Id"] +performance_disabled = false +performance_every_s = 5 + +[models.qwen-coder] +model = "/models/Qwen3-Coder-30B-A3B-Q4_K_M.gguf" +port = 1922 +args = ["--max-seq-len-override", "4096", "--num-tokens", "4096"] +description = "GMKtek EVO-X2 candidate coding profile" +name = "Qwen coder" +aliases = ["qwen-coder-compatible"] +concurrency_limit = 2 +ready_timeout_s = 300 +check_endpoint = "/ready" +proxy = "http://127.0.0.1:${PORT}" +use_model_name = "qwen-coder" +upstream_timeout_s = 600 +send_loading_state = false + +[models.qwen-coder.metadata] +tier = "candidate" +family = "qwen" + +[models.qwen-coder.capabilities] +in = ["text"] +out = ["text"] +tools = true +context = 4096 + +[models.qwen-coder.set_fields] +"max_tokens?" = 4096 +"chat_template_kwargs.enable_thinking?" = true + +[models.qwen-coder.set_fields_by_id."qwen-coder:high"] +"chat_template_kwargs.reasoning_effort" = "high" + +[models.qwen-chat] +model = "/models/Qwen3.5-27B-Q4_K_M.gguf" +args = ["--max-seq-len-override", "4096", "--num-tokens", "4096"] + +[selectors.preferred-chat] +strategy = "warm" +targets = ["qwen-coder-compatible", "qwen-chat"] +name = "Preferred chat model" +description = "Reuse a ready target, otherwise start the first target" + +[selectors.preferred-chat.metadata] +tier = "stable" + +[profiles.coding] +description = "Coding-focused routing mode" + +[profiles.coding.pins] +llm-code = "preferred-chat" +llm-plan = "qwen-coder:high" +image-gen = "" +``` + +```bash +ft daemon --catalog /etc/freetoken/models.toml +ft daemon models +ft daemon routing-profiles +ft daemon activate-routing-profile coding +ft daemon clear-routing-profile +ft daemon start-profile qwen-coder +ft daemon switch-profile qwen-chat +ft daemon health +``` + +`GET /router/profiles`, `PUT /router/profiles/active`, +`POST /engine/start-profile`, and `POST /engine/switch-profile` expose explicit +control-plane operations. They require `X-FT-Token` whenever the daemon has a +token configured. The start/switch endpoints select a concrete model lifecycle +profile; the PUT endpoint activates or clears a runtime routing profile. +`GET /models` is instead the pinned public-model-list alias of `GET /v1/models` +and uses catalog API-key authentication. Use `switch-profile --force` only for +the same recovery case as `ft daemon switch --force`: the final accounting +receipt may be incomplete when a failed engine cannot be observed. + +Model lifecycle entries accept allowlisted `model`, `port`, `args`, `description`, `aliases`, +`unlisted`, readiness, TTL/unload, priority, group, and safe JSON request-filter +fields. Optional `use_model_name` gives the same profile a distinct model name +for outbound JSON requests without changing its configured or requested routing +identity. Optional `name` and JSON-compatible `metadata` provide display-only +model-list information; canonical and listed alternate IDs share those values. +Router-owned `type`, `aliases`, and `modelID` metadata wins over conflicting +operator keys, and declared capabilities own their rendered architecture, +capability, parameter, and context fields. A nested `capabilities` table may declare `in`/`out` +text modalities, `tools`, and a nonnegative `context` length for compatible +model-list clients. This metadata does not enable model behavior: operators +must advertise tools only when the model and chat template actually support +them. Unsupported image, audio, video, and reranker claims are rejected rather +than fabricated. Alternate IDs resolve to the same canonical profile and +resident process. Alias names must be unique and cannot collide with canonical +profile names. Canonical and alternate model IDs may use slash-separated safe +segments such as `organization/model` and colon variants such as `model:high`; +empty, traversal-like, and non-ASCII +segments are rejected, and the complete ID is limited to 128 characters. +Group names and each dot-delimited request-field segment retain the narrower +safe-name grammar. An unlisted profile and all its aliases remain routable and +manageable but are omitted from `GET /v1/models`. Set +`router.include_aliases_in_list = true` to list aliases for visible profiles; +canonical visible IDs are always listed. `args` +is passed as an argument vector to `ft serve`; it is never interpreted by a +shell. A profile cannot set `--model` or `--port` in `args`, because those +fields are owned by the supervisor and are part of its conflict and re-adoption +identity. The model files and catalog remain local operational configuration, +not repository content. + +Runtime routing profiles are named, atomically selected maps under +`[profiles..pins]`. A pin replaces a client model ID before aliases, +selectors, and target filters; an empty target disables that ID. Pins may +target a configured canonical ID, alternate ID, or selector, allowing one +profile switch to change several stable client names together. No routing +profile is active by default; `router.startup_routing_profile` selects one +validated profile before serving. Catalog reload still clears runtime pinning. +Active non-disabled pins +that do not shadow configured model/alias/selector IDs appear in the public +model listing with `meta.freetoken.type = "profile"`; disabled pins are omitted. +Profile pins also use longest-prefix replacement on `/upstream/` paths, but a +pin that targets a selector remains invalid there because selectors are not +direct-upstream IDs. Concrete load/unload management ignores active pin maps. + +`router.preload_model` accepts one concrete canonical or alternate ID, resolves +aliases during catalog validation, and acquires that model through the same +readiness/accounting/rollback path during daemon startup. The singleton limit +matches native one-resident capacity; selectors and unknown IDs are rejected. +Use a singleton persistent group when the preloaded model must remain resident +until explicit unload. + +Selectors are inference-only virtual model IDs. `pin` always resolves to its +first ordered target. `warm` resolves to the first readiness-gated resident +target, then the first target already activating, and otherwise falls back to +the first target. Resolution rewrites the request's top-level `model` to the +selected canonical or alternate target before that target's ordered request +filters run. Selector IDs appear in `/v1/models` unless `unlisted = true`; +their loaded status follows only the first target for `pin` and any target for +`warm`. Optional JSON-compatible selector `metadata` is nested under +`meta.freetoken`, while router-owned `type`, `strategy`, and `targets` keys +cannot be overridden. Targets must be configured profiles or aliases, selector +chaining is rejected, and `/upstream/{model-id}` plus named unload remain +concrete profile/alias controls. The `spillover` strategy requires concurrent +multi-resident or peer capacity and is therefore rejected under FreeToken's +explicit one-resident policy rather than emulated inaccurately. + +`drop_fields` removes configured dot-delimited object paths. `set_fields` +forces JSON-compatible values; a quoted key ending in `?` sets the value only +when that path is absent, so explicit `null`, zero, and false remain client +choices. `set_fields_by_id` runs last and can override global assignments for a +canonical or alternate requested ID. Its table names automatically become +aliases of the same resident model, subject to the normal collision checks. +When configured, `use_model_name` first rewrites the outbound top-level `model`; +filters then run in `drop_fields`, `set_fields`, and `set_fields_by_id` order and +cannot directly configure that protected field. The by-ID table still keys on +the client-facing selected/requested ID rather than the upstream override. For +a selector request without an explicit override, the router first replaces the +field with the resolved target. All filters apply to the exact acquired target +snapshot, including JSON direct-upstream requests; non-JSON direct bodies and +profiles with no rewrite or filters remain byte-exact. +An active profile cannot have its filter policy changed by catalog reload. +There is no expression evaluator or lifecycle shell-hook language. + +Set `port = 0` to request a kernel-selected loopback port on every cold native +activation. The daemon records the concrete assigned port and uses that same +target for child identity, readiness, proxying, accounting, and re-adoption; +an already resident dynamic profile keeps its port until it is unloaded. +Dynamic binding occurs only when a request reaches the head of admission, so +simultaneous cold requests for one profile share the single committed target. +`models..check_endpoint` selects a safe absolute readiness path and +defaults to `/health`; the private native qualifier uses `/ready`. A non-health +endpoint follows pinned HTTP-success semantics while the daemon still checks +the exact managed PID before and after every probe. `models..proxy` may +add a safe path prefix to `http://127.0.0.1:${PORT}`. The `${PORT}` placeholder +is mandatory, and other schemes, hosts, explicit ports, credentials, queries, +fragments, empty path segments, and traversal are rejected. This deliberately +keeps proxy traffic on the exact manager-owned child rather than creating an +arbitrary SSRF or split-ownership target. + +`models..upstream_timeout_s` overrides `router.upstream_timeout_s` for +the acquired profile's fresh loopback HTTP connection and response reads. The +exact admitted profile snapshot supplies the timeout for both ordinary and SSE +requests. TLS-handshake and pooled keepalive timeout knobs from the reference +are inapplicable because native targets are restricted to fresh manager-owned +plain-HTTP loopback connections. + +Each profile admits at most 10 reserved requests by default across its canonical +and alternate IDs. Set `models..concurrency_limit` to a positive override. +`router.global_concurrency_limit = 0` leaves the global cap disabled; a positive +value caps all active, queued, and activating routed requests. Capacity is +reserved before loading, so excess work is rejected immediately with HTTP 429, +`Retry-After: 1`, and `error.type=concurrency_limit` rather than consuming a +queue slot or launching an engine. Status and Prometheus expose reserved work. + +`router.send_loading_state = true` enables optional cold-load feedback for +strictly streaming `POST /v1/chat/completions` requests. A profile can override +the global setting with `models..send_loading_state = true` or `false`. +After concurrency admission, a cold request receives HTTP 200 SSE reasoning +deltas with loading and queue-position text until the readiness-gated engine is +available, followed by the real upstream stream. Admission rejection remains a +normal HTTP 429 JSON response. Once loading SSE has committed HTTP 200, a later +activation or connection failure is delivered as an in-band `error` event and +terminated with `data: [DONE]`. The default is disabled, and warm, non-chat, and +non-streaming requests retain the ordinary byte/status/header-preserving proxy. + +The native capacity policy is deliberately one resident child. Therefore a +nonpersistent group must use `swap = true, exclusive = true`; a persistent +protected slot must be a one-member group with `swap = false, exclusive = true`. +Catalog reload rejects llama-swap coexistence configurations instead of silently +pretending that multiple FreeToken engines are resident. + +When started with `--catalog`, the daemon polls it once per second by default. +`--catalog-watch-interval 0` disables that watcher. A changed catalog is parsed +and fully validated before atomic installation; malformed files and active +profile redefinitions are rejected without disturbing the running child. The +watcher's last result appears in `GET /router/status` and its sanitized events +appear in `/router/logs`. + +## Native router API + +The routed inference surface is `GET /v1/models` plus `POST /v1/chat/completions`, +`/v1/completions`, `/v1/responses`, `/v1/messages`, and +`/v1/messages/count_tokens`. Canonical and alternate IDs share one canonical +residency, capability metadata, and loaded/unloaded listing status while +preserving the client's request body. Readiness-gated activation is reported as +loaded, and stale child identity is reported as unloaded. The public listing includes descriptions but +never model paths or launch arguments. Declared text modalities, tool calling, +and context length use the pinned llama-swap listing fields. Unknown IDs return +a stable 404; unsupported FreeToken modalities are not fabricated. `GET /router/status`, `/router/models`, +`/router/profiles`, `/router/requests`, and `/metrics` expose configured and +resident state, capacity, queues, lifecycle timing, response bytes and proxy +byte rate, concurrency reservations, cancellation, and eviction signals. These transport measurements do +not substitute for live engine token-throughput qualification. +Browser clients receive the pinned compatibility contract: any `OPTIONS` +preflight is answered without lifecycle side effects, requested header names +are restricted to valid HTTP tokens, and authenticated `GET /v1/models` +reflects its `Origin`. Preflight never authorizes the corresponding request; +inference and management routes still enforce their configured keys. +`activeIdentityMatchesEngine` makes a stale or out-of-band child visible rather +than reporting its configured alias as resident. +`PUT /router/profiles/active`, `POST /router/unload`, `/router/reload`, and +`/router/requests/{id}/cancel` control idle eviction, atomic catalog reload, +and a queued, connecting, or active request. The request list exposes reserved +IDs from admission through stream completion, so an operator can cancel any +owned phase. An unload body containing `name` targets that profile; +an omitted body unloads all residents, which is exactly the current resident +under the explicit one-engine capacity policy. `POST /router/load` activates a named profile through +the same native lifecycle transaction without fabricating an inference request. +`GET /router/logs?since=` is a bounded SSE event stream; +it records only event type, alias, registered route template, status, +cancellation state, and response byte count—never prompts, request bodies, +headers, concrete URL paths, query strings, model paths, or API keys. +Router Bearer, Basic-password, and `X-Api-Key` credentials plus the daemon +`X-FT-Token` are terminated at the router and never forwarded to the engine; +ordinary non-hop-by-hop application headers are otherwise preserved. + +FreeToken's legacy `POST /generate` body has no model identifier, so exposing it +at the stable router URL would require an implicit default and violate explicit +model-ID ownership. It is therefore intentionally absent there. Clients that +need this legacy protocol must select a configured alias explicitly with +`POST /upstream/{profile}/generate`; that guarded route still acquires the same +router lease and preserves the request and SSE response bytes. + +`GET /ready` is an unauthenticated, side-effect-free readiness probe for the +stable router URL. It returns 200 only while a resident routed engine reports +FreeToken's `status=ok` and `maintenance=serving` **and** still exactly matches +the resident alias's model, port, and argument vector. Identity and fresh +health are checked behind the admission barrier, so a conflicting swap cannot +begin between the identity snapshot and a successful response; the probe never +cold-loads a profile. The stateless backend's `GET /v1/responses/{id}` and +response-specific cancel endpoints are authenticated at the stable daemon URL +and return the backend's documented `invalid_request_error` 404 without loading +a model. They are therefore compatibility endpoints, not routing or lifecycle +operations. + +`GET /ui/` serves a dependency-free local management shell. It embeds no +catalog values, paths, keys, or machine data; the operator enters a bearer key +for the current browser session and it calls the authenticated router APIs. +The UI presents configured/resident models, load/unload/reload controls, router +status, and the privacy-preserving `GET /router/hardware` memory view. Authenticated +`GET /router/activity`, `/router/activity/stats`, and `/router/captures/{id}` +provide bounded diagnostics. Activity rows never contain bodies or headers. +Real daemon runs persist them as bounded, fsynced JSONL under the daemon-owned +state directory; the file is atomically compacted and corrupt/truncated rows +fail closed. API responses expose persistence health without exposing its path. +Captures are disabled unless `router.capture_buffer_mb` is positive, live only +in memory, cap each response at 1 MiB, obey the total serialized-byte budget, +and redact Authorization, proxy authorization, cookies, `X-Api-Key`, +`X-FT-Token`, and custom token/secret/API-key header names. Bodies use Base64 +fields for binary fidelity and are never persisted. The UI lists body-free +activity and fetches a capture only after an explicit click. Configured session +headers are validated, may not name credentials, and are stored/displayed only +as stable 16-character SHA-256 labels; raw identifiers are never retained. MCP +and Tailcat remain deferred product expansions. + +`GET /api/performance` and `/router/performance` expose at most one hour of +periodic samples at `router.performance_every_s` (minimum five seconds). +The compatible envelope contains `sys_stats` and `gpu_stats`; native +`sys_stats` rows are explicitly scoped to the owned engine process tree and +contain only RAM/VRAM bytes, availability, and probe source. `gpu_stats` is +empty because FreeToken does not fabricate adapter-wide utilization, +temperature, power, or fan data from process memory. Strict RFC3339 `after` +filtering is supported. The history is memory-only, authenticated, bounded, +and disabled with `router.performance_disabled = true`. + +When `router.api_keys` is configured, authentication accepts an +`Authorization: Bearer` value, an HTTP Basic password, or `X-Api-Key` for +inference and, absent a daemon token, router management. Explicit Authorization +credentials take precedence over `X-Api-Key`; malformed Basic may fall back to +it. Invalid requests include a `WWW-Authenticate` challenge. An explicit daemon +`X-FT-Token` remains the dedicated control-plane override. The guarded +`/upstream/{model-id}/...` passthrough uses the same lease but refuses a direct +engine `prepare-stop`, which only the lifecycle owner may invoke. For +slash-namespaced IDs, the longest configured canonical or alternate ID wins; +encoded model separators and the remaining escaped path and query are +forwarded without decoding. By default, direct-upstream paths ending in +`.js`, `.json`, `.css`, `.png`, `.gif`, `.jpg`, `.jpeg`, `.ico`, or `.txt` +return HTTP 409 while the selected model is unloaded, rather than activating +an engine for a speculative asset request. They proxy normally when that exact +model is resident. Configure the bounded, dot-suffix-only +`router.upstream_no_activation_suffixes` list, or set it to `[]` to disable the +guard. Matching is case-sensitive and excludes the query string. + +These are illustrative paths, not a list of qualified models. In particular, dense Qwen GGUF support requires a compatible AMD/model-loader branch and cannot be inferred from this control-plane PR. + +After a profile launch, the daemon polls uncached engine health, verifies the process identity again after each probe, and waits for `status=ok` and `maintenance=serving`. Readiness failure returns HTTP 503. The client returns a nonzero exit code and defaults to a 1920-second transport budget, covering two maximum 900-second readiness windows plus lifecycle overhead. A user-specified client timeout still takes precedence. + +For `switch-profile`, readiness failure attempts to restore the exact previous engine and probes its readiness using a second window of the requested profile's `ready_timeout_s`. The response remains HTTP 503 because the requested replacement failed, with a separate `rollback.readiness` result. Recovery is single-use and invalidated by any newer start, stop, switch, or shutdown. HTTP probing releases the lifecycle lock and runs in the proxy pool, so an operator can stop a loading engine without waiting for the readiness timeout. Accounting failure during recovery preserves the failed engine instead of silently forcing cleanup. An initial `start-profile` or a switch with no previous engine leaves the failed process managed for diagnosis. These policies do not change the direct llama-swap supervisor. + +If a replacement launch raises before an owned child exists, the daemon attempts to relaunch the previous model with its exact port and arguments, under the same lifecycle transaction. Both switch endpoints return HTTP 503 with `code=switch_launch_failed`, the original accounting receipt, and a `rollback` result. `rollback.launched` means only that the recovery process launched, not that it is ready. A failed recovery is reported explicitly. Accounting failures before stop preserve the original engine; a post-spawn failure that leaves an owned child does not trigger a second launch. These safeguards apply to daemon switches, not the separate direct llama-swap supervisor. + +FreeToken's `/health` remains a backwards-compatible diagnostic endpoint and can return HTTP 200 while loading or failed. `/ready` returns HTTP 503 for loading, failure, or maintenance, and HTTP 200 only when accepting requests. Configure llama-swap with `checkEndpoint: /ready`, never `/health` or `/v1/models` as a substitute. + +Use `ft serve` or `python -m freetoken.cli serve` in a process command. The legacy `python -m freetoken` entrypoint does not accept the `serve` subcommand. Use a revision-specific `TORCH_EXTENSIONS_DIR` and prebuild native GGUF kernels before a maintenance window so an abandoned shared build lock cannot stall model initialization. For SSE token metrics, clients should request `stream_options: {"include_usage": true}`. + +The opt-in native maintenance harness `benchmarks/swap/qualify_native_router.py` +generates a private API key scoped only to its temporary daemon origin. Its +acceptance result requires 401 responses without that key and authenticated +Bearer, Basic-password, `X-Api-Key`, model/profile inventory, Prometheus metrics, +bounded router-log SSE, and authenticated periodic-performance history with a +positive available owned-process RAM/VRAM sample and no PID/model/path fields; +the key, catalog, headers, and raw captures are never publication artifacts. + +## Cancellation qualification + +The opt-in Linux harness `benchmarks/swap/qualify.py --cancellation` adds a live disconnect gate to its maintenance-window run. It reads SSE incrementally, verifies that generation is active, closes the response after the first content delta, and polls backend statistics through `/upstream/model-a/v1/stats`. Passing requires the same backend instance to become idle without increasing the normal-completion count. A backend restart, an already-finished response, or a missing terminal abort fails the gate. It then checks fresh A-to-B-to-A streaming completions. Prefix bytes, backend snapshots, first-content timing, abort latency, and recovery responses are private artifacts. + +The gate passed against the real Qwen3.6 GPU workload on GMKtek EVO-X2 in an approved maintenance window. The same backend changed from one active request to zero, with its normal-completion count unchanged. Observed first content was 0.368 seconds and terminal abort was observed 0.254 seconds after disconnect. Post-cancellation A-to-B-to-A streaming, concurrent routing, idle eviction, and protected-service restoration also passed. This is one bounded cancellation case, not a cancellation endurance benchmark. Use `--extended` as well to retain concurrent-request and TTL gates. Existing mandatory source, model, protected-service, and artifact arguments still apply. The current harness additionally requires the exact operating-system hostname in `--expected-hostname` before artifact creation or service inspection; `--allow-maintenance` is not a substitute for operator approval. + +## Native model-failure recovery qualification + +`benchmarks/swap/qualify_native_recovery.py` exercises the actual daemon profile endpoints with real FreeToken child processes. It starts the supplied model, verifies generation, switches to a deliberately invalid GGUF fixture in its private artifact directory, checks the HTTP 503 response and automatic recovery readiness, then verifies streamed generation from the restored model. The real Qwen3.6 run passed after the loader reported `GGUF magic invalid`. The original engine's sealed accounting receipt was complete; the failed loader's crash receipt was explicitly degraded with unknown token totals. Cleanup and restoration of the protected llama.cpp workload passed. + +The harness accepts separate `--source` and `--daemon-source` paths so the AMD runtime and the swap feature branch can be tested together without modifying a live checkout. `--extensions-dir` must identify a private cache prebuilt from the selected runtime source. Required arguments also include `--python`, `--model`, `--protected-service`, `--protected-url`, `--artifacts`, `--expected-hostname`, and `--allow-maintenance`. The exact hostname must match before artifact creation or service inspection; mismatch errors do not disclose it. The fixture never replaces an existing model. Raw logs and result files contain private deployment details and must not be published unreviewed. + +## Provenance and scope + +The design was informed by [mostlygeek/llama-swap](https://github.com/mostlygeek/llama-swap), checked out locally at `41ec321b6216d838488b2a7d936274ed227c0c5e` on 2026-09-10. llama-swap is MIT licensed (`LICENSE.md`). No llama-swap or llama.cpp code is vendored, modified, or submitted by this feature. FreeToken remains the sole change and pull-request target. diff --git a/docs/gmktec-evo-x2-284b-capacity-manifest-20260904.md b/docs/gmktec-evo-x2-284b-capacity-manifest-20260904.md new file mode 100644 index 0000000000..733b0f95b6 --- /dev/null +++ b/docs/gmktec-evo-x2-284b-capacity-manifest-20260904.md @@ -0,0 +1,89 @@ +# GMKtek EVO-X2 284B capacity manifest + +This is a read-only capacity snapshot for the GMKtek EVO-X2 Strix Halo system. +It is not a claim that a 284B model fits or serves interactively. + +## Observed platform + +| Field | Observed value | +| --- | --- | +| GPU | AMD Radeon 8060S Graphics | +| GFX target | `gfx1151` | +| Dedicated VRAM reported by ROCm SMI | 2,147,483,648 bytes (2 GiB) | +| Dedicated VRAM currently used | 368,336,896 bytes | +| System memory total | 59 GiB | +| System memory available at capture | 18 GiB | +| Swap configured | 127 GiB | +| Swap used at capture | 2.2 GiB | +| GPU power | 12.048 W | +| GPU temperature | 30.0 C | +| GPU utilization | 0 percent | +| Performance policy | auto | + +The reported 2 GiB VRAM is not the whole unified-memory budget. Conversely, +the 59 GiB system-memory total is not proof that the GPU can safely allocate +59 GiB for model weights and KV cache. A valid capacity claim must measure +GPU-visible allocations, runtime reservations, model weights, expert storage, +KV cache, and swap behavior under the exact model configuration. + +## Model inventory + +A read-only search of the configured model directory found no file or directory +matching `284B`, `280B`, or `235B`. No 284B capacity test was therefore run. + +## Latest live refresh + +A second read-only probe on 2026-09-05T12:01:56Z confirmed the capacity +boundary while the protected Qwen service was healthy. The host reported 59 +GiB total memory, 18 GiB available, 127 GiB configured swap with 2.0 GiB in +use, and 769 GiB free on the model filesystem. ROCm reported the same 2 GiB +dedicated VRAM and `gfx1151`, with 367,693,824 bytes currently used, 32.0 C, +28.042 W, and zero percent GPU utilization. The configured model payloads were +approximately 22 GiB for Qwen NVFP4, 67 GiB for the Qwen safetensors source, +and 15 GiB for Gemma Q4. No DeepSeek or 284B model payload was found in the +configured FreeToken model directory. Source-code references and archived +configuration names are not model payloads and are not treated as admission +evidence. + +## Qualification result + +**INCOMPLETE.** The current manifest establishes the memory and GPU baseline, +but it does not qualify a 284B model. The next test requires the exact model +artifact, quantization, context length, expert-loading policy, KV reservation, +and a clean-memory run with process-scoped swap telemetry. + +## Primary-paper capacity facts + +The paper's primary text identifies DeepSeek-V4-Flash as a 284B-parameter MoE +with approximately 13B active parameters per token and six selected experts +from 256 routed experts across 43 layers. It states that the deployed FP4 +configuration requires roughly 140 GB of expert weights and presents the +interactive demonstration on a 32 GB RTX 5090-class GPU. The paper also +explains that only the active computation fits in the GPU budget while the +complete expert pool resides in host-side storage and moves through the +CPU-GPU path as needed. + +These facts explain why the local 2 GiB dedicated-VRAM reading alone does not +decide feasibility, but they also show why the missing exact payload and a +measured host-memory and bandwidth budget are mandatory before claiming that +the GMKtek EVO-X2 can reproduce the paper result. Source: [FreeToken paper, +arXiv:2608.16157](https://arxiv.org/abs/2608.16157), especially the model and +hardware description in the introduction and evaluation setup. + +## Checkpoint identity warning + +The current official Hugging Face model page for +`deepseek-ai/DeepSeek-V4-Flash-0731` identifies the released checkpoint as +**304B parameters** and lists BF16, I64, F32, and F8_E4M3 tensor types. That is +not automatically the same artifact as the paper's 284B FP4 deployment. The +paper reproduction therefore requires the exact 284B checkpoint or an +authoritative conversion recipe, not merely the current model-card name. +Source: [official DeepSeek-V4-Flash-0731 model +card](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731). + +## Evidence source + +The raw values were collected from read-only `free -h`, `swapon --show +--bytes`, `rocm-smi`, DRM memory-info files, and a model-directory inventory. +No production service, model file, ROCm setting, power policy, or kernel state +was changed. diff --git a/docs/gmktec-evo-x2-amd-paper-protocol-ledger.md b/docs/gmktec-evo-x2-amd-paper-protocol-ledger.md new file mode 100644 index 0000000000..846464df90 --- /dev/null +++ b/docs/gmktec-evo-x2-amd-paper-protocol-ledger.md @@ -0,0 +1,26 @@ +# GMKtek EVO-X2 paper protocol ledger + +This ledger records which FreeToken paper fields are available before a result +is called a strict replication. The primary paper is `2608.16157v1.pdf` in the +project root. The upstream summary is `docs/upstream-qwen-paper-protocol.md`. + +| Field | Paper evidence | State | GMKtek EVO-X2 consequence | +| --- | --- | --- | --- | +| Models | Qwen3.6-35B-A3B, DeepSeek-V4-Flash, GLM-5.2 | Confirmed | Qwen is primary AMD qualification model | +| RTX 4060 row | RTX 4060 Laptop 8 GB, Core i9-13900H, LPDDR5 32 GiB, PCIe 4.0 x8 | Confirmed | Hardware reference only | +| RTX 5090 desktop | RTX 5090 32 GB, Ryzen 9 9950X3D, DDR5 192 GiB, PCIe 5.0 x16 | Confirmed | Capacity and performance reference only | +| Qwen precision | BF16 on most paper systems, NVFP4 on 8 GB laptop | Confirmed | Q4 GGUF cannot claim parity | +| DSV4 precision | Native MXFP4 experts | Confirmed | Requires separate AMD capacity and correctness program | +| Workloads | AIME, OpenCode plus SWE, Claude Code plus SWE, OpenClaw email/calendar | Confirmed | Recreate as paper-inspired until exact fixtures recovered | +| Metric | Per-request mean decode TPS and per-request mean TTFT | Confirmed | Harness records client SSE timings separately | +| Tail claim | Worst FreeToken agent turn below 44 seconds | Confirmed | Requires complete multi-turn matrix | +| Exact prompt corpus | Not published in paper | Missing | Blocks strict replication | +| Exact output caps and stops | Not published in paper | Missing | Blocks strict replication | +| Warmup and scored sequence | Only partially described | Missing | Blocks strict replication | +| Exact cache and KV allocation | Not published in paper | Missing | Blocks strict replication | +| Exact commit, driver, CUDA stack | Not fully published | Missing | Blocks strict replication | + +## Decision rule + +Until every missing row is resolved from released artifacts or the authors, +call the result `GMKtek EVO-X2 paper-inspired`, never `paper replication`. diff --git a/docs/gmktec-evo-x2-amd-run-log.md b/docs/gmktec-evo-x2-amd-run-log.md new file mode 100644 index 0000000000..f5c3c60809 --- /dev/null +++ b/docs/gmktec-evo-x2-amd-run-log.md @@ -0,0 +1,405 @@ +# GMKtek EVO-X2 AMD FreeToken execution log + +This file is append-only. Each entry records UTC time, branch and commit, test +category, command or script, artifact location, quality result, outcome, and +restoration result. Do not replace a failed entry with a later passing entry. + +## Baseline record + +| UTC date | Evidence | Category | Outcome | +| --- | --- | --- | --- | +| 2026-09-04 | PR branch static validation | Integration validation | Both Gemma runner shell scripts passed `bash -n`; all four Gemma benchmark Python modules passed `py_compile`; required reports and harness files were present; the PR checkout was clean. | +| 2026-08-28 | `gmk-evo-x2-rocm-validation-2026-08-28.md` | Native AMD functionality | Qwen NVFP4 and Gemma Q4 served through native ROCm/HIP API paths | +| 2026-08-29 | `gmk-evo-x2-qwen-router-optimization-2026-08-29.md` | Local control and optimization | Rejected quality-changing router candidates; retained a safe configuration | +| 2026-08-30 | `gmk-evo-x2-qwen-q4-raw-control-20260830.md` | Local control | FreeToken Q4 50.63 TPS versus ROCm llama.cpp 50.29 TPS on fixed raw prompt | +| 2026-08-30 | `gmk-evo-x2-gemma4-q4-vision-control-20260830.md` | Native AMD and local control | Text and visible-image controls passed | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-nvfp4-tail-baseline-20260830T081500Z/` | GMKtek EVO-X2 warm NVFP4 baseline | Five fixed-length samples passed: 28.76 mean TPS, 363 ms mean TTFT, 37.93 ms p99 gap, 526.95 ms maximum gap | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-aime-quality-20260830T082000Z/quality.json` | Qwen deterministic quality | Expected AIME output hash passed: 28.34 TPS, 410 ms TTFT, 37.62 ms p99 gap | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-quality-suite-20260830T083000Z/quality-suite.json` | Qwen versioned quality suite | Three visible-output checks passed: exact canary, arithmetic, and JSON fields | +| 2026-08-30 | GMKtek EVO-X2 read-only memory snapshot | Capacity and measurement readiness | Host reports 64 GB total RAM and about 1.4 GB swap in use, mainly Qwen workers; timed acceptance is paused pending clean memory recovery | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260830T081547Z/` | Controlled Qwen recovery | Verified server restart completed only after health returned `status: ok`; cold serial expert loading took about 6 minutes 22 seconds | +| 2026-08-30 | GMKtek EVO-X2 swap-residency reset | Measurement remediation | Temporarily disabled and re-enabled configured swap after verifying 20 GB available RAM and 2.1 GB swapped; swap use returned to zero and Qwen stayed healthy | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/runtime-manifest-20260830T082300Z/` | Runtime provenance | Captured clean host, ROCm, GPU policy, source, memory, storage, and process state before accepted baseline | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-nvfp4-clean-baseline-20260830T082400Z/` | GMKtek EVO-X2 warm NVFP4 baseline | Five samples passed with zero swap: 28.69 mean TPS, 367 ms mean TTFT, 37.89 ms p99 gap, 39.08 ms maximum gap | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-nvfp4-clean-scheduler-20260830T082500Z/` | GMKtek EVO-X2 medium scheduler baseline | Three samples passed with zero swap: 27.89 mean TPS, 429 ms mean TTFT, 38.99 ms p99 gap, 71.23 ms maximum gap | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-multiturn-state-20260830T083100Z/multiturn.json` | Bounded multi-turn state control | Three dependent turns passed with zero swap: 411 ms mean TTFT, 440 ms worst TTFT, 38.49 ms worst token gap | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-long-context-2k-clean-20260830T083657Z/long-context.json` | GMKtek EVO-X2 1.8K-context retrieval control | Five of five exact marker retrievals passed at 1,845 reported prompt tokens with zero swap: 428 ms mean TTFT, 431 ms p99 TTFT, and 40.48 ms p99 token gap. This is a GMKtek EVO-X2 control, not a replication of the paper's 56K to 65K agent sessions. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-long-context-7k-calibration-20260830T083721Z/long-context.json` | Long-context limit discovery | Preserved expected failure: 6,845-token prompt was rejected because the live auto-cache geometry exposed only 2,068 prompt-plus-generation tokens despite `--max-seq-len-override 8192`. The server stayed healthy and swap-free. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-kv-8192-rebuild-20260830T083845Z/` | Reversible cache repair | Idle-only runtime rebuild succeeded: reduced the MoE cache from 8,974 to 8,700 slots and expanded KV pages from 2,068 to 8,192. Cache-budget arithmetic retained about 361 MB more dynamic-cache headroom than the original geometry; server remained healthy. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-long-context-7k-kv8192-rerun-20260830T084010Z/long-context.json` | 6.8K identical-prefix control | Five exact marker retrievals passed at 6,845 reported prompt tokens. The first request had 32.98 s TTFT while repeated identical-prefix requests were about 433 ms, demonstrating prefix-cache reuse. A brief 2.04 MB swap residency was remediated to zero before the next acceptance run. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-long-context-7k-cold-kv8192-20260830T084300Z/long-context.json` | 6.8K forced-cold-prefill control | Five of five exact marker retrievals passed at 6,856 reported prompt tokens with a unique early nonce per sample, preventing long-prefix reuse: 13.506 s mean TTFT, 13.520 s p99 TTFT, 44.43 ms p99 token gap, zero swap, and 38 C post-run GPU temperature. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-kv8192-short-decode-20260830T084448Z/summary.json` | Expanded-KV short decode control | Five 128-token throughput samples passed with zero swap: 28.85 mean TPS, 28.87 median TPS, and 0.071 TPS standard deviation. This is within measurement noise of the earlier 28.69 TPS clean baseline, so the 8K KV profile did not show a short-decode regression. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-concurrent-c1-kv8192-portable-20260830T085100Z/concurrent.json` | One-client concurrent-harness reference | Three rounds passed with zero swap: 25.59 mean aggregate TPS, 1.96 s p99 TTFT, and 38.86 ms p99 token gap. One cold or cache-miss round remains visible in the p99 rather than being discarded. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-concurrent-c2-kv8192-portable-20260830T085200Z/concurrent.json` | Two-client concurrent tail control | Three rounds passed with zero swap: 28.40 mean aggregate TPS, 3.90 s p99 TTFT, 70.76 ms p99 token gap, and a 3.19 s worst individual gap. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-concurrent-c4-kv8192-portable-20260830T085400Z/concurrent.json` | Four-client concurrent tail control | Three rounds passed with zero swap: 52.36 mean aggregate TPS, 1.44 s p99 TTFT, 76.17 ms p99 token gap, and 37 C post-run GPU temperature. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-concurrent-c8-kv8192-portable-20260830T085600Z/concurrent.json` | Eight-client saturation control | Three rounds passed and stayed swap-free: 53.29 mean aggregate TPS and 78.30 ms p99 token gap, but p99 TTFT was 19.70 s. Aggregate throughput therefore saturated while interactive admission latency became poor. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-vision-20260830T085943Z/quality.json` | Gemma4 rerun text quality | The isolated Gemma4 Q4 text control returned the expected `323` with matching 30 prompt and 4 completion tokens. The first-use run had 49.28 s TTFT while HIP GGUF kernels compiled. The suite was deliberately stopped before image checks after swap reached about 222 MB, so this is text-only evidence and not a vision pass. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260830T090140Z/` | Persistent 8K recovery validation | A full Qwen recovery after the isolated Gemma stop reached `status: ok` after serial expert load. The recovered server resolved 8,224 KV pages and 8,903 MoE slots from the persistent 8,192-token reserve; swap was safely reset to zero afterwards. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-multiturn-battery-30-swappiness1-20260830T091725Z/partial-summary.json` | Repeated multi-turn endurance boundary | Sixteen of 16 completed dependent state-retention sessions passed, but the requested 30-session battery was stopped by the swap guard at 26,279,936 bytes. Worst completed-turn TTFT was 22.69 s and worst token gap was 39.07 ms. This is not an endurance pass. | +| 2026-08-30 | GMKtek EVO-X2 read-only plus reversible swap-policy experiment | Swap diagnosis | Default `vm.swappiness=60` allowed Qwen workers to retain swapped pages despite about 18 GB available RAM. A temporary `vm.swappiness=1` plus swap reset kept a single health check at zero worker swap, but repeated sessions still reached the swap guard. The policy was restored to 60 after the experiment. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-multiturn-battery-30-swap256m-20260830T092021Z/battery/summary.json` | Bounded repeated multi-turn characterization | All 30 dependent state-retention sessions passed with a documented 256 MiB swap ceiling. Actual swap remained stable at about 3.1 MiB, worst turn TTFT was 417.73 ms, p99 worst-turn TTFT was 417.73 ms, and p99 token gap was 42.35 ms. This qualifies the bounded session workload, not a zero-swap or 24-hour endurance claim. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen35b-llamacpp-rocm10-fresh-20260830T092532Z/` | Concurrent-residency capacity control | Preserved expected failure: with Qwen FreeToken live, ROCm llama.cpp Q4_K_M could not allocate its 20,583.34 MiB device buffer and exited during initialization. FreeToken remained healthy. This proves the two 35B services cannot coexist in the tested 64 GB shared-memory configuration; it is not a llama.cpp throughput result. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen35b-llamacpp-rocm10-timeshare-20260830T092814Z/llamacpp-control/benchmark/summary.json` | Standalone ROCm llama.cpp practical control | Three fixed-harness Qwen Q4_K_M samples passed after FreeToken was stopped: 49.39 mean decode TPS, 49.39 median TPS, and 0.0122 TPS standard deviation. FreeToken was restored afterward. This is a time-shared, practical comparison because llama.cpp Q4_K_M and FreeToken NVFP4 are different model formats. | +| 2026-08-30 | `/home/operator/freetoken-amd/artifacts/qwen-freetoken-post-timeshare-20260830T093836Z/summary.json` | Post-recovery FreeToken Qwen control | Three fixed-harness NVFP4 samples passed after the time-shared llama.cpp control: 27.95 mean decode TPS, 27.96 median TPS, and 0.0198 TPS standard deviation. Health returned `status: ok`; the recovered server retained 8,224 KV pages and 8,903 MoE slots. Cold recovery temporarily used about 2.7 GB swap, so this result is not a zero-swap acceptance result. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c79-max-requests-8-concurrent-c4-prefill-20260902T080725Z/`, `/home/operator/freetoken-amd/artifacts/q4-c80-max-requests-4-concurrent-c4-prefill-20260902T081729Z/`, and `/home/operator/freetoken-amd/artifacts/q4-c81-max-requests-4-concurrent-c4-prefill-repeat-20260902T082828Z/` | Qwen Q4 four-client admission-cap control with prefill instrumentation | All three runs matched the same-source deterministic output hash `3302eda43396`, completed the three scheduler samples and three four-client rounds, and restored the normal service. The 8-request candidate recorded 4,621.18 mean aggregate prefill TPS, 91.89 aggregate decode TPS, 1.089 s p99 TTFT, and 41.54 ms p99 token gap. The four-request runs recorded 4,219.97 and 4,557.90 mean aggregate prefill TPS, 92.19 and 91.26 aggregate decode TPS, 1.394 and 1.104 s p99 TTFT, and 40.38 and 44.35 ms p99 token gap. The clean four-request repeat overlaps the 8-request result on all material dimensions, while decode did not improve. The 8-request setting is rejected as non-material and the qualified cap remains four. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c82-gdn-stage2-component-20260902T085352Z/`, `/home/operator/freetoken-amd/artifacts/q4-c83-gdn-stage4-component-20260902T090344Z/`, `/home/operator/freetoken-amd/artifacts/q4-c84-gdn-stage2-full-api-20260902T091149Z/`, and `/home/operator/freetoken-amd/artifacts/q4-c85-gdn-stage2-full-api-corrected-quality-20260902T092132Z/` | Bounded fused GDN pipeline-stage closure | C82 two stages improved the geometry-matched component kernel by 6.05 percent with exact output and recurrent-state equality. C83 four stages was 0.21 percent slower with exact parity. C84 preserved a controller rejection caused by an overlong quality reference and made no TPS claim. C85 used the canonical exact fingerprint `3302eda43396`, then completed all scheduler and four-client rounds. It recorded 3,115.37 mean single-request prefill TPS, 48.86 decode TPS, 389.04 ms warm TTFT, 4,356.98 mean aggregate C4 prefill TPS, 90.75 aggregate decode TPS, 1.380 s p99 TTFT, and 42.08 ms p99 gap. End-to-end performance did not improve, so two and four stages are rejected and the qualified three-stage launch remains. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c86-llamacpp-rocm10-protected-c4-20260902T093505Z/`, `/home/operator/freetoken-amd/artifacts/q4-c87-llamacpp-rocm10-protected-c4-corrected-source-20260902T094409Z/`, `/home/operator/freetoken-amd/artifacts/q4-c88-llamacpp-rocm10-protected-c4-final-20260902T095306Z/`, and `/home/operator/freetoken-amd/artifacts/q4-c89-llamacpp-rocm10-protected-c4-slots4-20260902T100349Z/` | Protected ROCm 10 llama.cpp Qwen Q4_K_M comparison | C86 and C87 are preserved harness-layout failures with no performance claim. C88 completed quality and workload artifacts but used one llama.cpp slot, serializing C4 clients and producing an invalid C4 comparison. C89 used four slots, passed exact canary, arithmetic, and JSON checks, then completed the fixed scheduler and all C4 rounds. It recorded 19,114.41 mean single prefill TPS, 47.24 decode TPS, 63.41 ms warm TTFT, 11,432.05 mean aggregate C4 prefill TPS, 94.79 aggregate decode TPS, 3.901 s p99 TTFT, and 90.00 ms p99 token gap. C4 prefill varied from 1,242.75 to 16,794.21 TPS and the full distribution is preserved. The normal Qwen API recovered after 503 controller probes and an independent completion returned HTTP 200. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c90-cold-prefill-freetoken-20260902T102141Z/`, `/home/operator/freetoken-amd/artifacts/q4-c91-cold-prefill-freetoken-ready-20260902T103251Z/`, and `/home/operator/freetoken-amd/artifacts/q4-c92-llamacpp-cold-prefill-20260902T104716Z/` | Cache-neutral Qwen Q4 cold-prefill comparison | C90 exposed a controller readiness error: HTTP health preceded real Q4 completion readiness, so three HTTP 503 responses produced no TPS claim; normal recovery completed after 536 probes. C91 repaired readiness, then three 1,016-token unique-prefix requests with distinct early nonces and prompt hashes returned exact `azure-17`, at 88.78, 307.48, and 306.81 cold-prefill TPS. Its optional cached-token field was absent. C92 ran the same workload against ROCm 10 llama.cpp and passed all three exact answers with explicit zero cached tokens and 983.15, 961.51, and 991.21 cold-prefill TPS. C91 and C92 normal-service recovery completed after 536 and 473 probes. This establishes a cache-neutral prompt-prefix comparison without using known cache-hit rounds as evidence. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/qwen35b-llamacpp-rocm10-timeshare-five-20260904T101357Z/` | Five-sample ROCm 10 llama.cpp Q4 control | Corrected time-share control completed five scored samples with zero failed samples. Mean prefill was 19,343.40 TPS, median 19,229.96 TPS; mean decode was 46.6625 TPS, median 46.7524 TPS, standard deviation 0.2404 TPS. Mean token gap was 21.43 ms. The server used the recorded b10141 ROCm 10 build and the matching Q4_K_M GGUF. Protected-service recovery was still in progress when this row was recorded, so recovery evidence must be verified separately before the run is considered operationally complete. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/qwen35b-freetoken-five-20260904T102530Z/` | Five-sample FreeToken Q4 scheduler control | Five of five samples completed against the recovered native ROCm/HIP endpoint with no failed samples. Mean prefill was 2,936.92 TPS and mean decode was 28.0438 TPS; mean token gap was 35.38 ms. Every sample used 1,212 prompt tokens and 255 completion tokens. The paired llama.cpp control used the same 1,212-token prompt but emitted 256 completion tokens, so this is a strong same-workload control but not a strict equal-output-token claim. Readiness was proven by a `READY.` completion before scoring. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T111440Z/` | Isolated NVFP4 Marlin gate/up 8x16 API validation | The same-process differential had already shown bit-identical output and about 7 percent lower kernel median latency. The isolated server used the exact `d6ee8cef479c` source/cache pair, `FREETOKEN_DISABLE_JIT=1`, and port 1922. Five throughput samples passed with 1,212 prompt and 255 completion tokens each, but mean decode was 24.9068 TPS (median 27.3090, stdev 5.3325), below the paired 28.0438 TPS baseline. The candidate is rejected for end-to-end promotion despite kernel-level equality and remains documented as a valid diagnostic result. The wrapper restored the protected Qwen service; recovery was verified by a subsequent health check. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T113256Z/` | Isolated NVFP4 MoE prefill-overlap API validation | Enabling MoE prefill overlap on the exact matched source/cache pair produced five of five completed throughput samples at 29.2104 mean decode TPS, 29.1985 median, and 0.0263 TPS standard deviation, versus 28.0438 TPS for the current no-overlap control. However, the candidate scheduler response SHA1 `052f0756fc9ba9fd677fd829b8ee047e3b9187ce` differed from the established control SHA1 `d493dabcf0e74e7b5582e2df7a3893869dca004a`. Because deterministic output equivalence is a promotion gate, the apparent 4.2 percent throughput gain is rejected pending a canonical quality run. The protected service was restored and returned to `status: ok`. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T115031Z/` | Current-source NVFP4 MoE prefill-overlap repeat | Repeating overlap with the current protected-service source reproduced the same candidate response SHA1 `052f0756fc9ba9fd677fd829b8ee047e3b9187ce`, confirming the mismatch is associated with the overlap path rather than only the older checkout. Five samples completed, but one fell to 21.3820 TPS; mean was 28.3894 TPS, median 30.2054, and standard deviation 3.9198. The candidate is rejected for both deterministic-quality mismatch and unstable tail behavior. The protected service was recovered and verified `status: ok`. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T121004Z/` | NVFP4 MoE cache-statistics control | The isolated no-overlap current-source control enabled `--moe-collect-stats` and completed all five throughput samples. The statistics snapshot recorded 61,200 layer calls, eight active experts per layer-step, 0.58696 missing experts per layer-step, and a 7.337 percent miss rate. All misses were CPU-resolved (`fetched_per_layer=0`), so the dominant remaining MoE cost is CPU-side expert execution/fetch rather than a GPU cache-copy path. The statistics-enabled run measured 25.9511 mean decode TPS with a 5.1125 TPS standard deviation, demonstrating that diagnostics are intrusive and not a throughput result. Raw cache statistics are preserved in `cache-stats.json`; the protected service was recovered and returned to `status: ok`. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T122653Z/` | NVFP4 hybrid expert-fetch candidate | The hybrid backend allowed one GPU fetch per layer-step while computing remaining misses on CPU. All five samples passed with stable 27.2949 mean decode TPS, 27.3337 median, and 0.0846 TPS standard deviation, using 1,212 prompt and 255 completion tokens per sample. This was below the accepted offload baseline of 28.0438 TPS, so the one-fetch hybrid setting is rejected. The protected service was restored and subsequently verified healthy. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T124245Z/` | NVFP4 larger-cache candidate at memory ratio 0.38 | Raising the memory ratio from 0.35 to 0.38 resolved `moe_cache_size=9919` versus the baseline 8,903 and left 17.77 GiB free after initialization. Five throughput samples completed with 29.7945 mean decode TPS, 30.0211 median, and 0.5069 TPS standard deviation, approximately 6.2 percent above the 28.0438 TPS control. This is promising but not promoted yet because the scheduler response fingerprint differs from the earlier control contract; a canonical AIME and API quality gate is required before acceptance. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T125838Z/` | NVFP4 larger-cache candidate with deterministic API quality gate | The 0.38 memory-ratio candidate passed the canonical three-case API quality suite: exact canary, exact arithmetic, and exact JSON-field response, all with `reasoning_effort=none`, temperature 0, and top-k 1. The five-sample throughput run also passed, recording 29.7615 mean decode TPS, 29.9740 median, 0.4875 TPS standard deviation, and 1,212 prompt/255 completion tokens per sample. This is eligible for the next AIME, long-context, state-retention, and concurrent-request gates, but is not yet promoted to endurance. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T125838Z/` | NVFP4 larger-cache canonical AIME gate | The candidate's canonical AIME run failed: output fingerprint `1cae5bae914f` versus required `3302eda43396`, despite a complete 127-token response and 30.3727 decode TPS. This definitively blocks promotion of the 0.38 memory-ratio cache setting. The basic API suite remains passed, but deterministic model quality takes precedence. Raw AIME evidence is preserved and the protected service was restored. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T134923Z/` | NVFP4 0.38 cache AIME repeat against re-anchored baseline | Repeating the larger-cache candidate against the current protected baseline `cd580f4978fb` yielded the same candidate fingerprint `1cae5bae914f`, while the five-sample throughput remained strong at 29.9673 mean TPS, 29.9703 median, and 0.0186 TPS standard deviation. The candidate therefore definitively changes model output despite passing the basic API suite and is rejected. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T132948Z/` | NVFP4 0.35 CPU-thread candidate with AIME gate | Setting `--moe-cpu-threads 24` on the accepted 0.35 configuration passed the basic API suite and recorded 29.7572 mean decode TPS, 29.9729 median, and 0.4904 TPS standard deviation. The canonical AIME request completed 127 tokens at 30.37 decode TPS but produced fingerprint `1cae5bae914f` instead of `3302eda43396`, so this thread-count candidate is rejected pending resolution of the deterministic sampling mismatch. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/protected-aime-control-20260904T134747Z/` | Protected AIME baseline re-anchor | Two consecutive read-only AIME controls against the healthy protected Qwen service produced the same fingerprint `cd580f4978fb` at 127 completion tokens, with decode rates 29.6430 and 29.6893 TPS. The prior `3302eda43396` value is retained as historical evidence, but the active quality baseline is now `cd580f4978fb`; future candidates must be compared against this current-source fingerprint. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/w2-paper-inspired-tool-control-20260904-512.json` | Paper-inspired W2 bounded coding-tool control | The native ROCm/HIP endpoint completed the three-turn read-tool, exact-patch, and visible-confirmation trajectory. All structured tool-call and sandbox SHA-256 gates passed. The run used 1,103 prompt tokens and 438 completion tokens, with 16.00 aggregate end-to-end TPS, 8.09 s mean visible TTFT, 14.05 s maximum visible TTFT, and 36.48 ms p99 visible token gap. This is a bounded local control, not strict OpenCode SWE-bench replication. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/w3-paper-inspired-long-context-20260904.json` | Paper-inspired W3 long-context retrieval control | Five of five prefix-variation retrieval samples passed at 4,856 prompt tokens each. Cold-prefill TPS mean was 403.67, median 354.22, and p95 544.36. TTFT mean was 12.58 s, p95 15.98 s. P99 visible token gap was 39.21 ms. The protected marker was recovered deterministically in every sample. This reaches the available 8,192-token service envelope but is not strict Claude Code replication of the paper's 56K to 65K context workload. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/w4-paper-inspired-state-retention-20260904.json` | Paper-inspired W4 multi-turn state-retention control | Three-turn state-retention suite passed with full prior visible conversation carried forward. Mean TTFT was 429.16 ms, maximum TTFT 443.01 ms, and p99 visible token gap 39.14 ms. This is a bounded local control, not strict OpenClaw email/calendar replication or the paper's 24.5K context floor. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-screen-20260904.jsonl` | NVFP4 decode kernel launch screen | Gate/up projection screen used 50 HIP-event iterations per configuration. Four warps was fastest and numerically matched the two-warp reference at 0.06097 ms median; two warps measured 0.09530 ms median. Eight warps measured 0.05663 ms median but produced a different output SHA-1, so it is rejected pending numerical investigation. This is a kernel screen only and carries no API throughput claim. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-down-screen-20260904.jsonl` | NVFP4 down-projection launch screen | Down projection screen used 50 HIP-event iterations per configuration. Four warps was the fastest numerically safe choice at 0.03470 ms median and matched the reference hash. Two warps measured 0.06044 ms median; eight warps regressed to 0.25439 ms median. All three configurations produced the same output SHA-1, so this screen closes the down-projection warp choice at four. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-gate-grid-20260904.jsonl` | NVFP4 gate/up tile-shape screen | Four-warp tile variants were screened for 50 HIP-event iterations. `BLOCK_SIZE_N=8, BLOCK_SIZE_KW=16` measured 0.05598 ms median, faster than the 16x16 reference screen at 0.06097 ms; 16x8 measured 0.07168 ms, 16x32 0.06038 ms, and 32x16 0.15733 ms. This remains a candidate only: the raw output hash differed from the earlier reference screen, so no API or quality acceptance is claimed until same-process differential validation. | +| 2026-09-04 | `bench_nvfp4_marlin_decode.py` same-process differential run (raw output retained in terminal evidence) | NVFP4 gate/up 8x16 candidate differential | Candidate `BLOCK_SIZE_N=8, BLOCK_SIZE_KW=16, warps=4` was compared against 16x16 using identical tensors in one process. Maximum and mean absolute differences were both 0, and `storage_equal=true`. Over 100 HIP-event iterations after 50 warmups, candidate median latency was 0.06016 ms versus reference 0.06470 ms, a 7.0% median improvement. The candidate is eligible for isolated API validation; it is not yet production-accepted. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c93-grouped-gate-up-component-20260902T110328Z/` and `/home/operator/freetoken-amd/artifacts/q4-c94-grouped-down-component-20260902T111336Z/` | Grouped Q4/Q5 prefill projection isolation | The vector reference was 81.29 ms on the real 1,024-token Q4_K_M layer screen. Grouping Q4 gate/up only measured 40.57 ms but changed final storage, with 0.001343 maximum and 0.0000720 mean absolute difference. Grouping Q5 down only measured 51.63 ms but also changed final storage, with 0.000679 maximum and 0.0000608 mean absolute difference. Both only passed the explicit component tolerance, not exact quality, and are rejected from API promotion. The normal Qwen API recovered after 476 and 489 controller probes. | +| 2026-09-02 | Live ROCm attention-backend availability probe | Attention candidate admission | The native PyTorch runtime is `2.13.0+rocm10.0.0` with HIP `7.15.26333`. `flashinfer` and `sgl_kernel` are absent. FreeToken metadata shows `fi` requires FlashInfer, `fa` requires SGL Kernel, and `trtllm` also requires NVIDIA `sm100`; therefore Triton is the only available native AMD attention backend. The normal Qwen endpoint remained responsive. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c96-cold-concurrent-freetoken-20260902T112843Z/` and `/home/operator/freetoken-amd/artifacts/q4-c97-cold-concurrent-llamacpp-20260902T113840Z/` | Cache-neutral Qwen Q4 C4 prefill comparison | One synchronized four-client round used distinct early nonces and prompt hashes, with 1,223 reported prompt tokens per request. C96 FreeToken recorded 327.28 aggregate prefill TPS, 82.57 mean per-request prefill TPS, and 14.95 s p99 TTFT; its cached-token field was absent. C97 ROCm llama.cpp recorded 997.26 aggregate prefill TPS, 251.60 mean per-request prefill TPS, 4.91 s p99 TTFT, and explicit zero cached tokens on every request. Both runtimes completed each client group over the same first-token interval, proving concurrent batch formation while preserving the core prefill gap. Normal Qwen recovery completed after 463 and 460 probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c98-two-rows-component-r1-20260902/`, `/home/operator/freetoken-amd/artifacts/q4-c100-two-rows-api-quality-r1-20260902/`, and `/home/operator/freetoken-amd/artifacts/q4-c102-two-rows-full-scheduler-r1-20260902/` | Exact HIP Q4_K/Q5_K two-row vector candidate | C98 reduced real-weight component median time from 81.168 ms to 74.488 ms with a bit-identical output SHA-256. C100 passed exact canary, arithmetic, and JSON API quality checks, then recorded 336.60 mean cold-prefill TPS. C102 matched the established AIME output SHA1 `3302eda43396`, completed three scheduler samples and three C4 rounds, and recorded 3,079.00 mean single prefill TPS, 48.70 decode TPS, 4,759.40 aggregate C4 prefill TPS, 92.92 aggregate C4 decode TPS, 1.059 s C4 p99 TTFT, and 41.08 ms C4 p99 gap. Normal recovery completed after 470 and 430 completion probes. The implementation remains default-off because its primary single-request metrics are slightly below the qualified generic-vector profile. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c103-two-rows-occupancy2-component-r1-20260902/` | Two-row HIP occupancy candidate | Compiling the bit-identical Q4_K/Q5_K two-row vector kernel for two resident blocks per compute unit measured 74.600 ms, versus 74.488 ms for the one-block two-row implementation. The exact output SHA-256 matched the reference. The 0.15 percent regression closed this occupancy variant before API testing; normal recovery completed after 465 completion probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c104-q5-two-block-component-r1-20260902/` | Q5_K-only two-row occupancy candidate | Q4_K remained at one block per compute unit while Q5_K alone compiled for two. Exact output SHA-256 matched the reference, but the 74.505 ms median was statistically indistinguishable from and nominally above the 74.488 ms qualified result. The per-format occupancy family is closed before API testing; normal recovery completed after 485 completion probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c105-two-rows-rocprof-prefill-r1-20260902/` | Quality-qualified two-row Q4 prefill ROCprof diagnostic | One 6,010-token isolated prefill completed under the exact two-row candidate and produced a finalized ROCprof SQLite database. The capture attributed 5,809.917 ms across 40 Q4_K two-row vector calls, 3,565.542 ms across 37 Q5_K two-row calls, 2,019.232 ms to BF16 direct copies, and 1,931.956 ms to the gated delta-rule solve. This is profiler diagnostic evidence, not a TPS benchmark. The normal Qwen completion-gate artifact was written after recovery. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c106-three-rows-component-r1-20260902/` | Exact HIP Q4_K/Q5_K three-row vector component candidate | One wave computed three adjacent rows while retaining each row's production vector-dot and reduction order. The candidate exactly matched SHA-256 `46f7495acbbb563b65e75a7bea6b6dab22d4ca16b805b1558d37bc546fff072d` and measured 72.805 ms, versus 80.848 ms for its same-run generic vector reference and 74.488 ms for the qualified two-row candidate. Normal recovery completed after 467 completion probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c107-three-rows-api-quality-r1-20260902/` | Exact HIP Q4_K/Q5_K three-row API quality and cold-prefill gate | Exact canary, arithmetic, and JSON quality checks passed. Three cache-neutral 1,016-token marker-retrieval requests all returned `azure-17`, at 352.640, 360.634, and 360.580 cold-prefill TPS, for a 357.951 TPS mean and 2.881 s p99 TTFT. This is a 6.34 percent mean increase over the C100 two-row cold-prefill result. Normal recovery completed after 425 completion probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c108-three-rows-full-scheduler-r1-20260902/` | Three-row full-gate controller invalidation | The isolated candidate reached real completion readiness, but the new AIME controller supplied a `/v1` API base to a helper that appends `/v1`; its resulting `/v1/v1/models` request returned HTTP 404 before quality or TPS work. This run makes no performance or quality claim. The normal service recovered after 433 completion probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c109-three-rows-full-scheduler-r2-20260902/` | Exact HIP Q4_K/Q5_K three-row complete serving gate | The canonical AIME hash `3302eda43396` passed. Three scheduler samples recorded 3,022.10 mean client prefill TPS, 47.73 decode TPS, and 401.05 ms warm TTFT. Three C4 rounds recorded 4,613.19 aggregate prefill TPS, 94.69 aggregate decode TPS, 1.060 s p99 TTFT, and 40.16 ms p99 token gap. The candidate improves C4 decode and tail gap but regresses primary warm prefill versus the qualified generic vector, so it remains default-off. Normal recovery completed after 477 completion probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c110-three-rows-occupancy2-component-r1-20260902/` | Three-row HIP two-block occupancy candidate | The real-weight component output exactly matched SHA-256 `46f7495acbbb563b65e75a7bea6b6dab22d4ca16b805b1558d37bc546fff072d`. The 72.803 ms median was only 0.002 ms, or 0.003 percent, below C106's 72.805 ms, which is measurement variation rather than a demonstrated gain. The candidate is closed before API testing; normal recovery completed after 479 completion probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c111-grouped-api-quality-r1-20260902/` and `/home/operator/freetoken-amd/artifacts/q4-c112-grouped-full-r1-20260902/` | Grouped-MoE Q4/Q5 prefill quality qualification | C111 enabled the existing grouped Q4 gate/up and Q5 down prefill path for prompts of two or more tokens. Its short API suite and all three cache-neutral marker retrievals passed, with 822.017 TPS on the first 1,016-token cold-prefill sample. C112 then applied the canonical deterministic AIME gate and rejected the candidate: required SHA1 `3302eda43396`, observed SHA1 `c6d77205c0de`. No warm scheduler or C4 TPS claim is valid from C112 because the quality admission gate stopped the workload first. The protected normal Qwen service recovered through a real completion after 474 probes for C111 and 470 probes for C112. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c113-grouped-q4-full-r1-20260902/` and `/home/operator/freetoken-amd/artifacts/q4-c114-grouped-gateup-full-r1-20260902/` | Grouped Q4 gate/up isolation | C113 is invalid: the old controller passed an unsupported selector name, then its readiness check accepted an error document. It makes no quality or TPS claim. The repaired controller requires a real HTTP 200 completion and accepts the runtime selectors `both`, `gate_up`, and `down`. C114 isolated `gate_up`, reached the canonical AIME gate, and was rejected: required SHA1 `3302eda43396`, observed SHA1 `68e196c42c75`. This proves the Q4 gate/up grouped path alone changes the deterministic output. Normal recovery completed after 446 probes for C113 and 417 probes for C114. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c115-grouped-down-full-r1-20260902/` | Grouped Q5 down isolation and family closure | The final isolated grouped projection, `down`, reached the canonical AIME gate and was rejected: required SHA1 `3302eda43396`, observed SHA1 `03fa3848f59c`. Together with C112 and C114, this closes the existing grouped-prefill family: each individual projection and their combination changes deterministic visible output. No grouped-prefill TPS result is eligible for default-selection evidence. The protected normal Qwen service recovered through a real completion after 435 probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c116-four-rows-component-r1-20260902/` | Exact HIP Q4_K/Q5_K four-row component screen | The opt-in four-row HIP kernel preserved the generic vector's real-weight SHA-256 exactly: `46f7495acbbb563b65e75a7bea6b6dab22d4ca16b805b1558d37bc546fff072d`, with zero maximum and mean absolute difference. Its 70.701 ms median device time is 2.89 percent below the C106 three-row result and 12.55 percent below C106's same-run generic vector. The protected normal Qwen service recovered through a real completion after 412 probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c117-four-rows-full-scheduler-r1-20260902/` | Exact HIP Q4_K/Q5_K four-row full serving gate | The canonical deterministic AIME SHA1 `3302eda43396` passed. Three scheduler samples recorded 3,050.44 mean client prefill TPS, 48.45 decode TPS, and 397.32 ms warm TTFT. Three C4 rounds recorded 4,561.20 aggregate prefill TPS, 91.42 aggregate decode TPS, 1.103 s p99 TTFT, and 42.08 ms p99 token gap. The candidate is quality-preserving and improves substantially over C109 at the component level, but its primary warm single-request prefill remains below the 3,118.90 TPS qualified generic-vector baseline. It remains default-off while retaining its exact quality and tail evidence. The protected normal Qwen service recovered through a real completion after 458 probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c118-four-rows-q4-only-component-r1-20260902/` | Q4_K-only four-row HIP isolation | The Q4_K-only selector preserved the generic vector's real-weight SHA-256 exactly, with zero maximum and mean absolute difference, but measured 81.382 ms. This is slower than the same-run generic-vector component and 15.11 percent slower than the all-format C116 four-row candidate at 70.701 ms. The candidate is rejected before API testing because it cannot justify an end-to-end quality or TPS disruption. The protected normal Qwen service recovered through a real completion after 445 probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c119-five-rows-component-r1-20260902/` | All-format five-row HIP geometry screen | The five-row HIP kernel preserved the generic vector's real-weight SHA-256 exactly, with zero maximum and mean absolute difference, but measured 81.307 ms. It is slower than the same-run generic-vector component and 15.00 percent slower than the all-format C116 four-row candidate at 70.701 ms. The candidate is rejected before API testing. The protected normal Qwen service recovered through a real completion after 472 probes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-vision-20260902T165806Z/` | Gemma 4 extended multimodal quality control | The isolated native ROCm/HIP Gemma API passed the exact arithmetic text control, all seven extended image fixtures for red, green, blue, yellow, and spatial left/right/top distinctions, and the bounded visual description control. The 51-word visible response correctly described the red-left and blue-right image and measured 53.27 visible decode TPS. The repaired controller then restored Qwen and its authoritative health endpoint returned `status: ok`. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/qwen-restart-timing-c120-20260902/` | Qwen NVFP4 restart-to-completion control | The controller stopped the managed normal service, launched the documented native ROCm/HIP NVFP4 configuration, and retried a real deterministic completion until it succeeded. HTTP health became available in 5.849 seconds after launch, but the first successful completion was available only after 396.407 seconds across 381 completion probes. The final result file records `qwen_restart_request_timing=passed`; the authoritative service endpoint subsequently returned `status: ok` with `maintenance: serving`. This separates socket or health readiness from actual model readiness. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/qwen-tool-workload-c122-20260902.json` | Bounded native OpenAI tool-using coding control | The normal native ROCm/HIP Qwen API emitted a constrained `read_fixture` tool call, then emitted the exact constrained `apply_exact_patch` tool call with a `tool_calls` finish reason on both turns. The runner applied the patch only inside a fresh artifact sandbox, verified the resulting file content and SHA-256 `ba1a531f581d2e6094e978ed6f7aca7a8d92eeb62c6e7ad73ee692f7f18bc772`, and the final visible response was exactly `PATCH_APPLIED`. The three API turns used 1,103 prompt tokens and 444 completion tokens, with 26.64 aggregate end-to-end TPS. This non-streaming controller records end-to-end latency, not TTFT or token-gap timing. It proves bounded local tool execution only, not paper W2, W3, or W4 parity. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/qwen-q4-24h-c123-20260902/` | Isolated Q4 minute-cadence endurance attempt | Sessions 1 through 47 passed the deterministic three-turn state suite with `runner_swap_kib=0`. Session 48 also passed all three visible answers, with 379.67 ms mean TTFT, 422.39 ms maximum TTFT, and 24.64 ms maximum token gap, but the verified Q4 process group then reported `runner_swap_kib=128376`. The controller correctly stopped before session 49 and entered its recovery trap, so this is an explicit zero-swap endurance failure, not a 24-hour pass. The normal native ROCm/HIP Qwen service subsequently returned `status: ok` and `maintenance: serving`; the next isolated diagnostic records per-process swap ownership without relaxing the zero-swap acceptance gate. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/qwen-q4-swap-diagnostic-c124-20260902/` | Isolated Q4 process-scoped zero-swap diagnostic | All 60 of 60 minute-cadence deterministic three-turn sessions passed, and every FreeToken Q4 process-group sample, including postflight, reported `runner_swap_kib=0`. This diagnostic crossed C123's session-48 failure boundary without recurrence. Whole-host swap remained 1,635,212 to 1,667,068 KiB and is retained as host telemetry, not attributed to FreeToken. The all-session maximum TTFT was 52.663 s because the first request after candidate startup was cold; it is retained in the complete summary. The separately labelled sessions 2 through 60 steady-state view measured 409.75 ms mean, 412.85 ms p95, and 414.83 ms p99 and maximum turn TTFT, with 26.36 ms p99 token gap. The controller then restored normal Qwen to `status: ok`, `maintenance: serving`, and a real OpenAI-compatible completion ended with `RECOVERY_OK` and `finish_reason: stop`. This is a successful one-hour diagnostic, not a 24-hour endurance qualification. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/qwen-tool-workload-c125-streaming-20260902.json` | Bounded native streaming OpenAI tool-using coding control | The normal native ROCm/HIP Qwen API completed the constrained `read_fixture` and `apply_exact_patch` calls with `tool_calls` finish reasons, the runner applied and verified the sandbox repair with SHA-256 `ba1a531f581d2e6094e978ed6f7aca7a8d92eeb62c6e7ad73ee692f7f18bc772`, and the final visible content was `PATCH_APPLIED`. The three streamed calls used 1,103 prompt tokens and 438 completion tokens, with 25.23 aggregate end-to-end TPS, 4.75 s mean visible TTFT, 7.45 s maximum visible TTFT, and 35.83 ms p99 visible token gap. The first structured tool-call latencies were 3.15 s and 7.77 s. This is a bounded local API and tool-execution control, not paper W2, W3, or W4 parity. The authoritative health endpoint remained `status: ok` with `maintenance: serving` afterward. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c126-grouped-differential-20260902/` | Real-weight grouped-versus-vector Q4 numerical differential | The isolated 1,024-token, top-k-eight, 256-expert differential used actual packed Qwen Q4_K gate/up and Q5_K down weights. The first mismatch is Q4 gate/up before SwiGLU: storage differs, maximum absolute difference 0.031494, mean absolute difference 0.002055. The Q5 down path also differs when fed the identical vector intermediate, with maximum absolute difference 0.003540 and mean absolute difference 0.000157. The final routed output differs, maximum absolute difference 0.001343, so the grouped path remains default-off and no grouped API TPS claim is eligible. The strengthened controller recovered normal Qwen only after a real `READY` completion with `finish_reason: stop` and 257 completion tokens, at attempt 483; the health endpoint then returned `status: ok` with `maintenance: serving`. | + +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c130-single-expert-20260902/differential.json` | Single-expert grouped-versus-vector real-weight differential | The C130 diagnostic used the same actual Qwen layer-zero Q4_K gate/up and Q5_K down weights, 1,024 deterministic BF16 activation rows, and top-k eight, but assigned every route to one expert. Q4 gate/up still differed before SwiGLU, with 0.031250 maximum and 0.001667 mean absolute difference. Q5 down also still differed with the identical vector intermediate, with 0.002441 maximum and 0.000133 mean absolute difference. The final output differed by up to 0.005859. This excludes mixed-expert sorting and cross-expert route ordering as the primary cause. The numerical repair must therefore target the grouped matrix tile loading, quantized dot, scale, or reduction arithmetic. No grouped API TPS claim is eligible. The guarded controller first built and imported its native HIP extensions in the disposable candidate checkout, then transferred GPU ownership and began normal-service recovery. | + +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c131-q8sum-single-expert-20260902/differential.json` | Grouped Q8_1 sum-contraction repair screen | C131 retained the C130 real-weight, 1,024-token, single-expert differential while changing only the grouped Q4_K and Q5_K min-term contraction to recompute the packed Q8_1 integer sum and apply the primary Q8 scale in FP32, matching the vector route's formulation. Q4 gate/up maximum difference fell from 0.031250 to 0.00390625 and its mean difference fell from 0.001667 to 0.0000000297. Q5 down with the identical vector intermediate fell from 0.002441 to 0.00024414 maximum and 0.00000000171 mean difference. The final tensor still was not storage-equal, with 0.00024414 maximum difference, so this is not a quality pass and no API TPS claim is eligible. The remaining mismatch is consistent with a residual packed-dot or reduction-order difference, not the previously rounded stored Q8 sum term. | + +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c133-q8sum-residual-samples-20260902/differential.json` | Sparse grouped residual coordinate screen | C133 retained C131's Q8-sum correction and added bounded coordinate samples. Q4 gate/up retained 936 differing elements out of 8,388,608, and Q5 down with an identical vector intermediate retained 2,904 out of 16,777,216. Sampled Q4 and Q5 residuals repeat over groups of eight routed rows while occurring at fixed output lanes, for example Q4 lane 641 and Q5 lane 1422. This excludes token order, mixed-expert sorting, and route scatter as the residual source. The remaining repair scope is the row-local grouped packed-dot arithmetic, tile representation, or reduction sequence. Outputs remain non-identical and no grouped API TPS claim is eligible. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c134-four-wave-control-20260902/differential.json` | Four-wave grouped numerical control | C134 rebuilt the same Q8-sum-repaired grouped Q4_K/Q5_K kernels in a fresh isolated ROCm extension with `FREETOKEN_GGUF_MOE_K_WARPS=4`, then repeated the single-expert, 1,024-token real-weight differential. It reproduced C131's residual counts and magnitudes exactly: Q4 gate/up differed in 936 of 8,388,608 elements with a 0.00390625 maximum difference, Q5 down with identical vector intermediate differed in 2,904 of 16,777,216 elements with a 0.000244140625 maximum difference, and the final tensor differed in 3,334 of 2,097,152 elements. This rules out the four-versus-eight routed-wave count as the primary residual source. The remaining repair scope is the Q4_K and Q5_K grouped tile-scale or packed primary-dot sequence shared by both builds. No grouped API TPS claim is eligible. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c135-q8sum-grouped-api-quality-20260902/quality-aime.json` | Q8-sum-repaired grouped API quality gate | The repaired grouped prefill path was built in a fresh isolated ROCm extension and enabled only for multi-token prompt processing; decode remained on the qualified vector route. The candidate reached a real API completion, but the deterministic greedy AIME control returned output SHA1 `e10880eae5f5` instead of the qualified `3302eda43396`. The quality harness marked the result failed before scheduler, latency, or TPS tests, so there is no candidate performance claim. This proves the sparse residual is still sufficient to change model-level output. The grouped path remains default-off and the controller began protected-service recovery. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c138-q5-four-rows-component-20260902/` | Exact Q5_K-only four-row HIP component screen | C138 isolated the Q5_K portion of the earlier exact four-row vector candidate, while retaining the qualified generic Q4_K and all other routes. With real layer-zero Qwen weights, 1,024 deterministic BF16 rows, 256 experts, and top-k eight routing, it reproduced the generic vector output SHA-256 `46f7495acbbb563b65e75a7bea6b6dab22d4ca16b805b1558d37bc546fff072d` exactly, with zero maximum and mean absolute difference. Median device time improved from 81.093 ms for the same-run generic vector control to 74.634 ms, a 7.97 percent component improvement. This is a component screen only, not an API TPS result. The candidate is eligible for the exact full API quality and performance gate after protected-service recovery completes. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c139-q5-four-rows-full-20260902T214440Z/` | Exact Q5_K-only four-row full API gate | C139 enabled only the exact Q5_K four-row vector treatment. The canonical deterministic AIME SHA1 `3302eda43396` passed. Three scheduler samples recorded 3,130.30 mean client prefill TPS, 48.20 decode TPS, and 387.19 ms warm TTFT. Three C4 rounds recorded 4,758.05 aggregate prefill TPS, 94.80 aggregate decode TPS, 1.025 s p99 TTFT, and 39.93 ms p99 token gap. Compared with the qualified generic-vector baseline, single-request prefill rose 0.37 percent, C4 aggregate prefill rose 4.39 percent, C4 aggregate decode rose 3.88 percent, p99 TTFT fell 7.1 percent, and p99 token gap fell 9.96 percent. The slight 1.91 percent single-request decode reduction remains separately recorded. The protected normal Qwen service recovered through a real `READY` completion with `finish_reason: stop` after 439 probes. This candidate is quality-preserving and improves the primary prefill metric, making it eligible for extended-tail and endurance qualification. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c140-q5-four-rows-endurance-20260902T215935Z/` | Q5_K-only endurance preflight rejection | C140 used the C139 Q5-only four-row configuration with 0.30 memory ratio and prefill overlap enabled, but the process-scoped endurance gate rejected it before session one. The candidate HTTP parent had 351,704 KiB `VmSwap`, while all three helpers remained at zero. This is a valid zero-swap failure rather than a quality, TPS, or endurance result. The controller stopped the isolated server and restored normal Qwen. The failure motivated a separately tested reversible swap-drain repair rather than weakening the process-scoped invariant. | +| 2026-09-02 | `/home/operator/freetoken-amd/artifacts/q4-c141-q5-swapdrain-proof-20260902T221030Z/` | Q5_K-only swap-drain endurance proof | C141 first drained swap after stopping normal Qwen, recorded 56 GiB available and 0 B host swap, then started the same Q5-only, 0.30-memory-ratio, overlap-enabled candidate. Its preflight measured zero swap across the HTTP parent and all workers. The exact three-turn state suite passed, postflight process-group swap remained zero, and the summary recorded zero host swap throughout. The initial cold request took 53.856 s; later turns measured 1.292 s and 425.8 ms TTFT with a 24.58 ms maximum visible token gap. Swap was restored before normal-service recovery, which concluded with a real `READY` completion. This is a bounded repair proof, not a 24-hour qualification. | +| 2026-09-03 | `/home/operator/freetoken-amd/artifacts/q4-c142-q5-swapdrain-endurance-20260902T222206Z/` | Q5-only four-row 1,440-session endurance qualification | C142 completed exactly 1,440 of 1,440 minute-cadence sessions. Every session JSON was valid and passed the deterministic three-turn state suite; the summary recorded zero failures, zero candidate process-group swap, and zero whole-host swap. Mean maximum-turn TTFT was 414.775 ms, p95 was 379.591 ms, p99 was 389.743 ms, and the retained maximum was 53.245 s for the cold-start boundary. Mean maximum visible-token gap was 25.252 ms, with p95 26.073 ms and p99 27.758 ms. The controller completed its terminal cleanup and preserved Q4 health plus protected normal-service recovery artifacts. This is an endurance and stability qualification, not a per-session prefill-TPS measurement. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/w1-paper-inspired-five-sample-20260904T094252/` | Pinned paper-inspired W1 AIME five-sample control | Five independent read-only warm samples used the pinned `math-ai/aime25` fixture revision, problem 0, a 54-token prompt, greedy sampling, and a forced 127-token completion. All five returned the expected output SHA1 `0acef4eab6f4`. Mean client-visible decode was 26.707 tokens/s, median 26.814, minimum 23.975, and maximum 28.531. Mean TTFT was 447.481 ms and mean p99 event gap was 44.171 ms. This is a reproducible W1-style control, not strict paper replication because the paper's original prompt, cache policy, and exact runner contract remain unpublished. | + +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-vision-20260904T144109Z/` | Gemma 4 repeated extended multimodal ROCm/HIP control | After building the missing HIP native extensions in the isolated candidate checkout, the text arithmetic gate passed (`323`, 30 prompt tokens, 4 completion tokens). The extended image suite passed all 21 cases across three repetitions: seven fixtures, exact color and spatial checks, and valid visible outputs. The visual-description control passed with 55 words, 309 prompt tokens, 64 completion tokens, 1,139 ms TTFT, and 52.57 visible decode TPS. The text control measured 51.18 decode TPS after its expected cold 53.11 s TTFT. The protected Qwen service was restored and verified `status: ok`, `maintenance: serving`. This closes repeated Gemma 4 functionality and visible-TPS evidence, but remains a bounded control rather than a full Gemma endurance or strict paper workload. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260904T150838Z/` | Gemma 4 fixed-length text performance matrix | After the mandatory arithmetic quality gate, one warmup and five fixed-length streamed samples completed successfully at 34 prompt and 127 completion tokens. The scored samples recorded mean TTFT 203.07 ms, mean client prefill 174.27 tokens/s, mean decode 50.87 tokens/s, median decode 53.21 tokens/s, p95 decode 53.67 tokens/s, and aggregate p99 token gap 132.36 ms. The first scored sample retained a 296.39 ms TTFT and 40.93 tokens/s decode, while samples 2 through 5 were steady at 53.17 to 53.67 tokens/s. The protected Qwen service returned `status: ok` and `maintenance: serving` after teardown. This closes the first repeatable Gemma text prefill/decode matrix, but not Gemma concurrency, long-context, endurance, or matched llama.cpp parity. | + +## Open work + +| ID | Required evidence | State | +| --- | --- | --- | +| P0 | Complete paper protocol fields or explicit unresolved record | Completed: `gmktec-evo-x2-paper-protocol-gap.md` separates the hardware and workload facts published by the paper from the missing strict-replication fixtures, traces, configuration, and raw-sample details. | +| P1 | Harness manifest and tail-summary validation | Completed: tail summaries and clean runtime manifest validated | +| P2 | Five-sample Qwen NVFP4 warm and cold baseline | Completed: warm short and medium baselines, long-context cache-hit and forced-cold-prefill controls, time-shared llama.cpp control, recovered FreeToken repeat, and real restart-to-first-completion timing are all recorded. Cold restart health readiness was 5.849 seconds and true completion readiness was 396.407 seconds. | +| P3 | Versioned Qwen and Gemma quality suite | Completed: Qwen three-case suite plus Gemma text, extended seven-fixture vision, and bounded visual-description quality controls passed. | +| P4 | Paper-inspired W1 to W4 agent workloads | Bounded state-retention and bounded native OpenAI tool-call plus sandbox-patch controls completed. The paper's OpenCode SWE-bench W2, Claude Code W3 with 56K to 65K contexts, and OpenClaw W4 with its 24.5K context floor remain unreplicated because their external harnesses, fixtures, and required context capacities are not yet available in this controlled campaign. | +| P5 | Tail-latency matrix and 24-hour endurance | Qwen long-context and 1/2/4/8-client tail controls are complete, C142 completed the separate 1,440-session minute-cadence endurance qualification with zero candidate and host swap, and the standardized cross-model cold/warm and concurrency matrix is consolidated. Gemma now also has matched five-sample text, four-client concurrency, corrected long-context, and 30-session bounded-endurance controls. A full 24-hour Gemma protocol remains optional publication evidence, not a functional gate. | +| P6 | 284B capacity manifest and guarded admission | Metadata gate completed: the pinned official payload is 155.425 GiB and the current observed host state yields a 4 GiB authoritative model budget after explicit headroom, so the full load is rejected. The exact expert-slice harness is prepared, but no full-model quality or TPS result is claimed. | +| P7 | Strict NVIDIA reference run | Blocked on reference hardware and missing paper fields | + +| P8 | Real-shape DeepSeek expert transfer slice | Completed in isolation on native ROCm: 80,216,064 bytes across six layer-0 experts and 36 tensors. The final three H2D samples averaged 76.645 GiB/s; post-cold H2D averaged 73.987 GiB/s and D2H averaged 64.073 GiB/s. The protected Qwen service remained healthy after recovery. This is transfer-path evidence only, not a 284B serving result. | +| P9 | Larger real-shape DeepSeek expert transfer slice | Completed in isolation: 213,909,504 bytes across 16 layer-0 experts and 96 tensors. Final-three H2D averaged 77.561 GiB/s; post-cold D2H averaged 64.762 GiB/s. No material H2D collapse was observed as the batch grew. | +| P10 | Cross-shard, multi-layer DeepSeek expert transfer slice | Completed in isolation: 160,432,128 bytes across six experts in layers 0 and 1, spanning shards 2 and 3. Final-three H2D averaged 77.976 GiB/s; post-cold D2H averaged 64.622 GiB/s. Cross-shard loading passed and the protected service remained healthy. | +| P11 | Expert-ID route-churn transfer controls | Completed four two-layer groups: experts 0 to 5, 16 to 21, 32 to 37, and 64 to 69. Post-cold H2D ranged from 77.758 to 78.577 GiB/s and D2H from 64.720 to 64.974 GiB/s. No material expert-ID sensitivity was observed. | + +## 2026-09-04 transfer and offload prototypes + +| UTC date | Evidence | Category | Outcome | +|---|---|---|---| +| 2026-09-04 | `docs/gmktec-evo-x2-rocm-transfer-prototype.md` | Contiguous ROCm transfer | Read-only PyTorch ROCm 10 prototype measured 79.79 GB/s host-to-device and 70.24 GB/s device-to-host for 64 MiB copies. This is a best-case bulk-copy bound. | +| 2026-09-04 | `docs/gmktec-evo-x2-expert-block-prototype.md` | Serialized scattered expert-like transfers | Random synchronized 64 KiB blocks reached 5.009 GB/s, while 4 KiB blocks reached 0.167 GB/s. Independent small misses are rejected as an offload strategy. | +| 2026-09-04 | `docs/gmktec-evo-x2-batched-expert-transfer-prototype.md` | Grouped expert-like transfers | Grouping 16 blocks raised transfer-only throughput to 29.79 GB/s, but total round throughput was 12.84 GB/s after CPU staging. Batching is necessary but not sufficient. | +| 2026-09-04 | `docs/gmktec-evo-x2-overlap-prototype.md` | Naive stream overlap | Per-group stream and event orchestration regressed wall time and was rejected. | +| 2026-09-04 | `docs/gmktec-evo-x2-persistent-overlap-prototype.md` | Persistent double-buffered overlap | Persistent streams and buffer reuse still regressed wall time, so Python-level overlap is rejected as the primary strategy. | +| 2026-09-04 | `docs/gmktec-evo-x2-hip-gather-prototype.md` | Compiled device-side gather | A compiled HIP gather from device memory reached 281.916 GB/s, establishing a fast kernel-side ceiling. | +| 2026-09-04 | `docs/gmktec-evo-x2-mapped-host-gather.md` | Mapped-host descriptor gather | One HIP descriptor-driven kernel gathered 64 random 64 KiB blocks from mapped host memory at 112.908 GB/s. This is the first credible low-level direction for reducing host intervention, but it is not model-TPS evidence. | +| 2026-09-04 | `docs/gmktec-evo-x2-fused-moe-prototype.md` | Fused expert-row compute prototype | A native HIP kernel processed 64 mapped-host expert rows, unpacked synthetic signed-int4 weights, performed FP32 dot products and reductions, and completed 100 timed launches at 32.672 GB/s effective packed-weight throughput. This is a combined transfer and compute baseline, not model TPS or production NVFP4 evidence. | +| 2026-09-04 | `docs/gmktec-evo-x2-nvfp4-shape-prototype.md` | Production-shape NVFP4 fused decode prototype | The production FreeToken Triton NVFP4 Marlin entry point executed the representative 8-expert, hidden 1,152, intermediate 512, top-k 8 shape through HIP. Ten timed calls returned finite output; steady-state mean was 0.125461 ms for gate/up plus down GEMV and activation, equivalent to about 56.4 GB/s of packed input traffic. Random device tensors were used, so this is kernel-path evidence, not model TPS or quality evidence. | +| 2026-09-04 | `docs/gmktec-evo-x2-nvfp4-marlin-parity-test.md` | NVFP4 Marlin numerical parity | Focused HIP execution of the repository's production NVFP4 Marlin tests passed 2 tests and skipped 4 optional or unrelated variants. The tests covered Marlin-versus-LUT output parity and cache reload after a full-layer prefill. This qualifies the path for further candidate testing but makes no TPS claim. | +| 2026-09-04 | `docs/gmktec-evo-x2-real-qwen-nvfp4-layer0-parity.md` | Real Qwen NVFP4 layer-zero parity | The source loader captured actual layer-zero Qwen3.6 NVFP4 packed weights, FP8 block scales, and FP16 global scales before stopping. Marlin versus LUT decode produced exactly identical output with zero maximum and mean absolute difference. The final eight of ten Marlin samples averaged 0.203962 ms for the routed layer operation, or about 61.7 GB/s of routed packed input traffic. This is real-weight component evidence, not model TPS. | +| 2026-09-04 | `docs/gmktec-evo-x2-real-qwen-nvfp4-route-matrix.md` | Real Qwen NVFP4 routed-expert matrix | Three deterministic real-weight layer-zero cases passed with finite output across contiguous, scattered, and repeated expert routes. Contiguous and scattered routes matched exactly; repeated routes differed by only 1.907e-6 maximum absolute value from reduction order. Marlin means were 0.187887, 0.114801, and 0.093191 ms. This qualifies route handling for an isolated serving candidate but is not end-to-end TPS evidence. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T180912Z/` | NVFP4 Marlin tile-8 API candidate | Five API samples completed at 29.8357 mean decode TPS and 29.8282 median TPS, but the deterministic AIME hash failed (`expected cd580f4978fb`, observed `1cae5bae914f`). The candidate was rejected, its source was restored to `BLOCK_N=16`, and the protected Qwen service recovered with `status: ok` and `maintenance: serving`. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T182532Z/` | NVFP4 Marlin warp-count API candidate | Five API samples completed at 30.2446 mean decode TPS and 30.2387 median TPS, but the deterministic AIME hash failed (`expected cd580f4978fb`, observed `1cae5bae914f`). The candidate was rejected, its source was restored to `_DECODE_MARLIN_WARPS = 4`, and the protected Qwen service recovered with `status: ok` and `maintenance: serving`. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T184547Z/` | NVFP4 Marlin `num_stages=2` API candidate | Five API samples completed at 30.0373 mean decode TPS and 30.0362 median TPS, but the deterministic AIME hash failed (`expected cd580f4978fb`, observed `1cae5bae914f`). The candidate was rejected, its source was restored, and the protected Qwen service recovered with `status: ok` and `maintenance: serving`. | +| 2026-09-04 | `/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T182532Z/` | NVFP4 Marlin warp-8 API candidate | Five API samples completed at 30.2446 mean decode TPS and 30.2387 median TPS, but the deterministic AIME hash failed (`expected cd580f4978fb`, observed `1cae5bae914f`). The candidate was rejected, its source was restored to `_DECODE_MARLIN_WARPS = 4`, and the protected Qwen service recovered with `status: ok` and `maintenance: serving`. | + +## 2026-09-05 Gemma text throughput qualification + +| UTC date | Evidence | Category | Outcome | +|---|---|---|---| +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T084837Z/` | Native ROCm/HIP Gemma4 Q4 text matrix | The isolated native FreeToken Gemma4 Q4 server passed its mandatory arithmetic quality gate and five fixed-length streamed samples. Mean decode was 53.0762 TPS (median 53.0353, p95 53.2595), mean prefill was 174.582 TPS, mean TTFT was 196.327 ms (p95 233.903 ms), and p99 token gap was 21.602 ms. The candidate was shut down and the protected Qwen service recovered to authoritative `status: ok`, `maintenance: serving`. The existing llama.cpp Gemma control uses a different long repeated prompt and is therefore not an apples-to-apples comparison; a matched-prompt control remains required. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/gemma4-llamacpp-vision-20260905T091532Z/` | Matched ROCm10 llama.cpp Gemma4 Q4 text control | The same fixed prompt, 128-token cap, five samples, and matrix verifier passed. Mean decode was 56.8293 TPS, mean prefill 737.004 TPS, mean TTFT 47.674 ms, and p99 token gap 18.141 ms. Against native FreeToken's 53.0762 decode TPS, llama.cpp was 7.07 percent faster; its prefill was 4.22 times higher and mean TTFT was 75.7 percent lower. Both controls passed their arithmetic quality gate. The protected Qwen service recovered to authoritative `status: ok`, `maintenance: serving`. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T095600Z/` | Gemma4 MoE prefill-overlap candidate | The explicit `FREETOKEN_GEMMA4_PREFILL_OVERLAP=1` candidate passed the arithmetic quality gate and five fixed-length samples with `prefill_overlap=True`. Mean decode was 53.4661 TPS, mean prefill 172.967 TPS, mean TTFT 196.590 ms, and p99 token gap 21.604 ms. Relative to the default candidate, decode improved only 0.74 percent while prefill regressed 0.93 percent and TTFT was unchanged. The candidate is rejected for promotion. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T100619Z/` and `/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T101619Z/` | Gemma4 ROCm Triton prefill warmup candidate | Two independent five-sample candidates with `FREETOKEN_ROCM_PREFILL_WARMUP=1` passed the arithmetic quality gate. Run one included a cold decode outlier, but run two was stable at 53.3319 mean decode TPS, 180.5788 mean prefill TPS, 189.904 ms mean TTFT, and 21.489 ms p99 token gap. Samples 2 through 5 in run two averaged approximately 188.35 prefill TPS and 180.52 ms TTFT. Relative to the default matrix, steady prefill improved about 3.5 percent, TTFT improved about 3.4 percent, decode remained within normal run variation, and token-gap tail did not regress. Warmup is promoted as the Gemma launcher default, with an environment override retained for A/B tests. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T102706Z/` | Gemma4 0.50 unified-memory expert-cache candidate | The candidate used `FREETOKEN_GEMMA4_MEMORY_RATIO=0.50` with warmup enabled and passed the arithmetic quality gate. Initialization left 17.78 GiB free and resolved 209,165 cache pages. Mean decode was 52.7735 TPS, mean prefill 177.688 TPS, mean TTFT 193.646 ms, and p99 token gap 21.671 ms. It was slower than the qualified 0.35 plus warmup configuration on all primary metrics, so the higher memory ratio is rejected. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T103652Z/` | Gemma4 prompt-scaling matrix, 16 repeated units | The native FreeToken candidate passed quality with a 544-token prompt and five scored samples. Mean decode was 48.1441 TPS, mean prefill 2,810.98 TPS, mean TTFT 194.884 ms, and p99 token gap 25.625 ms. Samples 2 through 5 averaged 2,921.08 prefill TPS and 186.23 ms TTFT. The short 34-token matrix's approximately 180 TPS prefill is therefore dominated by fixed request overhead and must not be treated as the model's steady-state long-prefill rate. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/gemma4-llamacpp-vision-20260905T104712Z/` | Matched Gemma4 544-token ROCm10 llama.cpp control | The same 16-repeat prompt shape, five samples, 128-token cap, and matrix verifier passed. llama.cpp tokenized the prompt as 545 tokens versus FreeToken's 544. Mean decode was 54.3743 TPS, mean prefill 7,413.30 TPS, mean TTFT 73.566 ms, and p99 token gap 18.723 ms. Against native FreeToken, llama.cpp was 12.9 percent faster on decode, 2.64 times faster on prefill, and 62.2 percent lower on TTFT. This establishes a genuine long-prefill gap after removing short-prompt overhead distortion. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T105149Z/` and `/home/operator/freetoken-amd/artifacts/gemma4-llamacpp-vision-20260905T110157Z/` | Matched Gemma4 four-client concurrency control | Both runtimes used the 16-repeat prompt, 128-token cap, four synchronized clients, three rounds, and passed all 12 requests. FreeToken achieved 22.188 aggregate decode TPS, 23.538 mean per-request decode TPS, 369.040 ms mean TTFT, and 68.291 ms aggregate p99 token-gap summary. llama.cpp achieved 21.310 aggregate decode TPS, 54.558 mean per-request decode TPS, 3.678 s mean TTFT, and 22.929 ms aggregate p99 token-gap summary. FreeToken's aggregate throughput was 4.1 percent higher and its mean TTFT was substantially lower under this contention pattern, despite lower isolated per-request decode speed. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T110609Z/` and `/home/operator/freetoken-amd/artifacts/gemma4-llamacpp-vision-20260905T111607Z/` | Matched Gemma4 eight-client concurrency stress control | Both runtimes used the 16-repeat prompt, 128-token cap, eight synchronized clients, three rounds, and passed all 24 requests. FreeToken achieved 14.874 aggregate decode TPS, 3.178 s mean TTFT, 6.124 s p95 TTFT, and 103.215 ms aggregate p99 token-gap summary. llama.cpp achieved 11.836 aggregate decode TPS, 8.484 s mean TTFT, 16.885 s p95 TTFT, and 20.268 ms aggregate p99 token-gap summary. FreeToken's aggregate throughput was 25.7 percent higher and its mean TTFT 62.5 percent lower, although the absolute FreeToken tail latency is no longer ideal at this load. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T113404Z/` | Gemma4 eight-client `max_running_req=8` scheduler candidate | The candidate passed all 24 requests with the same 16-repeat prompt and eight clients. Mean aggregate decode was 14.117 TPS, mean per-request decode 14.793 TPS, mean TTFT 476.434 ms, p95 TTFT 624.019 ms, and aggregate p99 token-gap summary 124.633 ms. Compared with the qualified `max_running_req=4` profile, TTFT fell approximately 85 percent and p95 TTFT approximately 90 percent, while aggregate throughput fell 5.1 percent and per-request decode fell substantially. This is retained as a latency-oriented alternate profile, not a universal default. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T114510Z/` | Gemma4 eight-client `max_running_req=6` scheduler candidate | The candidate passed all 24 requests. Mean aggregate decode was 15.219 TPS, mean per-request decode 22.090 TPS, mean TTFT 2.192 s, p95 TTFT 7.599 s, and aggregate p99 token-gap summary 90.867 ms. It slightly exceeded max-4 aggregate throughput but had highly variable tail latency and no consistent interactive advantage, so it remains an inconclusive alternate rather than a promoted profile. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T112013Z/` and `/home/operator/freetoken-amd/artifacts/gemma4-llamacpp-vision-20260905T112959Z/` | Matched Gemma4 two-client concurrency control | Both runtimes used the 16-repeat prompt, 128-token cap, two synchronized clients, three rounds, and passed all six requests. FreeToken achieved 31.039 aggregate decode TPS, 33.780 mean per-request decode TPS, and 359.816 ms mean TTFT. llama.cpp achieved 35.803 aggregate decode TPS, 54.956 mean per-request decode TPS, and 1.264 s mean TTFT. llama.cpp's aggregate throughput was 15.4 percent higher at two clients, while FreeToken had 71.5 percent lower mean TTFT. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/qwen-gguf-raw-20260905T121136Z/raw-quality.json` | Same-checkpoint Qwen3.6 Q4_K_M GGUF FreeToken raw-prompt control | The exact Q4_K_M GGUF checkpoint and tokenizer were used with the caller-rendered 54-token prompt and a 256-token cap. The expected answer path passed, with 50.0169 decode TPS across 255 generated tokens. TTFT was 54.311 s because this was a cold FreeToken model initialization. The full output hash is retained and differs from the llama.cpp control. The protected service was restored afterward. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/qwen-llama-raw-20260905T122310Z/raw-quality.json` | Same-checkpoint Qwen3.6 Q4_K_M GGUF llama.cpp ROCm10 raw-prompt control | The exact same Q4_K_M GGUF checkpoint, tokenizer, caller-rendered prompt, and 256-token cap were used. The expected answer path passed, with 49.3875 decode TPS across 256 generated tokens. Loaded-control TTFT was 234.038 ms. The full output hash is retained and differs from FreeToken. The protected service was restored afterward. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/qwen-gguf-warm-matrix-20260905T122817Z/` | Same-checkpoint Qwen3.6 Q4_K_M GGUF warmed FreeToken matrix | One loaded FreeToken server handled five consecutive caller-rendered raw-prompt requests. All five returned the expected answer path and the same output hash. Decode was 45.2713 TPS on the first request and 49.3630, 49.3096, 49.6545, and 49.4156 TPS on requests 2 through 5. Mean decode was 48.6028 TPS across all samples and 49.4357 TPS after the first request. Mean TTFT was 982.17 ms including the first request and 424.26 ms for requests 2 through 5. The protected service was restored and returned `status: ok`, `maintenance: serving`. | +| 2026-09-05 | `/home/operator/freetoken-amd/artifacts/qwen-llama-warm-matrix-20260905T124007Z/` | Same-checkpoint Qwen3.6 Q4_K_M GGUF warmed llama.cpp ROCm10 matrix | One loaded llama.cpp server handled five consecutive caller-rendered raw-prompt requests. All five returned the expected answer path and the same output hash. Decode was 48.8686 TPS on the first request and 49.1575, 49.1606, 49.1887, and 49.2019 TPS on requests 2 through 5. Mean decode was 49.1155 TPS across all samples and 49.1772 TPS after the first request. Mean TTFT was 92.07 ms including the first request and 58.83 ms for requests 2 through 5. The protected service was restored and returned `status: ok`, `maintenance: serving`. | + +## 2026-09-05 regression contract repair + +The HIP GGUF compiler helper records the reviewed default one-row launch shape +explicitly as `-DGGML_CUDA_MMV_Y=1`. Two host-side tests still expected the +older `-O3`-only flag list, which would have made the test contract disagree +with the actual build key and obscured the default AMD kernel shape. Commit +`803a3ce` updates both expectations and documents why the explicit flag is +required. A dependency-free contract harness exercised default, explicit +multi-target, two-row candidate, and invalid four-row rejection cases, with +four of four checks passing. Full pytest collection on Windows remains +environment-limited because the local test interpreter does not have PyTorch; +the authoritative ROCm runtime and protected service remained healthy after +the validation. +## 2026-09-05 current-branch MMV-Y4 requalification + +The opt-in `FREETOKEN_GGUF_MMV_Y=4` build from commit `ff76ede` was tested in +an isolated checkout after explicitly stopping the protected Qwen service. +The candidate reached API readiness with 56 GiB free before model loading and +23.07 GiB free after initialization. Three scheduler-shaped throughput samples +completed without failure: 45.4603 mean decode TPS, 2,753.3639 mean +client-observed prefill TPS, 0.0927 decode-TPS standard deviation, and 43.781 +ms maximum token gap. The canonical Q4 output hash was `3302eda43396`, selected +with the verifier's explicit `--expected-sha1 3302eda43396` contract option; +the historical `0acef4eab6f4` default remains separate. Relative to the +accepted current Q4 control near 48.28 decode TPS, Y4 was approximately 5.8 +percent slower and was rejected for promotion. A prior protected NVFP4 +re-anchor recorded `cd580f4978fb` under a separate contract, while this +artifact retains the historical Q4 contract `3302eda43396`. The default remains +Y1. The +protected Qwen service was restored and returned `status: ok` with +`maintenance: serving`. +## 2026-09-05 explicit AIME contract probe + +The repaired verifier was run against the healthy protected Qwen service with +`--expected-sha1 cd580f4978fb` and preserved the complete result at +`/home/operator/freetoken-amd/artifacts/protected-aime-contract-selector-20260905T150000Z.json`. +The request contract produced observed SHA1 `0acef4eab6f4`, so the verifier +correctly returned `failed` for that selected expectation without changing the +service. This is evidence that `cd580f4978fb` belongs to a different source or +request contract, not evidence of a model regression. Future quality artifacts +must record the exact model revision, prompt, tokenizer, sampling policy, and +expected fingerprint together. +## 2026-09-05 same-source MMV-Y1 control + +The current branch was rerun with `FREETOKEN_GGUF_MMV_Y=1` using the same +checkout, model, scheduler workload, memory ratio, and explicit Q4 quality +contract as the Y4 run. Three samples passed with 45.3341 mean decode TPS, +2,682.4559 mean client-observed prefill TPS, 0.0619 decode-TPS standard +deviation, and zero failed samples. The canonical Q4 quality check passed with +`--expected-sha1 3302eda43396`. The paired Y4 result was 45.4603 decode TPS +and 2,753.3639 prefill TPS, only 0.28 percent higher. Y4 is definitively +rejected for promotion. The protected service was restored and returned +`status: ok` with `maintenance: serving`. + +## 2026-09-05 upstream synchronization and ROCm guard regression + +The AMD branch was synchronized with the six commits newly present on +`upstream/main`; the merge completed without conflicts and the branch is now +zero commits behind upstream. This includes upstream's exact Triton sampling +correction and current repository metadata without changing the AMD runtime +scope. + +During the synchronization review, the ROCm test suite exposed a stale local +assertion: the implementation correctly excludes 240-byte and 400-byte rows +from the legacy 128-byte AOT copy catalog, but the test still asserted that +those rows were present. Commit `8ad5b63` changes the assertion to require +their absence, matching the implementation and strict no-JIT behavior. + +On the ROCm 10 environment, using the exact pushed branch, the focused guard +run passed 5 tests. It covered HIP runtime gating, HIP GGUF build flags, the +fused-copy grid selector, the legacy AOT catalog predicate, and the related +regression contracts. Native pinned-extension tests were not counted in that +run because the isolated checkout did not contain a freshly built host +extension; no production service was changed. + +The raw compiler output and extension checksums are preserved at +`/home/operator/freetoken-amd/artifacts/rocm-host-extension-build-5a56629/`. + +The same isolated checkout then ran a fresh `setup.py build_ext --inplace` +under PyTorch `2.13.0+rocm10.0.0` and HIP `7.15.26333`. The build compiled +`_pinned_tensor`, `_cpu_moe`, and `_ple_store`; the two GPU-facing host +extensions were explicitly compiled with `FREETOKEN_USE_ROCM=1` and linked +against `libamdhip64.so.7`. After the build, the complete focused suite passed +15 tests, including the pinned-memory, host-bank, AOT catalog, HIP runtime, +and GGUF flag checks. The protected serving process was not stopped or +modified for this validation. + +## 2026-09-05 FP8 GEMV tile32 repeatability gate + +The opt-in NVFP4 FP8 GEMV output-row tile candidate was evaluated with +`FREETOKEN_FP8_GEMV_BLOCK_N=32`, `FREETOKEN_FP8_GEMV_NUM_WARPS=1`, and +`FREETOKEN_FP8_GEMV_SCALE_ACTIVATION=0`. The candidate used the exact +prebuilt ROCm cache `kernel-cache-rocm-gfx1151-d6ee8cef479c` and +`FREETOKEN_DISABLE_JIT=1`; it did not JIT compile during the model-level run. + +The paired current baseline completed three scheduler samples at 28.1011 mean +decode TPS, 28.1072 median TPS, and 0.0260 TPS standard deviation. Its raw +artifacts are preserved at +`/home/operator/freetoken-amd/artifacts/qwen-fp8-paired-baseline-20260905T170100Z/`. + +The corrected tile32 repeat completed three of three samples at 28.4818 mean +decode TPS, 28.4799 median TPS, and 0.0148 TPS standard deviation. The +candidate artifact is +`/home/operator/freetoken-amd/artifacts/qwen-fp8-tile32-nvfp4-repeat2-scheduler-20260905T192000Z/`. +Against the paired baseline, this is a 1.35 percent mean decode improvement +with lower variation. The earlier independent candidate run recorded 28.4019 +TPS and is preserved at +`/home/operator/freetoken-amd/artifacts/qwen-fp8-tile32-nvfp4-20260905T083800Z/`. + +The repeated candidate also passed the canonical AIME quality gate. It +returned answer `70`, output SHA1 `0acef4eab6f4`, 127 completion tokens, +28.9804 decode TPS, 394.7 ms TTFT, 34.3702 ms event p50, and 37.3475 ms +event p99. The quality artifact is stored alongside the repeat scheduler +artifact as `aime-quality.json`. + +One earlier attempt is intentionally classified as an infrastructure failure, +not a performance result: an abbreviated cache directory omitted the required +`freetoken__index_4096_4_128_1_false` object and the server failed closed with +JIT disabled. The corrected exact cache path resolved this issue without +enabling JIT. The protected Qwen service was restored afterward and returned +`status: ok` with `maintenance: serving`. + +This evidence clears the repeatability and deterministic-quality gates for an +isolated tile32 promotion review. The production launcher remains unchanged at +tile16 until deployment policy is reviewed separately; the measured gain is +material but far below the original 50 percent campaign aspiration. + +## 2026-09-05 FP8 GEMV tile32 concurrency gate + +The tile32 candidate was evaluated with the established four-client, +three-round Qwen concurrency control using the same 48-unit scheduler prompt, +256-token cap, greedy sampling, and no-JIT cache policy. A first attempt ran +without the required scheduler prewarm and produced a cold-start result with +18.7875 seconds p99 TTFT. That result is retained as diagnostic evidence, but +it is not valid for promotion or apples-to-apples comparison. + +The valid warmed run performed the scheduler prewarm first, then completed all +three rounds and all twelve requests. Its complete artifact is +`/home/operator/freetoken-amd/artifacts/qwen-fp8-tile32-c4-warm-20260905T220000Z/c4.json`. +Tile32 recorded 53.1579 mean aggregate decode TPS, 53.4003 median round +aggregate TPS, 0.9207 seconds p99 TTFT, and 76.8555 ms p99 token gap. The +established qualified Qwen C4 profile is approximately 94.80 aggregate decode +TPS, 1.025 seconds p99 TTFT, and 39.93 ms p99 token gap. The warmed candidate +therefore fails the concurrency promotion gate because throughput is materially +lower and token-gap tail latency is materially worse, despite successful +requests and passing deterministic response checks. Tile32 remains default-off. + +The initial cold run and the warmed run are both preserved. The exact candidate +process group was terminated after testing, residual memory pressure was +cleared, and the protected service was restarted. The service was subsequently +verified with `status: ok` and `maintenance: serving`. + +## 2026-09-06 exact-branch ROCm host-extension regression + +The pushed `amd-rocm-gfx1151` head at commit `e7a5a92` was fetched into an +isolated validation worktree on the GMKtek EVO-X2. The protected Qwen service +was not stopped or modified. Under PyTorch `2.13.0+rocm10.0.0`, ROCm +`/opt/rocm-10.0`, HIP `7.15.26333`, and `FREETOKEN_USE_ROCM=1`, +`setup.py build_ext --inplace` compiled and linked `_pinned_tensor`, +`_cpu_moe`, and `_ple_store`. The GPU-facing extensions linked against +`libamdhip64.so`. + +After the native build, the exact-branch focused suite ran with the worktree +on `PYTHONPATH` and completed 13 of 13 tests: + +``` +tests/utils/test_rocm_runtime.py tests/kernels/test_pinned_tensor.py +13 passed in 2.51s +``` + +This closes the earlier false failure caused by importing an older deployment +checkout and separately distinguishes the first missing-extension run from +the final built-extension result. The isolated worktree and compiler log are +preserved at `/home/operator/freetoken-amd/validation-e7a5a92/` and +`/home/operator/freetoken-amd/validation-e7a5a92/build-rocm-validation.log`. + +## 2026-09-06 NVFP4 deep-K 8x16 candidate gate + +The existing production deep-K NVFP4 decode shape is `BLOCK_SIZE_N=8` and +`BLOCK_SIZE_KW=128`. A prior same-process differential screen found a +numerically identical `8x16` shape, so the current branch now exposes +`FREETOKEN_NVFP4_DEEPK_BLOCK_KW=16` as an opt-in experiment. The allow-list +accepts only 16, 32, 64, or 128, and the default remains 128. + +The candidate ran from the exact pushed branch in an isolated worktree with +the reusable ROCm kernel cache, JIT disabled, and the opt-in value 16. Its +artifact is +`/home/operator/freetoken-amd/artifacts/qwen-nvfp4-deepk16-candidate-20260906T000000Z/`. +The scheduler-shaped three-sample control completed all samples at 28.3782 +mean decode TPS. The canonical AIME gate passed with answer `70`, output SHA1 +`0acef4eab6f4`, 28.8547 decode TPS, 395.9 ms TTFT, 34.4671 ms event p50, and +36.7199 ms event p99. + +The warmed four-client, three-round control also completed all twelve requests +with deterministic responses. It recorded 52.8018 mean aggregate decode TPS, +0.9660 seconds p99 TTFT, and 76.7122 ms p99 token gap. This is slightly below +the prior tile32 warmed C4 result of 53.1579 TPS and 76.8555 ms p99 gap, and it +does not approach the qualified Q5 four-row C4 profile of 94.80 TPS and 39.93 +ms p99 gap. The deep-K 8x16 candidate is therefore rejected for promotion, +although its deterministic quality gate passed. The production default remains +the qualified 8x128 shape. + +The candidate process group was terminated only after its command, model path, +port, and process-group identity were verified. The protected Qwen service was +then restarted and verified with `status: ok` and `maintenance: serving`. + +## 2026-09-06 GGUF MMV-Y8 component attempt + +The next allowed GGUF MMV launch shape, `FREETOKEN_GGUF_MMV_Y=8`, was +admitted only to an isolated real-weight Qwen Q4_K and Q5_K component screen. +The protected Qwen service remained serving throughout. The native ROCm +extension build completed, but the 30-warmup and 300-repetition component +screen produced no JSON result after 4 minutes 48 seconds while contending for +the same unified-memory GPU and host resources as the protected service. The +benchmark process and its wrapper were verified by PID and command line, then +terminated without signaling the protected service. A subsequent read-only +health check returned `status: ok` and `maintenance: serving`. + +The incomplete artifact is +`/home/operator/freetoken-amd/artifacts/qwen-q4-mmv-y8-component-20260906T000000Z-build.log`. +Because no timed kernel result or output-equality record exists, this attempt +makes no performance or quality claim. A valid Y8 screen would require an +isolated GPU window with the protected service stopped and a verified recovery +afterward; Y8 remains unqualified and the production Y4 setting is unchanged. + +The follow-up clean-window attempt did stop the protected service through its +guarded script, but the real-weight benchmark still had not emitted its JSON +after more than two minutes of isolated execution. The benchmark process was +then terminated by its verified PID and the recovery server was relaunched. +The recovery artifact is +`/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260905T223454Z/`, +and the final health check returned `status: ok` with `maintenance: serving`. +The clean-window run also makes no TPS or quality claim because it has no +completed timed result. diff --git a/docs/gmktec-evo-x2-amd-validation-program.md b/docs/gmktec-evo-x2-amd-validation-program.md new file mode 100644 index 0000000000..b239bd93aa --- /dev/null +++ b/docs/gmktec-evo-x2-amd-validation-program.md @@ -0,0 +1,64 @@ +# GMKtek EVO-X2 AMD FreeToken validation program + +## Purpose + +This program establishes what the `amd-rocm-gfx1151` branch proves on GMKtek EVO-X2. +It separates native AMD functionality, GMKtek EVO-X2 performance, local ROCm control +comparisons, and strict replication of FreeToken's NVIDIA paper. A result may +only be labelled with the category its evidence supports. + +## Scope and safety contract + +- Every executable workload refuses hosts other than GMKtek EVO-X2. +- Candidate servers bind to loopback-only ports and never change llama-swap. +- Every temporary candidate run restores Qwen and waits for `/health` to report + `status: ok` before success. +- Every artifact directory is immutable. A duplicate run identifier is a + failure, not permission to overwrite old evidence. +- Timed runs reuse the native HIP extension cache. JIT compilation, swapping, + thermal throttling, unexpected disk traffic, or failed quality invalidates a + scored sample. +- The branch is validated only on `gfx1151`; it is not a general AMD claim. + +## Evidence categories + +| Category | Meaning | Current example | +| --- | --- | --- | +| Native AMD functionality | HIP, ROCm, API, and recovery work correctly | Qwen and Gemma serving on GMKtek EVO-X2 | +| Local control | Same local workload against an AMD control engine | Qwen Q4 FreeToken versus ROCm llama.cpp | +| Paper-inspired | Workload follows paper category but lacks exact paper fields | Future GMKtek EVO-X2 agent suite | +| Strict paper replication | Model, precision, prompts, warmup, policy, metrics, and scoring all match | Not yet available | + +## Metric definitions + +| Metric | Definition | +| --- | --- | +| Warm TTFT | Client monotonic time from request write to first content-bearing SSE event after warmup | +| Decode TPS | `(generated_tokens - 1) / (last_content_event - first_content_event)`; one-token outputs have no TPS | +| Output token gap | Adjacent client-observed content-bearing SSE timestamp difference | +| p50, p95, p99 token gap | Nearest-rank percentile of raw output-token gaps | +| Tail TTFT | Maximum whole-request TTFT across a named completed workload matrix | +| Quality result | Fixed expected answer, schema, executable test, or visible-output rule recorded with raw response | + +## Acceptance sequence + +1. Reproducibility and protocol ledger. +2. Native HIP, API, cache-reuse, and recovery regression. +3. Fixed Qwen and Gemma quality suite. +4. Five-sample cold and warm GMKtek EVO-X2 baseline matrix. +5. Paper-inspired agent workloads and tail analysis. +6. Twenty-four-hour endurance and recovery qualification. +7. Larger-model capacity assessment only after Qwen gates pass. +8. Strict NVIDIA comparison only with a reference system and complete paper protocol. + +Before every accepted baseline, run +`scripts/gmk-evo-x2/capture_validation_manifest.sh` against a new artifact path. +The resulting read-only manifest proves the host, source state, ROCm stack, +GPU policy, memory, swap, disk, and process context without exposing secrets. + +## Prohibited claims + +- A Q4 GGUF control is not a replication of NVFP4 or BF16 paper tests. +- A short warm request is not the paper's worst agent-turn TTFT. +- Hidden reasoning text is not a visible OpenAI-compatible answer. +- A larger model does not fit until a complete memory-reserve manifest proves it. diff --git a/docs/gmktec-evo-x2-batched-expert-transfer-prototype.md b/docs/gmktec-evo-x2-batched-expert-transfer-prototype.md new file mode 100644 index 0000000000..188b277109 --- /dev/null +++ b/docs/gmktec-evo-x2-batched-expert-transfer-prototype.md @@ -0,0 +1,54 @@ +# GMKtek EVO-X2 batched expert-transfer prototype + +This isolated prototype uses the same 64 randomly selected 64 KiB blocks as +the serialized expert-block test, but gathers blocks into pinned staging +buffers and performs one device copy per group. It separates CPU staging time +from host-to-device transfer time. It does not download or load a large model +and does not modify the protected service. + +## Method + +- GPU: AMD Radeon 8060S, gfx1151. +- PyTorch: `2.13.0+rocm10.0.0`. +- HIP: `7.15.26333`. +- Total payload per round: 4 MiB. +- Block size: 64 KiB. +- Three warmup rounds and ten measured rounds per grouping. +- Device synchronization after each grouped copy. + +## Results + +| Blocks per group | Groups per round | Total round rate | Transfer-only rate | Staging mean | +|---:|---:|---:|---:|---:| +| 1 | 64 | 4.67 GB/s | 5.83 GB/s | 0.171 ms | +| 4 | 16 | 9.44 GB/s | 15.28 GB/s | 0.168 ms | +| 16 | 4 | 12.84 GB/s | 29.79 GB/s | 0.185 ms | +| 64 | 1 | 9.46 GB/s | 33.64 GB/s | 0.318 ms | + +The serialized 64 KiB benchmark reached approximately 5.01 GB/s under its +different round and synchronization setup. Grouping 16 blocks improved the +transfer-only rate to approximately 29.79 GB/s, about 5.1 times the serialized +transfer-only result. The best end-to-end round rate was 12.84 GB/s at group +size 16 because CPU staging and synchronization remain part of the path. + +## Interpretation + +Batching and coalescing are necessary to make scattered expert movement +credible, but they do not recover the 79.79 GB/s contiguous-copy ceiling by +themselves. A production miss path must overlap staging with computation, +reuse pinned buffers, and choose a group size that avoids excessive CPU gather +cost. These measurements are synthetic and must not be converted directly to +model TPS. + +For the 284B feasibility question, this result means that a full-checkpoint +offload path would need a high locality cache plus grouped transfers. A design +that services every expert miss as an independent small copy is ruled out by +the earlier prototype. A design that batches misses has a plausible systems +direction, but still faces the approximately 148.66 GiB payload versus the +approximately 18 GiB live available-memory constraint. + +## Next gate + +The next optimization experiment should overlap grouped staging and device +copies with a synthetic compute kernel. It should report whether overlap hides +the approximately 0.17 to 0.32 ms staging cost without increasing tail latency. diff --git a/docs/gmktec-evo-x2-campaign-completion-audit.md b/docs/gmktec-evo-x2-campaign-completion-audit.md new file mode 100644 index 0000000000..8f0b234eb8 --- /dev/null +++ b/docs/gmktec-evo-x2-campaign-completion-audit.md @@ -0,0 +1,83 @@ +# GMKtek EVO-X2 FreeToken AMD campaign completion audit + +## Purpose and scope + +This audit is the controlling completion record for the native ROCm and HIP +FreeToken port evaluated on the authorized GMKtek EVO-X2. It separates what +has been proven on that system from paper-inspired evidence, from comparisons +that require an external NVIDIA reference system or unreleased author inputs. +It must be updated from immutable artifacts, not from a plan or an intended +command. + +The campaign may claim only GMKtek EVO-X2 results. A second host is outside +the authorized scope, so it cannot be silently substituted for a missing +result or used to claim broader AMD support. + +## Completion rules + +A row is **proven** only when its named artifact or tracked source demonstrates +the stated condition. A row is **in progress** when a live controller is +collecting the required evidence. A row is **external evidence unavailable** +when the required source, fixture, or hardware is not available to this +campaign. The latter is a documented limitation, never a passing result. + +The campaign is not complete while any in-scope proven or in-progress row +lacks its required evidence. The final audit must retain comparison limits +instead of converting a different model format, workload, hardware tier, or +metric boundary into an equal comparison. + +## Requirement matrix + +| Requirement | Required proof | Current status | Authoritative evidence or next action | +| --- | --- | --- | --- | +| Native ROCm and HIP execution | Native extension build, HIP runtime evidence, and no substitute backend | Proven | [`amd-rocm-gfx1151.md`](amd-rocm-gfx1151.md) and recorded Qwen and Gemma artifacts | +| OpenAI-compatible local API | Model listing plus completed streaming and non-streaming requests | Proven | Qwen and Gemma controls in [`gmktec-evo-x2-amd-run-log.md`](gmktec-evo-x2-amd-run-log.md) | +| Qwen deterministic visible-output quality | Versioned exact canary, arithmetic, JSON, and AIME records with raw responses | Proven for the controlled suite | C139 records canonical AIME SHA1 `3302eda43396`; the run log records the suite boundaries | +| Gemma 4 quality | Text, multimodal fixtures, and bounded visual description with raw outputs | Proven for the controlled suite | Gemma entries in [`gmktec-evo-x2-amd-run-log.md`](gmktec-evo-x2-amd-run-log.md) | +| Gemma 4 performance and stability | Single-request, concurrent, long-context, and bounded endurance controls with raw artifacts | Proven for bounded controls | [`gmktec-evo-x2-gemma4-comparison-report.md`](gmktec-evo-x2-gemma4-comparison-report.md) records FreeToken and ROCm 10 llama.cpp results. A full 1,440-session Gemma campaign is optional publication evidence, not a missing functional gate. | +| Gemma 4 performance comparison | Same model and fixed request contract for single, concurrent, and long-context controls | Proven for bounded controls | [`gmktec-evo-x2-gemma4-comparison-report.md`](gmktec-evo-x2-gemma4-comparison-report.md) records FreeToken and ROCm 10 llama.cpp matrices, including reasoning-off llama.cpp long-context results | +| Qwen Q4_K_M same-format comparison | Same checkpoint, tokenizer, caller-rendered prompt, completion cap, and five warmed requests on both runtimes | Proven for decode parity; TTFT remains a separate boundary | [`gmktec-evo-x2-cross-model-manifest-20260905.json`](gmktec-evo-x2-cross-model-manifest-20260905.json) records five-request artifacts. FreeToken steady decode was 49.4357 TPS versus 49.1772 TPS for llama.cpp, while all-sample means were 48.6028 and 49.1155 TPS respectively. | +| Q5-only four-row optimization correctness | Real-weight component parity and complete API quality gate | Proven | C138 exact component hash and C139 API quality evidence | +| Q5-only four-row performance value | Same configuration baseline comparison, scheduler, C4, and tail metrics | Proven for the stated local Qwen workload | C139 records higher C4 prefill and decode TPS plus lower C4 tails; it separately retains the slight single-request decode reduction | +| ROCm llama.cpp local control | Same host, recorded model format, API shape, quality suite, and timing matrix | Proven as a practical Q4 control; five-sample refresh recorded | C89 remains the earlier four-slot workload control. The 2026-09-04 five-sample refresh is preserved at `/home/operator/freetoken-amd/artifacts/qwen35b-llamacpp-rocm10-timeshare-five-20260904T101357Z/`: five of five samples passed, mean decode 46.6625 TPS, median 46.7524 TPS, mean prefill 19,343.40 TPS. Protected-service recovery completed and the paired FreeToken control is preserved at `/home/operator/freetoken-amd/artifacts/qwen35b-freetoken-five-20260904T102530Z/`. It is not a same-format NVFP4 equivalence claim. | +| Paper-inspired W1 control | Pinned AIME source, complete local request contract, five samples, raw responses, and quality result | Proven as paper-inspired control | Five raw samples and aggregate evidence are preserved at `/home/operator/freetoken-amd/artifacts/w1-paper-inspired-five-sample-20260904T094252`. All five matched output SHA1 `0acef4eab6f4`; the run log records token counts and timing. This remains a reproducible W1-style control, not strict paper replication, because the paper's original prompt, cache policy, and exact runner contract remain unpublished. | +| W2 through W4 strict replication | Authors' exact harnesses, fixtures, versions, policy, and scoring | External evidence unavailable | Public source audit documents that OpenCode SWE-bench, Claude Code, OpenClaw, and raw paper artifacts are not released | +| 24-hour Q5 endurance | All 1,440 minute-cadence sessions, zero candidate and host swap, final summary, restored swap, and real normal-service completion | Proven | C142 artifact `/home/operator/freetoken-amd/artifacts/q4-c142-q5-swapdrain-endurance-20260902T222206Z` contains exactly 1,440 valid session JSON files, zero failures, zero candidate and host swap, completed controller evidence, and preserved recovery artifacts. Per-session records measure state correctness, TTFT, token-gap tails, swap, and thermal telemetry. They intentionally do not claim per-session prefill TPS. | +| Normal service recovery | Recovered protected Qwen API produces a real completed response with `finish_reason: stop` | Proven | Read-only probe artifact `/home/operator/freetoken-amd/artifacts/qwen-protected-recovery-explicit-20260904T093948` records model `qwen3.6-35b-a3b-nvfp4-amd`, visible response `READY.`, and `finish_reason: stop` after the C142 recovery. | +| 284B capacity claim | Model manifest, reserved-memory evidence, load and quality result on comparable resources | Incomplete, metadata gate rejects full load | [`gmktec-evo-x2-paper-model-capacity-gate.md`](gmktec-evo-x2-paper-model-capacity-gate.md) pins the official release revision and records a reproducible metadata-only `REJECT_FULL_LOAD` result: 155.425 GiB payload versus a 4 GiB authoritative budget after explicit headroom. The new real-shape slice measures transfer only; full-model quality and serving throughput remain unmeasured. | +| Strict NVIDIA paper comparison | Same model, precision, workload, policy, metric boundary, and NVIDIA reference hardware | External evidence unavailable | The paper protocol still lacks exact released inputs and no reference NVIDIA system is in scope | +| Upstream-ready documentation | Reproducible, secret-safe tracked source and current evidence links | Proven for current evidence set | C142, Gemma comparison, Qwen same-format warmed matrix, machine-readable manifest, long-context boundaries, recovery proof, W1 result, and the capacity baseline are tracked. Strict NVIDIA parity and 284B qualification remain explicitly unresolved. | + +## Required terminal sequence for C142 + +1. Confirm exactly 1,440 session JSON records and a passing `summary.json`. +2. Inspect the full latency and swap summaries without discarding the cold + first-session result. +3. Confirm candidate process-group and whole-host swap remain zero throughout. +4. Confirm the controller restores configured swap before normal recovery. +5. Verify normal Qwen with a real OpenAI-compatible completion ending in + `finish_reason: stop`. +6. Add a C142 evidence entry, commit only tracked campaign documentation, and + re-run the documentation and benchmark-tool regression checks. +7. Run the pinned paper-inspired W1 control after normal-service recovery, then + update this matrix with the observed five-sample evidence and its remaining + strict-paper limitations. + +## Performance metric boundaries + +C139 is the qualified complete API performance gate. It records client-observed +prefill TPS, decode TPS, warm TTFT, C4 aggregate prefill and decode TPS, and +C4 tail latency under the exact Q5-only four-row candidate. C142 has a distinct +purpose: it establishes long-duration state, swap, thermal, and tail stability +under minute cadence. Its three-turn state suite has no controlled fixed-size +input throughput interval, so it must not be presented as a prefill-TPS +measurement. The final report must show the C139 TPS results and C142 endurance +results together, with their different measurement boundaries stated plainly. + +## Final reporting rule + +The final report must state separately: native AMD functionality, controlled +quality, local Q4 control comparisons, paper-inspired controls, strict-paper +limitations, and external hardware limitations. It may not state that a +GMKtek EVO-X2 result equals or exceeds a published NVIDIA result unless every +condition in the strict NVIDIA comparison row is proven. diff --git a/docs/gmktec-evo-x2-cross-model-manifest-20260905.json b/docs/gmktec-evo-x2-cross-model-manifest-20260905.json new file mode 100644 index 0000000000..dbdd9f0d18 --- /dev/null +++ b/docs/gmktec-evo-x2-cross-model-manifest-20260905.json @@ -0,0 +1,309 @@ +{ + "schema_version": "1.0", + "manifest_date_utc": "2026-09-05", + "purpose": "Machine-readable index of controlled native FreeToken and ROCm 10 llama.cpp comparison evidence on one GMKtek EVO-X2.", + "scope": { + "host_class": "GMKtek EVO-X2", + "gpu_architecture": "AMD Radeon 8060S gfx1151", + "execution_stack": "native ROCm/HIP", + "rocm_major": 10, + "llama_cpp_control_stack": "ROCm 10", + "protected_service_mutation_policy": "candidate processes are isolated and the protected service is restored and health-checked after each candidate" + }, + "metric_definitions": { + "prefill_tps": "Client-observed prompt-token throughput for the request interval reported by the harness.", + "decode_tps": "Client-observed generated-token throughput for the streamed completion interval reported by the harness.", + "ttft_ms": "Time to first visible streamed token in milliseconds.", + "p99_token_gap_ms": "99th percentile inter-token gap for the scored streamed completion.", + "aggregate_decode_tps": "Combined generated-token throughput across concurrently active requests.", + "quality_status": "Deterministic quality and API contract result recorded by the corresponding harness verifier." + }, + "runs": [ + { + "run_id": "qwen36_q4_mmv_y4_freetoken", + "model_id": "Qwen3.6-35B-A3B", + "runtime": "FreeToken", + "format": "Q4 packed GGUF route with MMV_Y=4", + "request_shape": { + "samples": 5, + "prompt_contract": "fixed API benchmark contract", + "completion_contract": "fixed API benchmark contract", + "warmup_policy": "harness warmup before scored samples", + "concurrency": 1 + }, + "metrics": { + "mean_prefill_tps": 2857.7769, + "mean_decode_tps": 48.0312, + "mean_client_ttft_ms": 424.0, + "quality_status": "passed" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/qwen-q4-mmv-y4-api5-20260905T075002Z" + }, + { + "run_id": "qwen36_q4_llama_cpp_rocm10", + "model_id": "Qwen3.6-35B-A3B", + "runtime": "llama.cpp", + "format": "Q4_K_M GGUF", + "request_shape": { + "samples": 5, + "prompt_contract": "matched Qwen API benchmark contract", + "completion_contract": "matched Qwen API benchmark contract", + "warmup_policy": "control warmup before scored samples", + "concurrency": 1 + }, + "metrics": { + "mean_prefill_tps": 18868.7707, + "mean_decode_tps": 48.7477, + "quality_status": "passed" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/qwen-llamacpp-paired-20260905T062500Z" + }, + { + "run_id": "qwen36_q4km_gguf_freetoken_raw", + "model_id": "Qwen3.6-35B-A3B", + "runtime": "FreeToken", + "format": "Q4_K_M GGUF", + "request_shape": { + "samples": 1, + "prompt_tokens_observed": 54, + "completion_tokens_observed": 255, + "prompt_contract": "caller-rendered raw prompt via /v1/completions", + "completion_contract": "256-token cap", + "warmup_policy": "cold model initialization included in TTFT", + "concurrency": 1 + }, + "metrics": { + "decode_tps": 50.0169, + "ttft_ms": 54311.0295, + "quality_status": "expected answer path passed; full output hash differs from llama.cpp" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/qwen-gguf-raw-20260905T121136Z/raw-quality.json" + }, + { + "run_id": "qwen36_q4km_gguf_llama_cpp_raw", + "model_id": "Qwen3.6-35B-A3B", + "runtime": "llama.cpp", + "format": "Q4_K_M GGUF", + "request_shape": { + "samples": 1, + "prompt_tokens_observed": 54, + "completion_tokens_observed": 256, + "prompt_contract": "caller-rendered raw prompt via /v1/completions", + "completion_contract": "256-token cap", + "warmup_policy": "model loaded before request", + "concurrency": 1 + }, + "metrics": { + "decode_tps": 49.3875, + "ttft_ms": 234.0382, + "quality_status": "expected answer path passed; full output hash differs from FreeToken" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/qwen-llama-raw-20260905T122310Z/raw-quality.json" + }, + { + "run_id": "qwen36_q4km_gguf_freetoken_warmed_matrix", + "model_id": "Qwen3.6-35B-A3B", + "runtime": "FreeToken", + "format": "Q4_K_M GGUF", + "request_shape": { + "samples": 5, + "prompt_tokens_observed": 54, + "completion_tokens_observed": 255, + "prompt_contract": "caller-rendered raw prompt via /v1/completions", + "completion_contract": "256-token cap", + "warmup_policy": "one loaded server; first scored request retained and samples 2 to 5 reported separately", + "concurrency": 1 + }, + "metrics": { + "mean_decode_tps_all_samples": 48.6028, + "mean_decode_tps_samples_2_to_5": 49.4357, + "mean_ttft_ms_all_samples": 982.1695, + "mean_ttft_ms_samples_2_to_5": 424.2563, + "quality_status": "passed; all five output hashes matched" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/qwen-gguf-warm-matrix-20260905T122817Z" + }, + { + "run_id": "qwen36_q4km_gguf_llama_cpp_warmed_matrix", + "model_id": "Qwen3.6-35B-A3B", + "runtime": "llama.cpp", + "format": "Q4_K_M GGUF", + "request_shape": { + "samples": 5, + "prompt_tokens_observed": 54, + "completion_tokens_observed": 256, + "prompt_contract": "caller-rendered raw prompt via /v1/completions", + "completion_contract": "256-token cap", + "warmup_policy": "one loaded server; first scored request retained and samples 2 to 5 reported separately", + "concurrency": 1 + }, + "metrics": { + "mean_decode_tps_all_samples": 49.1155, + "mean_decode_tps_samples_2_to_5": 49.1772, + "mean_ttft_ms_all_samples": 92.0729, + "mean_ttft_ms_samples_2_to_5": 58.8298, + "quality_status": "passed; all five output hashes matched" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/qwen-llama-warm-matrix-20260905T124007Z" + }, + { + "run_id": "gemma4_q4_freetoken_text", + "model_id": "Gemma 4 26B A4B", + "runtime": "FreeToken", + "format": "Q4_0 GGUF", + "request_shape": { + "samples": 5, + "prompt_tokens_observed": 34, + "completion_tokens_observed": 127, + "prompt_contract": "matched fixed arithmetic text prompt", + "completion_contract": "fixed output-token cap", + "warmup_policy": "ROCm prefill warmup enabled", + "concurrency": 1 + }, + "metrics": { + "mean_prefill_tps": 174.5816, + "mean_decode_tps": 53.0762, + "mean_ttft_ms": 196.3267, + "p99_token_gap_ms": 21.6015, + "quality_status": "passed" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T084837Z/text-matrix.json" + }, + { + "run_id": "gemma4_q4_llama_cpp_rocm10_text", + "model_id": "Gemma 4 26B A4B", + "runtime": "llama.cpp", + "format": "Q4_0 GGUF", + "request_shape": { + "samples": 5, + "prompt_tokens_observed": 34, + "completion_tokens_observed": 128, + "prompt_contract": "matched fixed arithmetic text prompt", + "completion_contract": "fixed output-token cap", + "warmup_policy": "control warmup before scored samples", + "concurrency": 1 + }, + "metrics": { + "mean_prefill_tps": 737.0039, + "mean_decode_tps": 56.8293, + "mean_ttft_ms": 47.6735, + "p99_token_gap_ms": 18.1409, + "quality_status": "passed" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/gemma4-llamacpp-vision-20260905T091532Z/text-matrix.json" + }, + { + "run_id": "gemma4_q4_freetoken_concurrency_2", + "model_id": "Gemma 4 26B A4B", + "runtime": "FreeToken", + "format": "Q4_0 GGUF", + "request_shape": { + "clients": 2, + "rounds": 3, + "prompt_contract": "matched 16-repeat prompt", + "completion_tokens": 128 + }, + "metrics": { + "aggregate_decode_tps": 31.0388, + "mean_ttft_ms": 359.8159, + "quality_status": "passed" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T112013Z/concurrency.json" + }, + { + "run_id": "gemma4_q4_llama_cpp_rocm10_concurrency_2", + "model_id": "Gemma 4 26B A4B", + "runtime": "llama.cpp", + "format": "Q4_0 GGUF", + "request_shape": { + "clients": 2, + "rounds": 3, + "prompt_contract": "matched 16-repeat prompt", + "completion_tokens": 128 + }, + "metrics": { + "aggregate_decode_tps": 35.8031, + "mean_ttft_ms": 1263.9829, + "quality_status": "passed" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/gemma4-llamacpp-vision-20260905T112959Z/concurrency.json" + }, + { + "run_id": "gemma4_q4_freetoken_concurrency_4", + "model_id": "Gemma 4 26B A4B", + "runtime": "FreeToken", + "format": "Q4_0 GGUF", + "request_shape": { + "clients": 4, + "rounds": 3, + "prompt_contract": "matched 16-repeat prompt", + "completion_tokens": 128 + }, + "metrics": { + "aggregate_decode_tps": 22.1884, + "mean_ttft_ms": 369.0398, + "quality_status": "passed" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T105149Z/concurrency.json" + }, + { + "run_id": "gemma4_q4_llama_cpp_rocm10_concurrency_4", + "model_id": "Gemma 4 26B A4B", + "runtime": "llama.cpp", + "format": "Q4_0 GGUF", + "request_shape": { + "clients": 4, + "rounds": 3, + "prompt_contract": "matched 16-repeat prompt", + "completion_tokens": 128 + }, + "metrics": { + "aggregate_decode_tps": 21.3101, + "mean_ttft_ms": 3678.1440, + "quality_status": "passed" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/gemma4-llamacpp-vision-20260905T110157Z/concurrency.json" + }, + { + "run_id": "gemma4_q4_freetoken_concurrency_8", + "model_id": "Gemma 4 26B A4B", + "runtime": "FreeToken", + "format": "Q4_0 GGUF", + "request_shape": { + "clients": 8, + "rounds": 3, + "prompt_contract": "matched 16-repeat prompt", + "completion_tokens": 128 + }, + "metrics": { + "aggregate_decode_tps": 14.8738, + "mean_ttft_ms": 3178.3465, + "quality_status": "passed" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260905T110609Z/concurrency.json" + }, + { + "run_id": "gemma4_q4_llama_cpp_rocm10_concurrency_8", + "model_id": "Gemma 4 26B A4B", + "runtime": "llama.cpp", + "format": "Q4_0 GGUF", + "request_shape": { + "clients": 8, + "rounds": 3, + "prompt_contract": "matched 16-repeat prompt", + "completion_tokens": 128 + }, + "metrics": { + "aggregate_decode_tps": 11.8362, + "mean_ttft_ms": 8483.9289, + "quality_status": "passed" + }, + "artifact": "/home/operator/freetoken-amd/artifacts/gemma4-llamacpp-vision-20260905T111607Z/concurrency.json" + } + ], + "unresolved": [ + "Strict NVIDIA paper reproduction remains unavailable because the paper does not publish all workload fixtures and no reference NVIDIA host is in scope.", + "DeepSeek-V4-Flash 284B has not been loaded because the exact checkpoint is not present and the host capacity baseline is materially below the paper desktop configuration.", + "Archived model identifiers without an admitted payload are not treated as benchmark failures." + ] +} diff --git a/docs/gmktec-evo-x2-cross-model-matrix-20260904.md b/docs/gmktec-evo-x2-cross-model-matrix-20260904.md new file mode 100644 index 0000000000..3e2dd7492e --- /dev/null +++ b/docs/gmktec-evo-x2-cross-model-matrix-20260904.md @@ -0,0 +1,93 @@ +# GMKtek EVO-X2 cross-model benchmark matrix + +This matrix consolidates the preserved September 4, 2026 controls for the +native ROCm/HIP FreeToken port and the local ROCm 10 llama.cpp controls. It is +an evidence index, not a claim that every row is a strict apples-to-apples +comparison. Each comparison must retain its model format, prompt contract, +sampling settings, warmup rules, and concurrency boundary. + +## Fixed-length text controls + +| Model and runtime | Samples | Prompt tokens | Completion tokens | Mean prefill TPS | Mean decode TPS | Mean TTFT | p99 token gap | Quality status | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| Gemma 4 Q4 FreeToken | 5 | 34 | 127 | 174.58 | 53.08 | 196.33 ms | 21.60 ms | Exact text control passed | +| Gemma 4 Q4 llama.cpp ROCm 10 | 5 | 34 | 128 | 737.00 | 56.83 | 47.67 ms | 18.14 ms | Matched visible prompt contract passed | + +The Gemma rows use the matched five-sample artifacts +`gemma4-gguf-text-20260905T084837Z/text-matrix.json` and +`gemma4-llamacpp-vision-20260905T091532Z/text-matrix.json`. The visible prompt +contract is matched; the one-token difference in tokenizer-reported prompt +length is retained as observed telemetry rather than silently normalized. + +## Qwen controls + +| Model and runtime | Samples | Prompt tokens | Completion tokens | Mean prefill TPS | Mean decode TPS | Tail evidence | Quality status | +| --- | ---: | ---: | ---: | ---: | ---: | --- | --- | +| Qwen3.6 35B-A3B FreeToken Q4 | 5 | 1,212 | 255 | 2,936.92 | 28.04 | Mean p99 gap 35.38 ms | Passed paired quality gate | +| Qwen3.6 35B-A3B llama.cpp Q4_K_M ROCm 10 | 5 | 1,212 | 256 | 19,343.40 | 46.66 | Mean gap 21.43 ms | Passed control suite | +| Qwen3.6 35B-A3B FreeToken Q5 four-row | 3 scheduler plus 3 C4 rounds | Fixed scheduler contract | Fixed scheduler contract | 3,130.30 | 48.20 single, 94.80 aggregate C4 | p99 TTFT 1.025 s; p99 gap 39.93 ms | Canonical AIME passed | +| Qwen3.6 35B-A3B FreeToken Q4 MMV_Y=4 | 5 | Fixed API contract | Fixed API contract | 2,857.78 | 48.03 | Mean client TTFT about 0.424 s | Quality and API checks passed | + +### Same-checkpoint and same-format raw-prompt control + +| Model and runtime | Samples | Prompt tokens | Completion tokens | Decode TPS | TTFT | Quality result | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Qwen3.6 35B-A3B FreeToken Q4_K_M GGUF | 1 | 54 | 255 | 50.0169 | 54.311 s cold request | Expected answer path passed; output hash differs from llama.cpp | +| Qwen3.6 35B-A3B llama.cpp Q4_K_M GGUF ROCm 10 | 1 | 54 | 256 | 49.3875 | 234.0 ms loaded control | Expected answer path passed; output hash differs from FreeToken | +| Qwen3.6 35B-A3B FreeToken Q4_K_M GGUF warmed matrix | 5 | 54 | 255 | 48.6028 all samples; 49.4357 samples 2 to 5 | 982.17 ms all samples; 424.26 ms samples 2 to 5 | All five output hashes match; expected answer path passed | +| Qwen3.6 35B-A3B llama.cpp Q4_K_M GGUF warmed matrix | 5 | 54 | 256 | 49.1155 all samples; 49.1772 samples 2 to 5 | 92.07 ms all samples; 58.83 ms samples 2 to 5 | All five output hashes match; expected answer path passed | + +The same 22 GiB Q4_K_M GGUF checkpoint, tokenizer, caller-rendered raw +prompt, and output harness were used. FreeToken was approximately 1.27 percent +faster on decode. TTFT is not a valid parity claim in this pair because the +FreeToken measurement includes its cold model initialization while llama.cpp +was already loaded. The two responses both reached the expected answer path, +but their full output hashes differ, so this run is a performance control and +not proof of bit-identical generation. + +The warmed FreeToken follow-up used one loaded server and five consecutive +requests. Its first scored request measured 45.2713 TPS while requests 2 to 5 +measured 49.3630, 49.3096, 49.6545, and 49.4156 TPS. This separates cold +startup and first-request effects from the steady request path. A warmed +five-sample llama.cpp matrix is still required before declaring a statistical +same-format winner. The warmed llama.cpp follow-up measured 48.8686 TPS on +the first request and 49.1575, 49.1606, 49.1887, and 49.2019 TPS on requests +2 through 5. FreeToken's samples 2 to 5 mean was 49.4357 TPS, approximately +0.53 percent above llama.cpp's 49.1772 TPS. Across all five samples, llama.cpp +was approximately 1.06 percent faster because FreeToken's first request was +slower. This is a near-parity result, not a material performance lead. + +The Q4 rows are practical local controls, not a same-format NVFP4 equivalence +claim. The Q5 four-row row is the currently qualified quality-preserving +optimization and is not directly comparable to the Q4 llama.cpp row without a +matched Q5 control. + +## Concurrency and long-context coverage + +| Model and runtime | Concurrency | Long-context | Endurance | Current conclusion | +| --- | --- | --- | --- | --- | +| Qwen FreeToken | 1, 2, 4, and 8-client tail controls | 4,856-token W3-style control passed | 1,440 sessions passed | Qwen stability qualification complete | +| Gemma 4 FreeToken | 2, 4, and 8-client matched controls passed | 2,528 and 5,033 prompt-token controls passed; 8,192-token ceiling reached | 30 sessions passed | Aggregate decode is 4.1 percent above llama.cpp at four clients and 25.7 percent above it at eight clients; TTFT is lower under concurrency | +| Gemma 4 llama.cpp ROCm 10 | 2, 4, and 8-client matched controls passed | Reasoning-off 2,528 and 5,033-token controls passed; 8,192-token ceiling reached | Not run | Matched local control, not strict paper replication | + +### Matched Gemma concurrency detail + +| Clients | FreeToken aggregate decode TPS | llama.cpp aggregate decode TPS | FreeToken mean TTFT | llama.cpp mean TTFT | Quality | +| ---: | ---: | ---: | ---: | ---: | --- | +| 2 | 31.04 | 35.80 | 359.8 ms | 1,264.0 ms | All requests passed | +| 4 | 22.19 | 21.31 | 369.0 ms | 3,678.1 ms | All requests passed | +| 8 | 14.87 | 11.84 | 3,178.3 ms | 8,483.9 ms | All requests passed | + +## Missing cells before final campaign closure + +1. Run every additional model only after its exact payload and backend are + admitted. The current active service inventory is already covered by the + Qwen and Gemma rows; archived models without payloads remain unqualified. +2. Add a format-matched Qwen control using the same checkpoint and + quantization on both runtimes before making a same-format claim. +3. Consolidate telemetry fields, cold-start policy, and cache state into a + machine-readable comparison manifest. +4. Keep a full Gemma 1,440-session campaign optional. It is not required for + the current functional or bounded-performance release gates. +5. Keep strict NVIDIA comparison and 284B capacity as separate unresolved + work items because their required external evidence is still missing. diff --git a/docs/gmktec-evo-x2-deepseek-capacity-gate-result-20260905.json b/docs/gmktec-evo-x2-deepseek-capacity-gate-result-20260905.json new file mode 100644 index 0000000000..620b16eb33 --- /dev/null +++ b/docs/gmktec-evo-x2-deepseek-capacity-gate-result-20260905.json @@ -0,0 +1,19 @@ +{ + "decision": "REJECT_FULL_LOAD", + "payload_bytes": 166886535336, + "payload_gib": 155.425, + "mem_available_gib": 18.0, + "rocm_vram_aperture_gib": 2.0, + "reserves_gib": { + "os": 8.0, + "runtime": 2.0, + "kv_cache": 2.0, + "recovery": 2.0, + "total": 14.0 + }, + "authoritative_model_budget_gib": 4.0, + "optimistic_budget_including_vram_gib": 20.0, + "authoritative_deficit_gib": 151.425, + "optimistic_deficit_gib": 135.425, + "interpretation": "The full payload cannot be admitted with the declared headroom. Do not download or load it on this host." +} diff --git a/docs/gmktec-evo-x2-deepseek-expert-slice-16-result-20260905.json b/docs/gmktec-evo-x2-deepseek-expert-slice-16-result-20260905.json new file mode 100644 index 0000000000..4cd5056b28 --- /dev/null +++ b/docs/gmktec-evo-x2-deepseek-expert-slice-16-result-20260905.json @@ -0,0 +1,1026 @@ +{ + "scope": "isolated real-shape routed expert transfer only", + "checkpoint": "checkpoint", + "device": "cuda:0", + "layers": [ + 0 + ], + "experts": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15 + ], + "repeats": 5, + "selected_tensor_count": 96, + "selected_bytes": 213909504, + "selected_mib": 204.0, + "tensors": [ + { + "name": "layers.0.ffn.experts.0.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.0.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.0.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.0.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.0.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.0.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.1.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.1.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.1.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.1.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.1.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.1.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.2.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.2.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.2.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.2.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.2.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.2.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.3.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.3.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.3.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.3.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.3.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.3.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.4.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.4.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.4.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.4.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.4.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.4.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.5.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.5.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.5.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.5.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.5.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.5.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.6.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.6.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.6.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.6.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.6.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.6.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.7.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.7.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.7.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.7.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.7.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.7.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.8.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.8.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.8.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.8.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.8.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.8.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.9.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.9.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.9.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.9.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.9.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.9.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.10.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.10.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.10.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.10.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.10.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.10.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.11.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.11.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.11.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.11.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.11.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.11.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.12.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.12.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.12.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.12.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.12.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.12.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.13.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.13.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.13.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.13.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.13.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.13.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.14.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.14.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.14.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.14.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.14.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.14.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.15.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.15.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.15.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.15.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.15.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.15.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + } + ], + "round_trips": [ + { + "h2d_seconds": 0.02892392093781382, + "d2h_seconds": 0.0031708949245512486, + "h2d_gib_per_second": 6.887681321917544, + "d2h_gib_per_second": 62.827294735474034 + }, + { + "h2d_seconds": 0.002597335958853364, + "d2h_seconds": 0.003068808000534773, + "h2d_gib_per_second": 76.70118658348238, + "d2h_gib_per_second": 64.91730664325823 + }, + { + "h2d_seconds": 0.002567927003838122, + "d2h_seconds": 0.0030759990913793445, + "h2d_gib_per_second": 77.57960008296187, + "d2h_gib_per_second": 64.7655425381371 + }, + { + "h2d_seconds": 0.002570027019828558, + "d2h_seconds": 0.003084828029386699, + "h2d_gib_per_second": 77.51620837561838, + "d2h_gib_per_second": 64.58018019228355 + }, + { + "h2d_seconds": 0.0025676570367068052, + "d2h_seconds": 0.0030751079320907593, + "h2d_gib_per_second": 77.58775691301499, + "d2h_gib_per_second": 64.78431144514384 + } + ], + "protected_service_touched": false, + "full_model_serving_claim": false +} diff --git a/docs/gmktec-evo-x2-deepseek-expert-slice-2layer-result-20260905.json b/docs/gmktec-evo-x2-deepseek-expert-slice-2layer-result-20260905.json new file mode 100644 index 0000000000..89b6719ec2 --- /dev/null +++ b/docs/gmktec-evo-x2-deepseek-expert-slice-2layer-result-20260905.json @@ -0,0 +1,777 @@ +{ + "scope": "isolated real-shape routed expert transfer only", + "checkpoint": "checkpoint", + "device": "cuda:0", + "layers": [ + 0, + 1 + ], + "experts": [ + 0, + 1, + 2, + 3, + 4, + 5 + ], + "repeats": 5, + "selected_tensor_count": 72, + "selected_bytes": 160432128, + "selected_mib": 153.0, + "tensors": [ + { + "name": "layers.0.ffn.experts.0.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.0.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.0.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.0.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.0.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.0.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.1.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.1.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.1.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.1.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.1.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.1.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.2.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.2.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.2.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.2.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.2.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.2.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.3.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.3.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.3.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.3.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.3.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.3.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.4.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.4.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.4.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.4.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.4.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.4.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.5.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.5.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.5.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.5.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.5.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.5.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.0.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.0.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.0.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.0.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.0.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.0.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.1.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.1.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.1.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.1.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.1.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.1.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.2.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.2.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.2.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.2.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.2.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.2.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.3.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.3.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.3.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.3.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.3.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.3.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.4.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.4.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.4.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.4.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.4.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.4.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.5.w1.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.5.w1.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.5.w2.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.5.w2.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.1.ffn.experts.5.w3.weight", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.1.ffn.experts.5.w3.scale", + "shard": "model-00003-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + } + ], + "round_trips": [ + { + "h2d_seconds": 0.026112683932296932, + "d2h_seconds": 0.002403973019681871, + "h2d_gib_per_second": 5.721896029048179, + "d2h_gib_per_second": 62.15296980320215 + }, + { + "h2d_seconds": 0.0019450889667496085, + "d2h_seconds": 0.0023495149798691273, + "h2d_gib_per_second": 76.81605574560544, + "d2h_gib_per_second": 63.593577304334815 + }, + { + "h2d_seconds": 0.0019166310084983706, + "d2h_seconds": 0.002299295971170068, + "h2d_gib_per_second": 77.95661336871615, + "d2h_gib_per_second": 64.98252698801801 + }, + { + "h2d_seconds": 0.0019194709602743387, + "d2h_seconds": 0.0023040659725666046, + "h2d_gib_per_second": 77.84127272164885, + "d2h_gib_per_second": 64.84799666285633 + }, + { + "h2d_seconds": 0.0019123710226267576, + "d2h_seconds": 0.0022964769741520286, + "h2d_gib_per_second": 78.13026903888698, + "d2h_gib_per_second": 65.06229506401691 + } + ], + "protected_service_touched": false, + "full_model_serving_claim": false +} diff --git a/docs/gmktec-evo-x2-deepseek-expert-slice-metadata-20260905.json b/docs/gmktec-evo-x2-deepseek-expert-slice-metadata-20260905.json new file mode 100644 index 0000000000..04b9815cdd --- /dev/null +++ b/docs/gmktec-evo-x2-deepseek-expert-slice-metadata-20260905.json @@ -0,0 +1,29 @@ +{ + "checkpoint_revision": "9e165c30e2704aec5d9d593cce3eebd58bbef1cb", + "source_files": [ + "model.safetensors.index.json", + "model-00002-of-00048.safetensors header range 0-1048575" + ], + "core_geometry": { + "layers": 43, + "routed_experts_per_layer": 256, + "active_experts_per_token_per_layer": 6, + "expert_tensor_names": ["w1.weight", "w2.weight", "w3.weight"], + "scale_tensor_names": ["w1.scale", "w2.scale", "w3.scale"] + }, + "observed_tensor_layout": { + "w1_weight": {"dtype": "I8", "shape": [2048, 2048], "bytes": 4194304}, + "w2_weight": {"dtype": "I8", "shape": [4096, 1024], "bytes": 4194304}, + "w3_weight": {"dtype": "I8", "shape": [2048, 2048], "bytes": 4194304}, + "each_scale": {"dtype": "F8_E8M0", "w1_w3_shape": [2048, 128], "w2_shape": [4096, 64], "bytes": 262144} + }, + "derived_sizes": { + "one_expert_bytes": 13369344, + "one_expert_mib": 12.75, + "all_experts_one_layer_gib": 3.1875, + "all_core_routed_experts_gib": 137.0625, + "six_active_experts_all_43_layers_gib": 3.22265625 + }, + "scope": "Core routed expert tensors only. Shared experts, attention, embeddings, MTP tensors, runtime buffers, KV cache, and allocator overhead are excluded.", + "decision": "A tiny-slice transfer experiment is technically meaningful, but it is not a full-model admission or serving result." +} diff --git a/docs/gmktec-evo-x2-deepseek-expert-slice-result-20260905.json b/docs/gmktec-evo-x2-deepseek-expert-slice-result-20260905.json new file mode 100644 index 0000000000..24d5f77d8d --- /dev/null +++ b/docs/gmktec-evo-x2-deepseek-expert-slice-result-20260905.json @@ -0,0 +1,416 @@ +{ + "scope": "isolated real-shape routed expert transfer only", + "checkpoint": "checkpoint", + "device": "cuda:0", + "layers": [ + 0 + ], + "experts": [ + 0, + 1, + 2, + 3, + 4, + 5 + ], + "repeats": 5, + "selected_tensor_count": 36, + "selected_bytes": 80216064, + "selected_mib": 76.5, + "tensors": [ + { + "name": "layers.0.ffn.experts.0.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.0.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.0.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.0.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.0.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.0.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.1.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.1.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.1.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.1.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.1.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.1.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.2.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.2.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.2.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.2.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.2.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.2.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.3.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.3.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.3.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.3.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.3.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.3.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.4.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.4.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.4.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.4.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.4.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.4.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.5.w1.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.5.w1.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.5.w2.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 4096, + 1024 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.5.w2.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 4096, + 64 + ], + "bytes": 262144 + }, + { + "name": "layers.0.ffn.experts.5.w3.weight", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.int8", + "shape": [ + 2048, + 2048 + ], + "bytes": 4194304 + }, + { + "name": "layers.0.ffn.experts.5.w3.scale", + "shard": "model-00002-of-00048.safetensors", + "dtype": "torch.float8_e8m0fnu", + "shape": [ + 2048, + 128 + ], + "bytes": 262144 + } + ], + "round_trips": [ + { + "h2d_seconds": 0.08656186307780445, + "d2h_seconds": 0.0016152210300788283, + "h2d_gib_per_second": 0.863047866505034, + "d2h_gib_per_second": 46.251893616289806 + }, + { + "h2d_seconds": 0.001131699071265757, + "d2h_seconds": 0.0011570280184969306, + "h2d_gib_per_second": 66.01315945805572, + "d2h_gib_per_second": 64.56803988813533 + }, + { + "h2d_seconds": 0.0009727838914841413, + "d2h_seconds": 0.0011578280245885253, + "h2d_gib_per_second": 76.79715084099735, + "d2h_gib_per_second": 64.52342633229124 + }, + { + "h2d_seconds": 0.0009776739170774817, + "d2h_seconds": 0.0011560780694708228, + "h2d_gib_per_second": 76.41303500590308, + "d2h_gib_per_second": 64.62109542843938 + }, + { + "h2d_seconds": 0.0009737049695104361, + "d2h_seconds": 0.0011937960516661406, + "h2d_gib_per_second": 76.7245044333722, + "d2h_gib_per_second": 62.579392137990354 + } + ], + "protected_service_touched": false, + "full_model_serving_claim": false +} diff --git a/docs/gmktec-evo-x2-deepseek-offload-feasibility.md b/docs/gmktec-evo-x2-deepseek-offload-feasibility.md new file mode 100644 index 0000000000..722911f551 --- /dev/null +++ b/docs/gmktec-evo-x2-deepseek-offload-feasibility.md @@ -0,0 +1,91 @@ +# DeepSeek-V4-Flash offload feasibility model + +This document converts the measured official checkpoint size into a bounded +feasibility calculation for the GMKtek EVO-X2 Strix Halo. It is a planning +artifact only. It does not download weights, start a model, or change the +protected service. + +## Inputs + +The current numerical payload measurements refer to the official +`DeepSeek-V4-Flash-0731` repository. The paper names that repository as its +official checkpoint, while the current model card reports a 304B label and the +paper reports 284B. The reproduction must therefore pin the repository commit +and record both labels rather than silently treating them as interchangeable. + +| Input | Value | Evidence | +|---|---:|---| +| Official `DeepSeek-V4-Flash-0731` safetensors payload | 166,886,535,336 bytes, approximately 155.43 GiB | Read-only `HEAD` measurement of all 48 shards at commit `7872f01b1d1fe23eabc4c98b48bffcef5a386062` | +| Routed expert pool described by paper | Approximately 140 GB | Supplied FreeToken paper | +| System memory available during live check | Approximately 18 GiB | `free -h` on the EVO-X2 | +| ROCm-reported VRAM aperture | 2 GiB | `rocm-smi` on the EVO-X2 | +| Model geometry | 43 layers, 256 routed experts, 1 shared expert, 6 routed experts active | Official `config.json` | + +The 18 GiB value is `MemAvailable`, not a guaranteed model allocation. The +operating system, protected service, runtime, KV cache, scheduler, and file +cache all compete for it. The 2 GiB ROCm aperture is reported separately and +must not be added to `MemAvailable` as if it were an independent pool available +for arbitrary model storage. + +## Resident-memory deficit + +Even an impossible best case that devoted all 18 GiB of currently available +system memory and the full 2 GiB device aperture to weights would provide only +20 GiB of addressable working space. The current official payload would still +exceed that optimistic budget by approximately 135.43 GiB. A realistic runtime budget +is smaller because it must reserve memory for execution and KV state. + +The payload-to-observed-availability ratio is approximately: + +```text +155.43 GiB / 18 GiB = 8.64x +``` + +This is a capacity deficit, not a tuning deficit. + +## Transfer lower bounds + +The paper states that a prefill can move roughly 140 GB of routed expert +weights. The following are ideal lower bounds for moving that volume once. They +exclude filesystem overhead, page faults, conversion, synchronization, and +repeated expert misses. + +| Sustained transfer rate | 140 GB lower bound | +|---:|---:| +| 50 GB/s | 2.80 seconds | +| 80 GB/s | 1.75 seconds | +| 100 GB/s | 1.40 seconds | +| 25 GB/s | 5.60 seconds | +| 10 GB/s | 14.00 seconds | +| 1 GB/s | 140 seconds | + +Decode is more demanding than this one-time bound because it repeatedly needs +routed expert blocks. If the working set is not resident, each miss incurs +additional transfer and synchronization. The expected token rate therefore +depends on routing locality, cache size, and the actual sustained source and +destination bandwidth, not only on the 13B active-parameter count. + +## Decision + +The current host cannot hold the official checkpoint in memory while retaining +a usable runtime and KV cache. A swap-backed run could be attempted only as a +separate stress experiment, and its throughput and latency would need to be +reported as offload behavior. It would not establish the paper's interactive +284B result. + +The correct next gate is therefore a metadata-only or tiny-slice prototype that +measures the actual layer-transfer path without downloading the full model. A +full checkpoint download is justified only if that prototype demonstrates a +sustained transfer path and a cache policy capable of keeping per-token misses +within an explicitly interactive latency budget. + +## Required evidence before a full attempt + +1. Exact model conversion and runtime support for the official FP4 plus FP8 + mixed format. +2. A measured layer-transfer bandwidth using a small synthetic tensor with the + same access pattern, without altering the protected service. +3. A calculated resident budget after reserving OS, runtime, KV, and recovery + headroom. +4. A predicted per-token miss volume and worst-case transfer latency. +5. A stop condition that prevents uncontrolled swap growth or system thrash. diff --git a/docs/gmktec-evo-x2-deepseek-route-transfer-projection-20260905.json b/docs/gmktec-evo-x2-deepseek-route-transfer-projection-20260905.json new file mode 100644 index 0000000000..a7cd19e269 --- /dev/null +++ b/docs/gmktec-evo-x2-deepseek-route-transfer-projection-20260905.json @@ -0,0 +1,46 @@ +{ + "scope": "analytical transfer-only lower bound", + "expert_bytes": 13369344, + "layers": 43, + "active_experts_per_layer": 6, + "routed_bytes_per_token_at_100_percent_miss": 3449290752, + "routed_gib_per_token_at_100_percent_miss": 3.21240234375, + "measured_h2d_gib_per_second": 77.976, + "rows": [ + { + "miss_rate": 1.0, + "moved_gib_per_token": 3.21240234375, + "transfer_seconds_per_token": 0.041197321531625114, + "transfer_only_tokens_per_second": 24.27342270861833 + }, + { + "miss_rate": 0.75, + "moved_gib_per_token": 2.4093017578125, + "transfer_seconds_per_token": 0.030897991148718836, + "transfer_only_tokens_per_second": 32.36456361149111 + }, + { + "miss_rate": 0.5, + "moved_gib_per_token": 1.606201171875, + "transfer_seconds_per_token": 0.020598660765812557, + "transfer_only_tokens_per_second": 48.54684541723666 + }, + { + "miss_rate": 0.25, + "moved_gib_per_token": 0.8031005859375, + "transfer_seconds_per_token": 0.010299330382906279, + "transfer_only_tokens_per_second": 97.09369083447332 + } + ], + "excluded": [ + "matrix computation", + "router and dispatch", + "attention and recurrent state", + "KV cache", + "synchronization", + "allocator overhead", + "cache lookup and eviction", + "D2H traffic" + ], + "full_model_serving_claim": false +} diff --git a/docs/gmktec-evo-x2-expert-block-prototype.md b/docs/gmktec-evo-x2-expert-block-prototype.md new file mode 100644 index 0000000000..7f5acc6053 --- /dev/null +++ b/docs/gmktec-evo-x2-expert-block-prototype.md @@ -0,0 +1,53 @@ +# GMKtek EVO-X2 expert-block transfer prototype + +This isolated prototype approximates MoE expert-cache misses with random, +non-contiguous host slices copied to a device tensor. Each block is +synchronized before the next block, making the result intentionally closer to +a serialized miss path than to an ideal bulk copy. It does not download or +load DeepSeek-V4-Flash and does not modify the protected service. + +## Environment and method + +- GPU: AMD Radeon 8060S, gfx1151. +- PyTorch: `2.13.0+rocm10.0.0`. +- HIP: `7.15.26333`. +- Host source buffer: 64 MiB float32 tensor. +- Each round: 64 randomly selected, 4 KiB-aligned blocks. +- Three warmup rounds and ten measured rounds per block size. +- `torch.cuda.synchronize()` after every block copy. + +## Results + +| Block size | Bytes per round | Effective host-to-device rate | Mean block latency | +|---:|---:|---:|---:| +| 4 KiB | 256 KiB | 0.167 GB/s | 24.6 microseconds | +| 16 KiB | 1 MiB | 0.914 GB/s | 17.9 microseconds | +| 64 KiB | 4 MiB | 5.009 GB/s | 13.1 microseconds | +| 256 KiB | 16 MiB | 16.876 GB/s | 15.5 microseconds | + +The earlier contiguous 64 MiB prototype measured 79.79 GB/s host-to-device. +The contrast shows that launch and synchronization overhead, not only the +memory fabric, dominates small scattered transfers. + +## Interpretation for DeepSeek offload + +These are synthetic lower-level measurements, not a prediction of model TPS. +They nevertheless bound the cost of a cache policy that services many small +expert misses individually. A policy that transfers 4 KiB blocks one at a time +would operate at roughly 0.21 percent of the contiguous-copy rate. Even 64 KiB +blocks reach only approximately 6.3 percent of that rate. + +The result strengthens the capacity decision: a 148.66 GiB checkpoint cannot be +made interactive merely by relying on fast contiguous unified-memory copies. +The runtime would need to batch and coalesce expert transfers, retain a very +high-locality working set, or accept much lower throughput. A full checkpoint +download remains unjustified until the actual runtime miss granularity and +coalescing behavior are demonstrated on a small synthetic model. + +## Next gate + +The next useful experiment is a batched version that copies a fixed total byte +count using one grouped operation per layer, then compares it with the +serialized result above. This will quantify how much batching the runtime must +provide before a large-model offload attempt can be considered technically +credible. diff --git a/docs/gmktec-evo-x2-final-campaign-report.md b/docs/gmktec-evo-x2-final-campaign-report.md new file mode 100644 index 0000000000..537ba55de3 --- /dev/null +++ b/docs/gmktec-evo-x2-final-campaign-report.md @@ -0,0 +1,172 @@ +# GMKtek EVO-X2 native ROCm FreeToken campaign report + +## Executive result + +The native ROCm and HIP port is functional and quality-qualified on the +GMKtek EVO-X2 with an AMD Radeon 8060S `gfx1151` GPU. The port serves Qwen +text and Gemma 4 GGUF workloads through an OpenAI-compatible local API. The +controlled Qwen Q4_K_M same-format decode result is effectively at parity with +the ROCm 10 llama.cpp control. Gemma FreeToken is slower than llama.cpp for +isolated single-request decode and long-prefill work, but it has lower TTFT and +higher aggregate throughput at the tested four- and eight-client loads. + +This report does not claim strict parity with the published NVIDIA results. +The paper does not expose every required fixture and policy field, and no +reference NVIDIA system is part of this campaign. + +## Platform and build + +| Field | Value | +| --- | --- | +| Host | GMKtek EVO-X2 | +| GPU | AMD Radeon 8060S | +| GFX target | `gfx1151` | +| ROCm | 10.0 | +| HIP | 7.15.26333 | +| PyTorch | `2.13.0+rocm10.0.0` | +| Execution mode | Native ROCm/HIP, no CUDA compatibility fallback | +| API | OpenAI-compatible local HTTP API | +| llama.cpp control | ROCm 10 build on the same host | + +## Qualified functionality + +- Native HIP extension build and import succeeded. +- CUDA-only capability detection and launch options are gated away on HIP. +- Qwen text streaming and non-streaming requests passed. +- Gemma 4 text and multimodal image controls passed. +- Deterministic text, JSON, multi-turn, long-context, and visual checks passed + within their documented scopes. +- The protected normal Qwen service was restored and health-checked after + every isolated candidate run. + +## Performance evidence + +### Qwen Q4_K_M same-format control + +Both runtimes used the same Q4_K_M GGUF checkpoint, tokenizer, caller-rendered +54-token raw prompt, and 256-token completion cap. + +| Runtime | Mean decode all samples | Mean decode samples 2 to 5 | Mean TTFT samples 2 to 5 | +| --- | ---: | ---: | ---: | +| FreeToken ROCm/HIP | 48.6028 TPS | **49.4357 TPS** | 424.26 ms | +| llama.cpp ROCm 10 | **49.1155 TPS** | 49.1772 TPS | 58.83 ms | + +FreeToken is 0.53 percent faster on the warmed requests 2 through 5. Across +all five samples, llama.cpp is 1.06 percent faster because FreeToken's first +request is slower. This is decode near-parity, not a material FreeToken lead. +The TTFT values have different cache behavior and are not an apples-to-apples +latency claim. + +Evidence: + +- FreeToken: `/home/operator/freetoken-amd/artifacts/qwen-gguf-warm-matrix-20260905T122817Z` +- llama.cpp: `/home/operator/freetoken-amd/artifacts/qwen-llama-warm-matrix-20260905T124007Z` + +### Gemma 4 + +The five-sample matched text control measured 53.0762 TPS for FreeToken and +56.8293 TPS for llama.cpp. The long 544-token prompt control measured 48.1441 +TPS for FreeToken and 54.3743 TPS for llama.cpp. FreeToken's concurrent +aggregate decode was 4.1 percent higher at four clients and 25.7 percent +higher at eight clients, with substantially lower mean TTFT in both cases. + +The exact values, prompt contracts, quality status, and raw artifact paths are +in [`gmktec-evo-x2-cross-model-manifest-20260905.json`](gmktec-evo-x2-cross-model-manifest-20260905.json). + +## Reliability and endurance + +The Qwen Q5 endurance campaign completed exactly 1,440 minute-cadence session +records with zero candidate and host swap, valid JSON, passing deterministic +state checks, and successful normal-service recovery. This campaign measures +state correctness, swap, thermal state, TTFT, and token-gap behavior. It does +not claim per-session prefill TPS. + +Gemma has completed bounded 30-session endurance, long-context, multimodal, +and concurrency controls. A full 1,440-session Gemma campaign remains +optional publication evidence and is not required for the current functional +release gate. + +## Rejected optimization candidates + +The following candidates were tested and not promoted because they regressed +quality, stability, or the primary throughput target: + +- Grouped Q4 and Q5 numerical paths with non-identical real-weight output. +- Python-level prefill overlap. +- Larger Gemma memory ratio. +- NVFP4 Marlin tile, warp-count, and staging variants that changed the + deterministic AIME result. +- One-fetch hybrid expert transfer. +- Six-request and eight-request scheduler alternatives as universal defaults. + +All rejected candidates retain raw artifacts and recovery evidence. + +## Unresolved claims + +### Strict NVIDIA comparison + +Not proven. The paper does not publish every input fixture, harness policy, +trace, and scoring detail required for strict reproduction, and no reference +NVIDIA hardware is in scope. + +### 284B interactive serving + +Not qualified. The exact 284B checkpoint is not present in the configured +model directory. The live capacity audit recorded 59 GiB system memory, 18 +GiB available at capture, 2 GiB dedicated VRAM reported by ROCm, and no 284B +payload. The primary paper describes DeepSeek-V4-Flash as a 284B model with +about 13B active parameters, six selected experts from 256, and roughly 140 GB +of FP4 expert weights, demonstrated on a 32 GB RTX 5090-class GPU. Those facts +make the exact model payload, host-memory budget, and bandwidth behavior +mandatory evidence before a Strix Halo reproduction can be claimed. See the +[primary paper](https://arxiv.org/abs/2608.16157) and the local +[`284B capacity manifest`](gmktec-evo-x2-284b-capacity-manifest-20260904.md). + +The paper reports approximately 53.8 GB/s host bandwidth on its 16-core +desktop reference. Our local transfer prototypes measured 79.79 GB/s for +contiguous 64 MiB copies, 5.009 GB/s for serialized random 64 KiB blocks, and +12.84 GB/s for grouped expert-like rounds after CPU staging. These are useful +transport bounds, but they are not evidence that the full 284B model will fit +or reach paper throughput. The scattered and staged measurements show why a +real checkpoint and production-shaped expert access pattern are still required. + +There is also a checkpoint identity gate. The paper names the official +`deepseek-ai/DeepSeek-V4-Flash-0731` checkpoint and describes it as 284B with +native MXFP4 routed experts. The current model card reports 304B parameters +and BF16, I64, F32, F8_E4M3, and I8 tensor types. The repository's release +commit `9e165c30e2704aec5d9d593cce3eebd58bbef1cb` predates the current model-card +update and must be pinned for a faithful reproduction. A valid capacity or +throughput claim must record that revision and report the 284B versus 304B +metadata discrepancy explicitly. + +The pinned release commit and the current model-card commit both expose 48 +safetensors shards totaling 166,886,535,336 bytes, approximately 155.43 GiB. +This is the measured capacity reference for the named official repository; +the parameter-count discrepancy must remain visible in the final report. The +`config.json` bytes are identical at both commits, with SHA-256 +`6c8f3d2d3b48707541b88f32f22ef3f0f8a6b57d8523281e2b8d3cdb0ae9a023`. + +### Additional archived model identifiers + +Archived routing names without an admitted payload are not treated as failed +FreeToken benchmarks. Each would require an exact checkpoint, backend +admission, deterministic quality fixture, TPS matrix, and recovery evidence. + +## Reproduction index + +- [`gmktec-evo-x2-upstream-handoff-checklist.md`](gmktec-evo-x2-upstream-handoff-checklist.md) +- [`gmktec-evo-x2-campaign-completion-audit.md`](gmktec-evo-x2-campaign-completion-audit.md) +- [`gmktec-evo-x2-cross-model-matrix-20260904.md`](gmktec-evo-x2-cross-model-matrix-20260904.md) +- [`gmktec-evo-x2-cross-model-manifest-20260905.json`](gmktec-evo-x2-cross-model-manifest-20260905.json) +- [`gmktec-evo-x2-amd-run-log.md`](gmktec-evo-x2-amd-run-log.md) +- [`gmktec-evo-x2-284b-capacity-manifest-20260904.md`](gmktec-evo-x2-284b-capacity-manifest-20260904.md) + +## Review status + +The source branch containing the native ROCm/HIP implementation and this +evidence set is the branch proposed in upstream PR #260. The PR remains open. +Merge status and external maintainer review are separate from the completed +local AMD qualification gates. The latest upstream page still shows the PR as +open. The two Copilot build-detection findings are marked outdated and the +branch records their fix in commit `54d6ab2`; no new actionable review request +was visible during the latest handoff check. diff --git a/docs/gmktec-evo-x2-freetoken-qwen-replication-plan.md b/docs/gmktec-evo-x2-freetoken-qwen-replication-plan.md new file mode 100644 index 0000000000..dc3483eacb --- /dev/null +++ b/docs/gmktec-evo-x2-freetoken-qwen-replication-plan.md @@ -0,0 +1,476 @@ +# GMKtek EVO-X2 FreeToken Qwen replication and Strix Halo optimization plan + +## Decision and success statement + +This plan targets only GMKtek EVO-X2, a Ryzen AI Max+ 395 with Radeon 8060S +(`gfx1151`) and shared LPDDR5X memory. It does not alter secondary test host, another test host, +llama-swap, or any production model service. + +The first target is the exact model used for FreeToken's documented 8 GB laptop +result: `Qwen/Qwen3.6-35B-A3B`, using the upstream-supported deployment +format that can be reproduced on both systems. The published claim to +replicate is 39.3 generated tokens per second on an 8 GB RTX 4060 laptop. +This is a model-specific reference, not a general statement that all FreeToken +models fit in 8 GB of VRAM. + +The program is successful only when GMKtek EVO-X2 can run the documented Qwen +workload through the native ROCm and HIP FreeToken server with: + +1. A fully recorded, exact model and workload contract. +2. Deterministic greedy-output equivalence to a trusted reference for each + test prompt, plus task-level quality scores where deterministic equality is + unsuitable. +3. Repeated warm and cold performance measurements with raw artifacts. +4. Explicit measurement of decode TPS, prompt TPS, TTFT, tail token latency, + memory use, cache behavior, CPU utilization, GPU utilization, clocks, + temperatures, and throttling. +5. An evidence-backed comparison to the published 39.3 TPS reference on an + equivalent workload, without claiming equality when prompts, sampling, + hardware tier, or metric definitions differ. +6. A stable configuration that is safe to expose through FreeToken's + OpenAI-compatible API after the campaign, but before any llama-swap work. + +The longer-term ambition is to exceed the published result on the same model. +That ambition is a hypothesis, not an acceptance assumption. It must be +supported by a matched benchmark and quality evidence. + +## Why this is a different engineering problem on Strix Halo + +FreeToken's 8 GB RTX 4060 result uses a discrete GPU, dedicated VRAM, host +DRAM, and a PCIe link. Its MoE policy can retain hot experts in VRAM while +placing other experts in host memory, fetching misses or computing selected +misses on the CPU. + +GMKtek EVO-X2 has UMA. Its CPU and Radeon 8060S access the same memory pool. This +can remove PCIe-copy cost and can permit a larger hot-expert cache than an 8 GB +discrete GPU. It can also be worse if the CPU fallback, GPU compute, KV cache, +and operating system contend for the same LPDDR5X channels. A direct copy of +the CUDA policy is therefore an invalid optimization target. The AMD runtime +needs a measured UMA policy. + +## Non-negotiable controls + +### Scope and safety + +- Maintain a GMKtek EVO-X2 host allowlist in every benchmark launcher and refuse any + other hostname or IP address before contacting a server. +- Use an isolated work directory under `/home/operator/freetoken-amd/artifacts/`. +- Bind experiments to loopback or a non-production GMKtek EVO-X2 test port. +- Do not change llama-swap configuration, routes, model aliases, startup + services, or model files used by production services. +- Store credentials only as environment-variable references. Do not save, + print, commit, or upload secrets. +- Do not overwrite previous evidence. Every experiment receives a UTC + timestamp, a unique run identifier, and a manifest. + +### Environment freeze + +For each candidate, save a machine-readable manifest containing: + +- Git commit, branch, clean or dirty tree state, and patch hash. +- Linux distribution, kernel, CPU microcode, BIOS version, memory amount, + configured UMA aperture, and storage mount information. +- ROCm runtime, HIP compiler, AMD GPU driver, PyTorch ROCm build, Triton + version, Python ABI, compiler flags, and `HSA_OVERRIDE_GFX_VERSION` if set. +- `rocminfo`, `rocm-smi`, CPU topology, NUMA map, memory-frequency data where + exposed, and the process CPU affinity and priority. +- Exact model revision, all shard checksums, tokenizer revision, configuration + files, conversion output checksums, and FreeToken weight-format version. +- `TORCH_EXTENSIONS_DIR` location and native-extension binary hash. A normal + benchmark must reuse the compiled HIP extension, not compile during timing. + +Reject a performance comparison if any material environment item differs and +the difference is not recorded in the comparison table. + +## Phase 0: establish the paper replication contract + +### 0.1 Extract the authors' actual benchmark protocol + +Read the paper, repository history, benchmark scripts, released configs, issue +threads, and desktop defaults. Produce +`artifacts//upstream-protocol.md` with citations and exact quotes kept +short. Resolve, rather than assume: + +- Checkpoint name, source revision, quantization, weight format, and total + downloaded size used for the 39.3 TPS RTX 4060 result. +- RTX 4060 laptop CPU, RAM capacity and speed, operating system, CUDA version, + driver, FreeToken commit, GPU memory budget, and any desktop-app defaults. +- Prompt text and token count, completion length, warmup procedure, context + reuse state, concurrency, sampling parameters, stop tokens, and whether the + first generated token is excluded from decode measurement. +- Whether 39.3 TPS is mean, median, best run, a workload average, or a + single-run sample; record its confidence interval if provided. +- MoE backend, expert cache budget, CPU thread count, `ft bench bw` result, + CPU split, and any automatic policy selected by the reference machine. +- TTFT definition and whether server-internal timing or client-observed timing + is used. + +Do not label a GMKtek EVO-X2 result as a reproduction until all fields are known or +explicitly listed as unavailable from the authors. + +### 0.2 Define a metric dictionary before testing + +All benchmark scripts must emit these definitions unchanged: + +| Metric | Definition | +| --- | --- | +| Cold start | Process launch through first successful non-streamed response, including model load and extension compilation only when intentionally requested. | +| Warm TTFT | Client-observed request send to first SSE content token after a completed warmup. | +| Decode TPS | `(completion_tokens - 1) / (last_content_token_time - first_content_token_time)`. Report zero or one token completions separately. | +| Prompt TPS | Input tokens divided by client-observed TTFT, labelled end-to-end rather than kernel-only. | +| Token-gap p50, p95, p99 | Distribution of streamed content-token intervals, excluding SSE framing-only events. | +| Quality score | Task-specific judged result, with model output, scorer version, and parsing errors retained. | +| Effective memory bandwidth | Actual bytes moved divided by measured wall time for the relevant serving phase. Never substitute a microbenchmark ceiling. | + +Report client and runtime-internal figures in separate columns. Do not compare +one runtime's internal timing to the other's HTTP timing. + +### 0.3 Create the baseline protocol package + +Create a versioned benchmark package under `benchmarks/gmk_evo_x2/` with: + +- A static JSON request corpus and expected tokenizer counts. +- A local API client that captures raw SSE timestamps using a monotonic clock. +- A warmup runner, a cold-start runner, a fixed-length decode runner, and a + multi-turn agentic runner. +- A process guard that checks the host identity and fails closed outside + GMKtek EVO-X2. +- Telemetry collection with timestamps aligned to each request. +- A manifest writer and checksum verifier. +- A result parser that emits JSON, CSV, and a Markdown table without changing + raw logs. +- Unit tests for the TPS calculation, token-event parsing, error + classification, host allowlist, and manifest validation. + +## Phase 1: qualify the model and its quality before optimization + +### 1.1 Use the exact primary model path + +The main candidate is the official `nvidia/Qwen3.6-35B-A3B-NVFP4` checkpoint +already supported upstream and validated functionally on GMKtek EVO-X2. Preserve +the original model directory as read-only. Build any FreeToken fast-weight +conversion once, checksum it, and reuse it across every trial. + +Run a separate, explicitly labelled compatibility check for the checkpoint and +format used by the authors if it differs from NVFP4. Do not merge the two +results into one number. + +### 1.2 Establish three independent correctness references + +Use three types of evidence: + +1. **FreeToken NVIDIA reference**: upstream FreeToken on supported NVIDIA + hardware when available. Fix greedy decoding and retain raw token IDs. +2. **Independent AMD control**: llama.cpp ROCm on GMKtek EVO-X2 using a compatible + Qwen quantization and a carefully documented template. It is a quality + control, not a performance proxy when the format differs. +3. **Model-level evaluation**: a small, fixed benchmark suite with exact + prompts and deterministic scoring. + +### 1.3 Quality suite + +The suite must contain at least: + +- Arithmetic and structured reasoning questions with machine-verifiable final + answers. +- Code generation tasks that execute in a sandboxed test harness. +- Retrieval and instruction-following prompts with explicit required facts. +- Tool-call JSON generation with schema validation. +- A multi-turn editing and correction set that exercises FreeToken's semantic + cache behavior. +- Long-context prompts at 2K, 8K, 16K, and the largest stable context that + fits the selected KV allocation. + +For every prompt, retain rendered prompt text, token IDs when practical, model +response, finish reason, scorer result, and error class. Categorize +differences as template or tokenizer, numerical drift, truncation, parsing +failure, serving failure, or genuine task-quality regression. + +The gate before performance tuning is: + +- No crash, corruption, NaN, malformed streaming sequence, or silent fallback. +- Greedy outputs must be byte-identical where the same tokenizer, template, + weights, precision, and decode settings are used. +- Where cross-runtime byte identity is impossible, the quality suite must show + no statistically meaningful regression relative to the selected reference. + +## Phase 2: establish unoptimized but comparable GMKtek EVO-X2 baselines + +### 2.1 Baseline matrix + +Run five clean, independently started server samples per row, after a defined +warmup. Capture at least: + +| Row | Purpose | +| --- | --- | +| Upstream-like automatic policy | Establish how the current port behaves without hand tuning. | +| GPU-resident maximum safe expert cache | Test UMA's likely advantage. | +| Explicit offload | Establish whether FreeToken's discrete-GPU policy transfers at all. | +| Explicit CPU | Measure CPU-only expert-miss cost and shared-memory contention. | +| Explicit hybrid | Measure whether concurrent CPU and GPU execution helps or hurts on UMA. | +| llama.cpp ROCm control | Establish the current AMD alternative using a documented compatible workload. | + +For every row record warm and cold measurements, 128-token decode, 512-token +decode, the paper-matched workload, and the agentic workload. Separate stable +samples from samples contaminated by active disk I/O, CPU contention, thermal +transition, process leaks, or unexpected compilation. + +### 2.2 Telemetry and contamination controls + +Collect, at one-second cadence and around every request: + +- GPU clock, temperature, power, memory activity, GPU busy, and reset events. +- CPU frequency, package power, per-core utilization, migrations, page faults, + context switches, major faults, and memory pressure. +- RAM and swap use, cache residency, paging activity, I/O throughput, and + blocked processes. +- Expert-cache hits, misses, evictions, bytes fetched, CPU expert work, + resident slots, KV bytes, and cache resize events. +- HIP graph-capture status, stream synchronization counts, kernel launch + counts, and extension cache hits. + +Reject and rerun any sample with swapping, thermal throttling, unexpected +compilation, stale server processes, active model-copy jobs, or unexplained +host I/O contention. Preserve rejected evidence and its rejection reason. + +## Phase 3: make the AMD implementation observable + +### 3.1 Add low-overhead serving counters + +Add a structured, opt-in telemetry mode. It must never alter numerical output +or become enabled by default. Counters must include per token and aggregate: + +- Router duration, selected expert IDs, unique experts, and route reuse. +- GPU-cache hits, misses, evictions, resident bytes, and cache wait time. +- Expert staging or read time, CPU compute time, HIP copy time if any, and + queue overlap time. +- Attention, dense projection, MoE projection, normalization, sampling, and + synchronization time. +- GPU and CPU work submitted versus completed, including backpressure. + +Use an event-ring buffer and bulk flush at request completion. Do not emit +one log line per kernel during a scored run. + +### 3.2 Build a profiler ladder + +Use three tiers, in order: + +1. Application counters for every performance run. +2. HIP events around named execution regions for candidates that pass the + application gate. +3. ROCm profiler traces only on representative runs because tracing changes + timing. + +For each suspected bottleneck, first prove that its percentage of end-to-end +decode time is large enough to matter. A microbenchmark improvement is not a +candidate for integration unless it survives a complete API run with identical +quality output. + +## Phase 4: derive a Strix Halo UMA execution policy + +### 4.1 Measure the actual memory system + +Create targeted measurements for: + +- CPU-only sequential and realistic expert-shaped reads. +- GPU-only read and quantized GEMV throughput on Qwen dimensions. +- Concurrent CPU expert compute and GPU expert compute. +- Concurrent CPU reads and GPU compute. +- Expert-cache promotion and eviction under the actual route sequence. +- KV-cache growth with 2K, 8K, 16K, and long-context requests. + +The critical output is a contention curve, not a peak bandwidth number. Plot +decode TPS and token latency against CPU contribution, hot-expert cache size, +and KV allocation. + +### 4.2 Replace PCIe-centric assumptions + +Implement a `uma` policy mode that starts from measured resource contention: + +- Prefer GPU execution for hot experts when sufficient shared-memory headroom + exists. +- Cap CPU fallback when concurrent CPU work reduces GPU progress more than it + contributes. +- Make the expert-cache target a function of free memory, current KV use, + observed route locality, and the measured contention curve. +- Allocate pinned host buffers only if profiling proves they help on ROCm UMA. + Do not assume pinned memory is beneficial just because it helps PCIe. +- Avoid copy paths that duplicate bytes inside the same physical memory pool + when a direct-access or zero-copy path is correct and measurable. +- Resize the cache only at scheduler safe points, with hysteresis to avoid + cache thrash during alternating long-context and short decode requests. + +The policy must fall back to the existing portable behavior on discrete AMD +hardware and must not modify the CUDA decision path. + +### 4.3 Optimizer acceptance rule + +For every policy candidate, run the full five-sample API matrix and quality +check. Accept only when all apply: + +- Median decode TPS improves by at least 1 percent over the current accepted + baseline, or the confidence interval proves a smaller improvement is real. +- Mean TTFT does not regress by more than 5 percent unless the candidate is + explicitly a decode-only mode. +- p99 token gap does not materially worsen. +- No quality, determinism, memory-safety, crash, or resource-leak regression. +- The improvement persists in a second clean-host matrix. + +## Phase 5: HIP and ROCm kernel program + +### 5.1 Start with profile-ranked Qwen kernels + +Use the Qwen trace, not Gemma's profile, to rank work. Expected candidates are +the NVFP4 expert GEMV path, quantization deblocking, routed-expert gather and +scatter, router reductions, attention decode, and synchronization between +small expert operations. Recompute the ranking after every accepted system +policy change. + +### 5.2 Kernel development principles + +- Maintain separate CUDA and HIP paths behind compile-time guards. +- Use `gfx1151` feature gates that do not accidentally apply to other AMD + architectures. +- Prefer current ROCm intrinsics and inspect generated ISA before judging a + kernel by source appearance. +- Match Qwen's real matrix sizes, expert count, top-k, activation dtype, and + batch shape in every microbenchmark. +- Measure register pressure, occupancy, wave size, LDS use, cache behavior, + and memory coalescing before changing launch geometry. +- Fuse adjacent operations only when the full decode trace proves that launch + or global-memory overhead dominates and the fusion does not reduce occupancy. +- Preserve numerically safe accumulation and check output tolerances against a + high-precision reference. + +### 5.3 Likely high-value technical leaps to investigate + +These are experiments, not promised outcomes: + +1. **Route-aware expert prefetch.** Predict a small next-token expert set + from recent routes, stage it into the shared-memory cache during current + token compute, and measure false-prefetch cost versus cache-miss reduction. +2. **Layer-local route batching.** Group duplicate expert selections within a + layer without changing route order or output semantics, reducing tiny launch + and synchronization overhead. +3. **Persistent decode scheduler.** Replace host-driven chains of small HIP + launches with a graph-captured or persistent sequence where ROCm profiling + proves launch overhead is dominant. +4. **Qwen NVFP4 RDNA3.5 GEMV specialization.** Specialize the exact expert + shapes and quantization layout for `gfx1151`, using vectorized loads and + ROCm-supported dot-product instructions where the generated ISA confirms + the intended instruction sequence. +5. **Unified KV and expert cache allocator.** Use a single pressure-aware + allocator so the cache gives back memory to long context before paging or + repeated reallocation occurs. +6. **Cooperative CPU-GPU routing budget.** Choose the CPU share per token or + per layer from recent observed service time, rather than one static hybrid + ratio from an isolated bandwidth benchmark. + +Each experiment requires a design note, a baseline, a rollback commit or +feature flag, microbenchmark evidence, complete-server evidence, quality +evidence, and a documented accept or reject decision. + +## Phase 6: paper-parity and superiority trials + +### 6.1 Replication trial + +Once protocol fields are resolved, run the exact paper-matched Qwen workload +on GMKtek EVO-X2 with the selected stable configuration: + +- At least five independent warm-server samples. +- At least three cold-start samples, reported separately. +- Identical prompt and output length rules. +- Greedy decoding unless the source protocol specifies otherwise. +- Same reported statistic as the paper, plus mean, median, standard deviation, + min, max, bootstrap confidence interval, and raw samples. +- Full telemetry and quality artifact bundle. + +The parity threshold is the paper's 39.3 TPS only if the metric and workload +are matched. If they are not, report an explicitly labelled comparable result +and enumerate every remaining difference. + +### 6.2 Better-than-NVIDIA trial + +Only after a successful replication trial, test claimed advantages of UMA: + +- Higher expert-cache residency at equivalent KV capacity. +- Lower cache-miss cost without PCIe transfer. +- Stable decode under long multi-turn contexts. +- Better tail token latency under the same quality and concurrency settings. +- Sustained throughput with no thermal or memory-pressure degradation. + +Use the NVIDIA reference as a published comparison point, not as a reason to +hide protocol differences. A claim that GMKtek EVO-X2 exceeds the NVIDIA result +requires a same-model, same-precision, same-workload, same-TPS-definition +comparison, or a clearly bounded claim such as "higher end-to-end warm decode +TPS on this specified request." + +## Phase 7: reliability and API qualification + +Run a 24-hour endurance test only after performance and quality gates pass. +It must use bounded test traffic, a non-production endpoint, and rotate among +short, paper-matched, long-context, streaming, non-streaming, and cache-edit +workloads. Record: + +- Request success rate, error taxonomy, restarts, memory high-water marks, + cache resizes, context truncations, GPU resets, and server leaks. +- TPS and TTFT trend over time, including first-hour versus final-hour values. +- Exact output hash for repeated deterministic canary prompts. +- Process cleanup after shutdown and no orphan worker, blocked I/O, or runaway + compilation processes. + +Pass conditions: no data corruption, no silent fallback to CPU-only or Vulkan, +no swap thrash, no unbounded growth, no unacceptable output regression, and a +documented recovery procedure tested once. + +## Phase 8: publication and upstream readiness + +Publish a reproducibility bundle in the fork containing: + +- Executive result table that clearly separates functionality, quality, + paper-parity, and superiority claims. +- Full build guide, environment manifest, model provenance, and checksums. +- Benchmark harness source, fixed request corpus where licensing permits, raw + result JSON, sanitised logs, and result-generation script. +- Performance tables with client versus runtime timing clearly separated. +- Rejected-candidate register so future work does not repeat failed paths. +- Architecture explanation of why UMA differs from the CUDA offload design. +- Known limitations, non-goals, and exact commands required to reproduce. + +Before updating the existing upstream pull request, split changes into focused +commits: portable HIP correctness, instrumentation and tests, and optionally a +portable AMD optimization. Keep GMKtek EVO-X2-specific evidence and tuning defaults +in this fork unless upstream maintainers request them. Do not claim general +AMD support from a single `gfx1151` result. + +## Required acceptance table + +| Gate | Evidence required | Status at plan creation | +| --- | --- | --- | +| Native ROCm and HIP execution | Compiled HIP extension, ROCm telemetry, no substitute backend | Achieved for existing Qwen and Gemma validation | +| OpenAI-compatible serving | `/v1/models`, streaming and non-streaming responses | Achieved for existing Qwen and Gemma validation | +| Qwen model quality | Exact reference or scored task suite | Not yet complete | +| Authors' 39.3 TPS protocol reconstructed | Cited protocol artifact with every field resolved or marked unavailable | Not yet complete | +| Matched Qwen TPS replication | Five-sample paper-matched result and raw evidence | Not yet complete | +| Exceeds paper result | Same-model, same-workload, same-metric evidence | Not yet complete | +| Beats AMD llama.cpp Qwen control | Matched ROCm quality and performance matrix | Not yet complete | +| UMA policy is beneficial | Complete API matrix, telemetry, second clean-host confirmation | Not yet complete | +| Long-run reliability | 24-hour isolated endurance artifact | Not yet complete | +| Upstream-ready documentation | Reviewed, reproducible, secret-safe bundle | Not yet complete | + +## Immediate next actions + +1. Resolve the authors' 39.3 TPS protocol and freeze the Qwen benchmark + contract. +2. Implement the GMKtek EVO-X2-only harness and manifest schema before altering + another performance kernel. +3. Re-run the current Qwen NVFP4 baseline with five samples, correct telemetry, + and quality canaries. +4. Measure UMA contention across cache size and CPU contribution. +5. Use the resulting trace to select one systems-policy candidate and one + profile-ranked HIP-kernel candidate. + +No larger MoE model is admitted to the performance campaign until the Qwen +paper-replication gate is complete. A larger model can receive a separate +capacity feasibility assessment, but it must not consume the evidence or +optimization budget needed to establish this primary result. diff --git a/docs/gmktec-evo-x2-fused-moe-prototype.md b/docs/gmktec-evo-x2-fused-moe-prototype.md new file mode 100644 index 0000000000..80af24b1a4 --- /dev/null +++ b/docs/gmktec-evo-x2-fused-moe-prototype.md @@ -0,0 +1,69 @@ +# GMKtek EVO-X2 fused MoE expert prototype + +## Purpose + +This bounded native HIP experiment measures a representative expert-row +operation on the GMKtek EVO-X2 without downloading or loading a large model +checkpoint. It is intended to identify whether a device kernel can consume +multiple routed expert rows from mapped host memory while performing the dot +product and reduction on the GPU. + +The prototype is not a model benchmark and must not be reported as FreeToken +tokens per second. Its packed signed-int4 data is deliberately simpler than +the production Qwen NVFP4 format. The result is therefore a kernel-path +baseline for the next implementation step, not proof of model equivalence. + +## Configuration + +| Field | Value | +| --- | --- | +| Backend | Native HIP, compiled with `hipcc` | +| Target | `gfx1151` | +| Experts per launch | 64 | +| Values per expert row | 16,384 | +| Packed bytes per row | 8,192 | +| Weight source | HIP mapped pinned host memory | +| Activation type | FP32 | +| Workgroup | 256 threads per expert row | +| Timed launches | 100, after one warmup launch | +| Measurement | HIP events around the timed launch loop | + +Each byte contains two signed four-bit weights. One workgroup processes one +expert row, dequantizes the nibbles in the kernel, multiplies by the resident +activation vector, and reduces to one output value. + +## Result + +The remote compile and execution completed successfully: + +```text +experts=64 values=16384 packed_bytes=8192 rounds=100 elapsed_ms=1.604692 effective_weight_GBps=32.672189 +``` + +The measured effective packed-weight read rate was **32.672 GB/s** for this +serialized mapped-host expert-row workload. The compiler emitted only unused +return-value warnings for the intentionally compact prototype; the kernel +completed and returned exit code zero. + +## Interpretation + +This result confirms that a fused HIP kernel can perform expert-row address +selection, mapped-host reads, on-device signed-int4 unpacking, multiply, and +reduction in one launch. It does not establish that the production NVFP4 +kernel will reach this rate, because NVFP4 metadata, scaling, tensor layout, +routing, and the production hidden dimensions are different. + +The result is also not directly comparable to the earlier 281.916 GB/s +device-resident gather or 112.908 GB/s mapped-host gather microbenchmarks. +Those tests measured bulk gather bandwidth without the dequantization, +dot-product, and reduction work. The present experiment intentionally includes +that compute to expose the combined path that a fused MoE implementation must +optimize. + +## Next action + +Replace the synthetic signed-int4 row format with the exact production NVFP4 +metadata and tensor dimensions, then compare the fused kernel against the +current FreeToken expert implementation using deterministic output hashes. +Only a complete API run that preserves quality, TTFT, decode TPS, tail +latency, recovery, and concurrency can promote a fused candidate. diff --git a/docs/gmktec-evo-x2-gemma4-comparison-report.md b/docs/gmktec-evo-x2-gemma4-comparison-report.md new file mode 100644 index 0000000000..2eb4fa678c --- /dev/null +++ b/docs/gmktec-evo-x2-gemma4-comparison-report.md @@ -0,0 +1,127 @@ +# Gemma 4 AMD comparison report + +This report consolidates the Gemma 4 Q4 GGUF evidence collected on the GMKtec +EVO-X2 Strix Halo system. FreeToken used the native ROCm/HIP path. The +comparison control used the ROCm 10 llama.cpp build. Both runs used the same +14 GB Gemma 4 26B A4B Q4_0 GGUF and the same isolated loopback test procedure. + +The report distinguishes measured user-visible API behavior from internal or +protocol-limited observations. It does not claim strict replication of the +FreeToken NVIDIA paper because the paper's complete prompts, fixtures, cache +policy, and reference hardware are not available. + +## Executive result + +FreeToken produced the stronger interactive result in the tested concurrent +matrix. Its four-client aggregate decode was 26.53 tokens/s with 363 ms mean +TTFT. llama.cpp produced 22.42 aggregate tokens/s with 3.47 s mean TTFT and +6.90 s p95 TTFT. llama.cpp's isolated per-request decode rate was higher, but +its one-slot configuration serialized concurrent requests. + +For single fixed-length requests, FreeToken averaged 50.87 decode tokens/s +versus 30.50 for llama.cpp. llama.cpp had faster client-observed prefill in +that matrix, 478.13 versus 174.27 tokens/s. These are runtime controls, not a +claim that the two engines have identical scheduler internals. + +## Single-request matrix + +| Metric | FreeToken native ROCm/HIP | llama.cpp ROCm 10 | +| --- | ---: | ---: | +| Completed samples | 5/5 | 5/5 | +| Prompt tokens | 34 each | 34 each | +| Completion tokens | 127 each | 127 each | +| Mean TTFT | 203.07 ms | 118.71 ms | +| Mean client prefill | 174.27 tokens/s | 478.13 tokens/s | +| Mean decode | 50.87 tokens/s | 30.50 tokens/s | +| Median decode | 53.21 tokens/s | 22.43 tokens/s | +| p95 decode | 53.67 tokens/s | 46.98 tokens/s | +| Aggregate p99 token gap | 132.36 ms | 283.97 ms | + +FreeToken artifact: +`/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260904T150838Z/` + +llama.cpp artifact: +`/home/operator/freetoken-amd/artifacts/gemma4-llamacpp-vision-20260904T152038Z/` + +## Four-client concurrency matrix + +| Metric | FreeToken native ROCm/HIP | llama.cpp ROCm 10 | +| --- | ---: | ---: | +| Requests | 12/12 | 12/12 | +| Mean per-request decode | 28.50 tokens/s | 56.76 tokens/s | +| Aggregate decode | 26.53 tokens/s | 22.42 tokens/s | +| Mean TTFT | 363.13 ms | 3.471 s | +| p95 TTFT | 494.80 ms | 6.901 s | +| Mean per-request p99 gap | 46.65 ms | 18.02 ms | +| Aggregate p99 gap | 56.69 ms | 18.54 ms | + +The llama.cpp result has a higher isolated decode rate but a lower aggregate +rate because its tested one-slot configuration serialized concurrent work. +FreeToken admitted the four clients with substantially lower TTFT. + +FreeToken artifact: +`/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260904T152716Z/` + +llama.cpp artifact: +`/home/operator/freetoken-amd/artifacts/gemma4-llamacpp-vision-20260904T161303Z/` + +## Long-context behavior + +FreeToken used the OpenAI chat protocol and passed the exact `LONG_OK` marker: + +| Prompt size | Reported prompt tokens | TTFT | Client prefill | Decode | +| ---: | ---: | ---: | ---: | ---: | +| 4,096 characters | 2,528 | 8.238 s | 306.86 tokens/s | 57.00 tokens/s | +| 8,192 characters | 5,033 | 12.879 s | 390.80 tokens/s | 65.46 tokens/s | + +The 16,384-character request failed closed at the configured 8,192-token +context ceiling. + +With llama.cpp's explicit `--reasoning off --reasoning-budget 0` flags, the +4,096- and 8,192-character requests also passed visible `LONG_OK` checks. They +reported 1.866 and 1.992 s TTFT, 1,354.65 and 2,526.23 client prefill tokens/s, +and 63.36 and 47.80 decode tokens/s. The 16K request remained rejected at the +8,192-token context ceiling. Earlier default-reasoning attempts are retained +as protocol diagnostics, not mixed into this accepted comparison. + +## Quality and multimodal evidence + +FreeToken's extended image suite passed 21 of 21 cases across three repetitions. +The suite checked exact colors, spatial relationships, valid visible outputs, +and a constrained visual description. The visual description used 309 prompt +tokens and 64 completion tokens, with 1,139 ms TTFT and 52.57 visible decode +tokens/s. + +The short Gemma arithmetic control returned the exact `323` answer. The +llama.cpp wrapper also produced image-quality artifacts, but its long-context +visible-answer contract remained unresolved and is not treated as equivalent +quality evidence. + +## Endurance and recovery + +FreeToken completed a bounded 30-session Gemma control with 30 exact answers, +zero protocol errors, 216.61 ms mean TTFT, and 234.25 ms p95 TTFT. Protected +Qwen recovery returned `status: ok` and `maintenance: serving` after teardown. + +The Qwen FreeToken path separately completed the full 1,440-session endurance +qualification. Gemma has not yet completed a 1,440-session or 24-hour +endurance campaign. + +## Remaining limitations + +1. The llama.cpp Gemma long-context response contract needs a native invocation + that produces a comparable visible answer before that matrix can be scored. +2. The paper's exact NVIDIA reference conditions remain unavailable, so strict + paper replication is not proven. +3. The 50 percent AMD speed-improvement target has not been reached by a + quality-preserving candidate. +4. The 284B capacity claim still requires a clean GPU-visible unified-memory + manifest and a model-specific capacity test. + +## Source evidence + +- `gmktec-evo-x2-amd-run-log.md` contains the dated artifact ledger. +- `benchmark_gemma4_gguf_text_matrix.py` defines the single-request metrics. +- `benchmark_gemma4_concurrency.py` defines the concurrent metrics. +- `benchmark_gemma4_long_context.py` defines the long-context quality gate. +- `benchmark_gemma4_endurance.py` defines the bounded endurance gate. diff --git a/docs/gmktec-evo-x2-gemma4-q4-text-control-20260830.md b/docs/gmktec-evo-x2-gemma4-q4-text-control-20260830.md new file mode 100644 index 0000000000..6079b72787 --- /dev/null +++ b/docs/gmktec-evo-x2-gemma4-q4-text-control-20260830.md @@ -0,0 +1,25 @@ +# GMKtek EVO-X2 Gemma4 Q4 text control, 2026-08-30 + +The native ROCm/HIP FreeToken GGUF path was qualified against the on-host +`gemma-4-26B_q4_0-it.gguf` text model. The test was isolated on port 1923 and +the Qwen production recovery helper ran when the temporary process exited. + +The caller rendered Gemma's embedded canonical chat template once, submitted +the resulting raw prompt to `/v1/completions`, and disabled extra special-token +insertion. The server and local GGUF tokenizer agreed on 30 prompt tokens. + +| Field | Result | +| --- | --- | +| Prompt hash | `0f65acd07a4f57b2644f7720b725d7795999406b90a9f91486da5effa39bb95d` | +| Question | `What is 17 times 19? Reply with only the decimal number.` | +| Expected output | `323` | +| Actual output | `323` | +| Server prompt tokens | 30 | +| Completion tokens | 4 | +| Steady decode TPS | 57.05 | + +The preserved GMKtek EVO-X2 evidence is +`/home/operator/freetoken-amd/artifacts/gemma4-gguf-text-20260830T035542Z/quality.json`. +This proves text-only loader, template, OpenAI-compatible completion API, +token accounting, and a deterministic basic quality response. It does not yet +qualify image input through the matching multimodal projector. diff --git a/docs/gmktec-evo-x2-gemma4-q4-vision-control-20260830.md b/docs/gmktec-evo-x2-gemma4-q4-vision-control-20260830.md new file mode 100644 index 0000000000..6b0f8aeb57 --- /dev/null +++ b/docs/gmktec-evo-x2-gemma4-q4-vision-control-20260830.md @@ -0,0 +1,139 @@ +# GMKtek EVO-X2 Gemma 4 Q4 GGUF vision control + +## Scope + +This control proves the AMD ROCm/HIP path for the locally available +`gemma-4-26B_q4_0-it.gguf` plus its sibling +`gemma-4-26B-it-mmproj.gguf` projector. It is isolated from the protected Qwen +service: the candidate binds only `127.0.0.1:1923`, and the runner restarts +Qwen on `127.0.0.1:1919` on every exit path. + +## Build and runtime contract + +- Host: GMKtek EVO-X2, Radeon 8060S (`gfx1151`) unified-memory GPU. +- Backend: native ROCm/HIP and Triton. No CUDA compatibility path was used. +- Text GGUF: `gemma-4-26B_q4_0-it.gguf`. +- Vision projector: sibling `gemma-4-26B-it-mmproj.gguf`. +- Opt-in: `FREETOKEN_LOAD_VISION=1`. +- Vision geometry recovered from the projector and Gemma 4 release contract: + 27 layers, hidden width 1152, 16 heads, MLP width 4304, 16-pixel patches, + 10,240 position entries, 3 by 3 pooling, and at most 280 soft tokens per + image. +- Image API: OpenAI-compatible `messages[].content[]` with `type: image_url`. + The initial local-safe implementation accepts `data:image/...;base64,...` + values. It intentionally rejects remote URLs, preventing the serving process + from becoming an arbitrary LAN or Internet fetch client. + +## Evidence + +Latest artifact directory on GMKtek EVO-X2: + +`/home/operator/freetoken-amd/artifacts/gemma4-gguf-vision-20260830T045559Z` + +The runner completed both controls before it shut down the candidate and +started Qwen recovery. + +| Control | Result | Prompt tokens | Completion tokens | Observed latency or rate | +| --- | --- | ---: | ---: | --- | +| Text arithmetic | `323` | 30 | 4 | TTFT 2471.37 ms, 45.52 decode tok/s across two decode steps | +| Solid red PNG data URL | `red` | 284 | 2 | 1.89 s end-to-end request time | +| Solid green PNG data URL | `green` | 284 | 2 | 1.08 s end-to-end request time | +| Red-left, blue-right PNG | `red` for the left half | 282 | 2 | 1.06 s end-to-end request time | + +The image prompts had 282 to 284 tokens because the processor produced 256 +image soft tokens, plus the rendered text/template tokens. All three controls +returned their expected one-word answer. The spatial split-color control shows +that the path preserves image position rather than merely detecting a dominant +global color. Together they verify decoding, resizing, patchification, shaped +inter-process tensor transport, ROCm vision-tower execution, projector +execution, image-token replacement, and OpenAI response formatting. + +## Reproduction + +From the isolated checkout on GMKtek EVO-X2, first ensure the protected server health +is exactly `status: ok`, then run: + +```bash +bash scripts/gmk-evo-x2/run_gemma4_gguf_text_control.sh \ + /home/operator/freetoken-amd/validation-qwen-gguf-d1dd473 vision +``` + +The control runner saves `quality.json` for the text control and +`image-quality.json` for the OpenAI image control before its cleanup trap +restarts Qwen. The image verifier is also independently callable against an +already-running isolated candidate: + +```bash +PYTHONPATH=python /home/operator/freetoken-amd/.venv/bin/python \ + scripts/gmk-evo-x2/verify_gemma4_gguf_image.py \ + --base-url http://127.0.0.1:1923 \ + --model gemma4-26b-q4-amd \ + --artifact /tmp/gemma4-image-quality.json +``` + +## Matched ROCm 10 llama.cpp control + +The matched llama.cpp runner used the same text GGUF, sibling projector, +ROCm 10 installation, loopback-only OpenAI API contract, and deterministic +image fixtures. Its artifact is: + +`/home/operator/freetoken-amd/artifacts/gemma4-llamacpp-vision-20260830T051736Z` + +| Control | FreeToken AMD ROCm/HIP | llama.cpp ROCm 10 | Result | +| --- | --- | --- | --- | +| Text arithmetic | `323`, 47.97 decode tok/s, 1687.71 ms TTFT | `323`, 30.99 decode tok/s, 204.23 ms TTFT | Both correct. FreeToken decoded this two-step short control 54.8% faster, while llama.cpp had lower first-token latency. | +| Solid red image | `red`, 284 prompt and 2 completion tokens, 2.27 s wall time | `red`, 82 prompt and 92 completion tokens, 56.10 generated tok/s, 1.96 s wall time | Both correct. llama.cpp emitted 91 reasoning tokens before its visible answer. | +| Solid green image | `green`, 284 prompt and 2 completion tokens, 1.08 s wall time | `green`, 82 prompt and 84 completion tokens, 56.00 generated tok/s, 1.81 s wall time | Both correct. llama.cpp emitted 83 reasoning tokens before its visible answer. | +| Red-left, blue-right image | `red`, 282 prompt and 2 completion tokens, 1.06 s wall time | `red`, 79 prompt and 121 completion tokens, 56.16 generated tok/s, 2.44 s wall time | Both correct. llama.cpp preserved spatial information but emitted 120 reasoning tokens first. | + +This is a real OpenAI-compatible quality comparison, not an equivalence claim +for visual TPS. The runtimes tokenize image inputs differently, and llama.cpp +deliberately exposes a long `reasoning_content` trace on this Gemma template. +That makes its reported 56 tok/s an internally useful decode measurement but +not directly comparable to FreeToken's two-token user-visible response. On the +user-visible contract FreeToken completed the green and split-image controls +faster; on the first cold red request llama.cpp was faster. + +The text result is directly comparable because both runners used the same +caller-rendered prompt and returned the same four completion tokens. It shows +that the current native FreeToken ROCm/HIP path exceeds the matched llama.cpp +decode rate for that bounded control, but it does not establish a general +long-output advantage. + +## Long visual response boundary + +The deterministic split-color fixture was extended to require a 45 to 65 word +visible description containing the colors and their left-to-right arrangement. +FreeToken passed this quality gate with 51 visible words, 63 completion tokens, +1,093.83 ms TTFT, and 53.87 completion tokens per second over a 1.169 s +stream window. Its artifact is +`/home/operator/freetoken-amd/artifacts/gemma4-gguf-vision-20260830T055500Z`. + +The matched ROCm 10 llama.cpp model recognized the same image correctly but +placed every generated token in `reasoning_content`, leaving visible `content` +empty. This remained true at both 512 and 1,024 completion-token caps. The +1,024-token diagnostic reached 55.91 generated tokens per second but failed +the visible-output quality gate, so it is not comparable to FreeToken's 53.87 +visible-output TPS. This is a response-format/API-contract limitation of this +llama.cpp Gemma invocation, not evidence that it failed visual understanding. + +## Recovery-contract result + +The final isolated FreeToken vision run is +`/home/operator/freetoken-amd/artifacts/gemma4-gguf-vision-20260830T053317Z`. +It passed all three image controls (`red`, `green`, and spatial `red`), then +shut down the candidate and restored the protected Qwen server. Qwen reported +the authoritative `status: ok` after about eight minutes and twenty seconds; +the control runner then exited cleanly. This verifies that the runner now +handles the real serial-NVFP4 recovery envelope without a false success while +Qwen is still loading or a false failure from re-running its launcher. + +## Boundary and next measurement + +The image controls are functionality and short-request measurements, not a +long-output visual throughput benchmark. The next performance phase should use +a fixed visual-description task, a quality rubric that scores both visible and +reasoning channels separately, warmup exclusion, multiple repetitions, and +streaming telemetry. That will measure prompt TPS, first-token latency, and +decode TPS without rewarding a runtime merely for emitting more hidden +reasoning tokens. diff --git a/docs/gmktec-evo-x2-hip-gather-prototype.md b/docs/gmktec-evo-x2-hip-gather-prototype.md new file mode 100644 index 0000000000..b295e659d3 --- /dev/null +++ b/docs/gmktec-evo-x2-hip-gather-prototype.md @@ -0,0 +1,48 @@ +# GMKtek EVO-X2 compiled HIP gather prototype + +This prototype compiled a small device-side gather kernel with `hipcc` and ran +it on the Radeon 8060S. It gathers 64 randomly spaced 64 KiB blocks from a +64 MiB device-resident source buffer into a contiguous output buffer. It does +not access model files or change the protected service. + +## Build and runtime + +- Compiler: ROCm `hipcc` 7.15.26333 from ROCm 10.0. +- Target: `gfx1151`. +- Runtime: PyTorch environment reported HIP 7.15.26333 for the surrounding + host, while the kernel was launched through the HIP runtime directly. +- Kernel launch: 64 blocks, 256 threads per block. +- Warmup: one synchronized launch. +- Measurement: 100 launches timed with HIP events. + +## Result + +| Block size | Blocks per launch | Bytes per launch | Effective device gather rate | +|---:|---:|---:|---:| +| 64 KiB | 64 | 4 MiB | 281.916 GB/s | + +The kernel completed 400 MiB of gathered output in approximately 1.488 ms. +This is a device-to-device gather result, not a host-transfer or model-TPS +result. + +## Interpretation + +The result shows that compiled device-side indexing and gather can be much +faster than the host-orchestrated miss path. It does not by itself solve model +offload because the source expert blocks would still need to arrive in device +memory. It does, however, identify a credible implementation direction: +transfer and expert selection should be represented as persistent device-side +work, with host intervention reduced to batched descriptor submission. + +The 281.9 GB/s figure must not be compared directly with the earlier 79.8 +GB/s contiguous host-to-device measurement. They measure different links and +different operations. The useful comparison is against the 0.167 to 16.876 +GB/s scattered host-transfer results, which are dominated by launch and +synchronization overhead. + +## Next gate + +The next prototype should use mapped or pinned host memory and a HIP kernel +that consumes a descriptor list, then compare one descriptor submission for 64 +blocks against the serialized Python path. That will measure how much host +intervention can be removed while retaining the real host-to-device boundary. diff --git a/docs/gmktec-evo-x2-in-scope-model-inventory-20260904.md b/docs/gmktec-evo-x2-in-scope-model-inventory-20260904.md new file mode 100644 index 0000000000..33e65ff17f --- /dev/null +++ b/docs/gmktec-evo-x2-in-scope-model-inventory-20260904.md @@ -0,0 +1,108 @@ +# GMKtek EVO-X2 in-scope model inventory + +This inventory separates the model that is active now from models found in +archived model-routing qualification configurations. It is based on a +read-only process and filesystem inspection on September 4, 2026. An archived +configuration is evidence that a model was previously considered or tested; +it is not evidence that the model is currently loaded or production-routed. + +## Active native FreeToken service + +| Model identity | Runtime state | Existing evidence | +| --- | --- | --- | +| `qwen3.6-35b-a3b-nvfp4-amd` using `/home/operator/freetoken-amd/models/Qwen3.6-35B-A3B-NVFP4` | Active native ROCm/HIP service | Qwen quality suite, Q5 four-row qualification, ROCm 10 comparison, W1 to W4 bounded controls, recovery, and 1,440-session endurance | + +The active command line uses the native Triton attention path, the offload MoE +backend, automatic expert-cache sizing, serial expert loading, and an 8,192 +token context override. No llama-swap process was found active during the +inventory probe. + +## Payload admission result + +The read-only FreeToken model-directory probe found the Qwen3.6 35B-A3B +safetensors and NVFP4 payloads plus the Gemma 4 Q4 GGUF and vision projector. +The archived model store also contains a Qwen3.8 27B Q4_K_M GGUF, but it is not +under the FreeToken model directory. GLM-4.7, KAT-Coder, Laguna, and Ornith +payloads are likewise outside that directory. No model was copied or loaded +during this inventory step. + +The Qwen3.8 27B payload was checked with FreeToken's metadata-only GGUF +admission path. Its `general.architecture` is `qwen35`, and the current parser +returns `ValueError: GGUF architecture 'qwen35' is not supported (known: +['gemma4'])`. The current GGUF registry therefore cannot qualify this archived +Qwen3.8 artifact without a deliberate dense `qwen35` loader and model-path +implementation. A filename match is not sufficient evidence of support. + +The metadata comparison also rules out a simple alias to the supported Qwen3.5 +MoE GGUF path. Qwen3.8 reports 64 layers, hidden size 5,120, feed-forward size +17,408, and no expert-count or expert-feed-forward metadata. The qualified +Qwen3.6 MoE GGUF reports 40 layers, hidden size 2,048, 256 experts, and eight +active experts. Qwen3.8 therefore requires a dense hybrid-attention loader and +cannot safely reuse the current routed-expert loader by changing only the +registry string. + +The first implementation slice now admits `qwen35` in the GGUF registry, +maps its dense Q4_K MLP and embedding tensors, decodes the Q8_0 recurrence +matrices, and selects the Q6_K full-attention output projection. ROCm-side +metadata and real-file tensor-walk checks passed: the Qwen3.8 payload parsed as +64 layers, hidden size 5,120, intermediate size 17,408, zero experts, and 659 +unique runtime tensors were emitted without an unmapped-field error. This is a +loader and tensor-admission milestone, not yet proof that the complete model +can serve or that its outputs match llama.cpp. + +The ROCm-side construction probe also passed for representative linear-attention +and full-attention layers, including the final layer. Each constructed the +existing `Qwen3_5DenseMLP` branch with the expected dense configuration. The +probe allocated individual layers only; full-model loading and serving remain +separate gates. + +The source tree does contain model code for Qwen3.8-Flash-Next (`qwen4_exp`) +and GLM-4.7 parser support. Source support alone does not prove that the +archived routed checkpoints are compatible with the current AMD path. + +## Archived text-model routing entries + +These identifiers were found in archived model-routing configuration files and +remain candidates for a deliberate FreeToken qualification decision: + +| Model identifier | Modality | FreeToken qualification status | +| --- | --- | --- | +| `remote-qwen38-27b` | Text and image input, text output | Not yet qualified through the current standardized FreeToken matrix | +| `remote-qwen36-27b-control` | Text and image input, text output | Qwen family control exists, but this specific routed artifact needs an explicit matrix record | +| `remote-glm47-flash` | Text | Not yet qualified through the current standardized FreeToken matrix | +| `KAT-Coder-V2.5-Dev-Q8_0` | Text and tools | Not yet qualified through the current standardized FreeToken matrix | +| `remote-laguna-xs21` | Text and tools | Not yet qualified through the current standardized FreeToken matrix | +| `remote-ornith15-9b` | Text and tools | Not yet qualified through the current standardized FreeToken matrix | +| `remote-ornith15-35b-a3b` | Text and tools | Not yet qualified through the current standardized FreeToken matrix | +| `remote-glm47-ggml-q4k` | Text and tools | Archived llama-swap artifact-comparison entry; no FreeToken AMD matrix record | +| `remote-glm47-unsloth-q4km` | Text and tools | Archived llama-swap artifact-comparison entry; no FreeToken AMD matrix record | +| `remote-glm47-bartowski-q4km` | Text and tools | Archived llama-swap artifact-comparison entry; no FreeToken AMD matrix record | + +The existing Qwen3.6 35B-A3B NVFP4 result is not a substitute for these rows. +Each row requires its own exact checkpoint, tokenizer, quantization, prompt +contract, quality result, TPS measurements, and recovery evidence. + +## Archived non-text or multimodal entries + +The following entries were also found, but they are not ordinary text MoE +serving targets and therefore require a separate backend-admission decision: + +| Model identifier | Function | Current decision | +| --- | --- | --- | +| `remote-whisper-large-v3-turbo` | Audio transcription | No native FreeToken text-generation qualification claim | +| `remote-qwen3-tts-0.6b-base` | Text-to-speech | No native FreeToken text-generation qualification claim | +| `remote-qwen-image-2512-gguf` | Image generation or editing | No native FreeToken text-generation qualification claim | +| `remote-flux2-klein-4b` | Image generation or editing | No native FreeToken text-generation qualification claim | + +These models must not be counted as missing FreeToken tests until their +backend, input and output contract, and AMD implementation scope are defined. + +## Required qualification order + +1. Reconfirm the exact current serving inventory before changing any service. +2. Qualify the text MoE rows first, beginning with Qwen3.8 and GLM-4.7-Flash. +3. Add KAT-Coder, Laguna, and Ornith only after model-format support and + deterministic quality fixtures are confirmed. +4. Treat image, audio, and speech entries as separate backend projects. +5. Run every admitted model through the cross-model matrix in + `gmktec-evo-x2-cross-model-matrix-20260904.md`. diff --git a/docs/gmktec-evo-x2-mapped-host-gather.md b/docs/gmktec-evo-x2-mapped-host-gather.md new file mode 100644 index 0000000000..8ebb376a5d --- /dev/null +++ b/docs/gmktec-evo-x2-mapped-host-gather.md @@ -0,0 +1,47 @@ +# GMKtek EVO-X2 mapped-host descriptor gather + +This prototype compiled a HIP kernel that consumes one device-side descriptor +list and gathers expert-like blocks directly from mapped pinned host memory. It +is the first test of a device-side miss path with one kernel submission rather +than one host synchronization per block. It does not load a model or alter the +protected service. + +## Method + +- GPU: AMD Radeon 8060S, gfx1151. +- Compiler: ROCm 10 `hipcc`, target `gfx1151`. +- Mapped host source: 64 MiB allocated with `hipHostMallocMapped`. +- Descriptor list: 64 random 64 KiB-aligned block offsets. +- Output: contiguous 4 MiB device buffer. +- Kernel: one HIP launch with 64 blocks and 256 threads per block. +- Warmup: one synchronized launch. +- Measurement: 50 launches timed with HIP events. + +## Result + +| Block size | Descriptors per launch | Bytes per launch | Effective mapped-host gather rate | +|---:|---:|---:|---:| +| 64 KiB | 64 | 4 MiB | 112.908 GB/s | + +The measured 50 launches completed in approximately 1.857 ms. + +## Interpretation + +The result is substantially better than the earlier serialized host-transfer +path, which ranged from 0.167 to 16.876 GB/s depending on block size, and it +approaches the 79.79 GB/s contiguous host-to-device copy ceiling measured in a +separate test. It demonstrates that reducing host intervention to one +descriptor-driven device operation is a credible optimization direction on +Strix Halo. + +This is not yet a model result. Mapped host reads use the unified-memory fabric +directly and do not prove that a 148.66 GiB checkpoint can remain resident or +that real expert tensors will have the same locality. The kernel also omits +format conversion, cache eviction, routing, KV state, and model computation. + +## Next gate + +Integrate a descriptor-list gather into a small synthetic MoE layer with the +actual expert tensor shapes and FP4 or FP8 conversion path. Measure quality, +per-token latency, and p95/p99 miss behavior before considering any full +DeepSeek checkpoint download. diff --git a/docs/gmktec-evo-x2-nvfp4-marlin-parity-test.md b/docs/gmktec-evo-x2-nvfp4-marlin-parity-test.md new file mode 100644 index 0000000000..b317dfcb49 --- /dev/null +++ b/docs/gmktec-evo-x2-nvfp4-marlin-parity-test.md @@ -0,0 +1,42 @@ +# GMKtek EVO-X2 NVFP4 Marlin parity test + +## Purpose + +This focused test verifies that the production NVFP4 Marlin-style decode GEMV +produces the same result as FreeToken's original LUT-gather decode kernel on +the HIP runtime. It is a numerical correctness gate for the optimization +path, not an end-to-end throughput claim. + +## Command and environment + +The test ran on the GMKtek EVO-X2 using the source checkout's existing Python +environment and the native HIP runtime: + +```text +python -m pytest tests/moe/test_nvfp4_backends.py -k "marlin" --maxfail=1 -q +``` + +The selected tests include Marlin output comparison against the baseline +kernel and the cache-stomp sequence that reloads experts after a full-layer +prefill. The test module's CUDA marker is satisfied by the available HIP +device through PyTorch's CUDA-compatible device API. + +## Result + +```text +sss..s [100%] +2 passed, 4 skipped, 6 deselected in 4.30s +``` + +The two executed tests passed. The four skips are unrelated backend variants +or optional dependencies selected by the module and do not invalidate the +Marlin parity result. + +## Interpretation + +The production Marlin decode path is numerically equivalent to the retained +baseline within the test's specified tolerance and survives the cache +reload-after-prefill scenario. This qualifies it for further real-service +comparison work. It does not prove that the Marlin path is faster than the +current complete serving configuration, so no promotion or TPS claim follows +from this test alone. diff --git a/docs/gmktec-evo-x2-nvfp4-marlin-stages2-rejection.md b/docs/gmktec-evo-x2-nvfp4-marlin-stages2-rejection.md new file mode 100644 index 0000000000..e1992df9fc --- /dev/null +++ b/docs/gmktec-evo-x2-nvfp4-marlin-stages2-rejection.md @@ -0,0 +1,49 @@ +# GMKtek EVO-X2 NVFP4 Marlin `num_stages=2` candidate rejection + +## Candidate + +This isolated candidate kept the Marlin tile (`BLOCK_N=16`), warp count (4), +and reduction expression unchanged. It changed only Triton's launch staging +parameter by adding `num_stages=2` to the production Marlin kernel launch. + +## Performance observation + +Five fixed API samples completed without protocol errors: + +```text +mean decode TPS 30.037302 +median decode TPS 30.036226 +stdev 0.006263 +``` + +The values are retained as diagnostic observations only because the candidate +failed the deterministic quality gate. + +## Quality result + +The exact canary, arithmetic, and JSON checks passed. The deterministic AIME +hash failed: + +```text +expected output SHA-1: cd580f4978fb +observed output SHA-1: 1cae5bae914f +``` + +Raw artifacts are preserved under: + +```text +/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T184547Z/ +``` + +## Recovery + +The candidate stopped normally. The wrapper restored the original kernel file, +and the protected Qwen service returned `status: ok` with +`maintenance: serving`. + +## Decision + +**Rejected.** Even a launch-staging-only change altered deterministic output. +The current four-warp, default-staging Marlin configuration remains the +qualified baseline. Future optimization must avoid changing Triton execution +schedule unless its numerical consequences are fully controlled. diff --git a/docs/gmktec-evo-x2-nvfp4-marlin-tile8-rejection.md b/docs/gmktec-evo-x2-nvfp4-marlin-tile8-rejection.md new file mode 100644 index 0000000000..e968180287 --- /dev/null +++ b/docs/gmktec-evo-x2-nvfp4-marlin-tile8-rejection.md @@ -0,0 +1,63 @@ +# GMKtek EVO-X2 NVFP4 Marlin tile-8 candidate rejection + +## Candidate + +The isolated candidate changed the production Marlin decode output-row tile +from `BLOCK_N=16` to `BLOCK_N=8`. It used the same native ROCm/HIP runtime, +Qwen3.6 NVFP4 checkpoint, offload policy, cache sizing, and API benchmark as +the qualified control. The protected service was stopped only after its +identity and health were verified, and the launcher restored it in its exit +path. + +## Performance observation + +Five fixed API samples completed without protocol errors: + +```text +mean decode TPS 29.835737 +median decode TPS 29.828223 +stdev 0.013950 +``` + +These numbers are retained as an observation only. They are not an accepted +performance result because the candidate failed the required quality gate. + +## Quality result + +The exact canary, arithmetic, and JSON checks passed. The deterministic AIME +check failed: + +```text +expected output SHA-1: cd580f4978fb +observed output SHA-1: 1cae5bae914f +observed decode TPS: 30.398296 +observed TTFT: 396.048 ms +``` + +The candidate stopped after the failure, preserving the raw benchmark and +quality artifacts under: + +```text +/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T180912Z/ +``` + +## Recovery verification + +The launcher restored the original source file, leaving: + +```text +_DECODE_MARLIN_BLOCK_N = 16 +``` + +The protected Qwen service then returned: + +```json +{"status":"ok","maintenance":"serving"} +``` + +## Decision + +**Rejected.** The tile-8 candidate is not eligible for API promotion. The +throughput observation is useful for diagnosis, but deterministic quality is a +hard gate and the changed output hash demonstrates that this tile configuration +cannot be used as a like-for-like optimization. diff --git a/docs/gmktec-evo-x2-nvfp4-marlin-warps8-rejection.md b/docs/gmktec-evo-x2-nvfp4-marlin-warps8-rejection.md new file mode 100644 index 0000000000..7a19b3eae1 --- /dev/null +++ b/docs/gmktec-evo-x2-nvfp4-marlin-warps8-rejection.md @@ -0,0 +1,56 @@ +# GMKtek EVO-X2 NVFP4 Marlin warp-count candidate rejection + +## Candidate + +This isolated candidate retained the production Marlin output tile +(`BLOCK_N=16`) and changed only the decode launch warp count from four to eight. +It used the same Qwen3.6 NVFP4 checkpoint, native HIP runtime, offload cache, +API workload, and deterministic quality gates as the qualified control. + +## Performance observation + +Five fixed API samples completed without protocol errors: + +```text +mean decode TPS 30.244579 +median decode TPS 30.238741 +stdev 0.014834 +``` + +The AIME request itself reported 30.764005 decode TPS and 389.508 ms TTFT. +These numbers are retained as diagnostic observations only. + +## Quality result + +The exact canary, arithmetic, and JSON checks passed. The deterministic AIME +hash failed: + +```text +expected output SHA-1: cd580f4978fb +observed output SHA-1: 1cae5bae914f +``` + +The raw artifacts remain under: + +```text +/home/operator/freetoken-amd/artifacts/nvfp4-marlin-api-candidate-20260904T182532Z/ +``` + +## Recovery + +The candidate process stopped normally. The recovery launcher restored the +production setting: + +```text +_DECODE_MARLIN_WARPS = 4 +``` + +The protected Qwen service returned `status: ok` and +`maintenance: serving` after recovery. + +## Decision + +**Rejected.** Changing only the warp count changes the deterministic model +output. The measured speed increase is not admissible as a quality-preserving +optimization. Future work must preserve the current launch geometry and target +memory scheduling or orchestration overhead instead. diff --git a/docs/gmktec-evo-x2-nvfp4-shape-prototype.md b/docs/gmktec-evo-x2-nvfp4-shape-prototype.md new file mode 100644 index 0000000000..e3517eadb3 --- /dev/null +++ b/docs/gmktec-evo-x2-nvfp4-shape-prototype.md @@ -0,0 +1,68 @@ +# GMKtek EVO-X2 production-shape NVFP4 prototype + +## Purpose + +This experiment moves the previous fused-expert investigation to the +production NVFP4 tensor layout used by the FreeToken Qwen path. It uses random +device-resident tensors, so no model checkpoint or production service is +involved. The goal is to measure kernel launch behavior at representative +dimensions before attempting a production implementation change. + +## Configuration + +| Field | Value | +| --- | --- | +| Backend | Native FreeToken Triton NVFP4 Marlin decode kernel through HIP | +| GPU target | AMD `gfx1151` | +| Experts in bank | 8 | +| Hidden size | 1,152 | +| Intermediate size | 512 | +| Routed experts per token | 8 | +| Gate/up packed shape | `[8, 1024, 576]` uint8 | +| Gate/up scale shape | `[8, 1024, 72]` uint8, one scale per 16 values | +| Down packed shape | `[8, 1152, 256]` uint8 | +| Down scale shape | `[8, 1152, 32]` uint8, one scale per 16 values | +| Activation | BF16 input, SiLU gated path | +| Timed samples | 10, after one warmup call | + +The production function performs both gate/up and down expert GEMV operations, +the SiLU activation, routed-weight handling, and final expert reduction. The +benchmark uses the production Marlin-style entry point rather than a separate +synthetic CUDA or HIP kernel. + +## Result + +The isolated run completed successfully and returned finite output: + +```text +experts=8 hidden=1152 intermediate=512 top_k=8 +samples_ms=[0.305189, 0.184113, 0.176063, 0.165334, 0.107996, + 0.104776, 0.097746, 0.099526, 0.099096, 0.094497] +mean_ms=0.143434 output_finite=true +``` + +The first timed sample includes residual one-time runtime work. Excluding that +sample, the steady-state mean was **0.125461 ms** for the complete two-GEMV +fused expert operation. The packed gate/up plus down input footprint was +7,077,888 bytes per call, equivalent to approximately **56.4 GB/s** of packed +input traffic at that steady-state mean. + +## Interpretation + +This is the first format-faithful kernel-path result in the investigation. It +confirms that the production NVFP4 Marlin decode entry point can execute the +representative routed shape on HIP with finite output and sub-millisecond +steady-state latency. + +It is not model TPS. The weights are random, the bank has only eight experts, +and this test excludes router execution, attention, KV management, token +scheduling, API overhead, and host-side expert-cache misses. It therefore +cannot be used to claim quality or end-to-end speed improvement. + +## Next action + +Instrument the same entry point with the actual cache bank and deterministic +Qwen layer inputs, then compare its output hash and latency against the current +production candidate. A replacement kernel must preserve exact quality and +survive the complete API, concurrency, tail-latency, and recovery gates before +it can be promoted. diff --git a/docs/gmktec-evo-x2-overlap-prototype.md b/docs/gmktec-evo-x2-overlap-prototype.md new file mode 100644 index 0000000000..6387c6fb1a --- /dev/null +++ b/docs/gmktec-evo-x2-overlap-prototype.md @@ -0,0 +1,40 @@ +# GMKtek EVO-X2 grouped-transfer overlap prototype + +This prototype tested whether a naive two-stream pipeline could hide grouped +expert staging and transfer behind synthetic GPU work. It is an isolated +systems experiment. It does not load a large model or alter the protected +service. + +## Method + +- GPU: AMD Radeon 8060S, gfx1151. +- PyTorch: `2.13.0+rocm10.0.0`. +- HIP: `7.15.26333`. +- Four groups per round, each containing sixteen random 64 KiB blocks. +- Pinned host staging buffers and a separate transfer stream. +- Separate compute stream with an event dependency after each copy. +- Comparison against a synchronized serial implementation. + +## Results + +| Synthetic compute per group | Serial mean | Overlap mean | Overlap speedup | +|---:|---:|---:|---:| +| 0 matrix multiplications | 0.331 ms | 2.723 ms | 0.122x | +| 1 matrix multiplication | 1.793 ms | 10.039 ms | 0.179x | + +The naive overlap pipeline was slower in both cases. With no compute it was +approximately 8.2 times slower, and with one matrix multiplication it was +approximately 5.6 times slower. + +## Interpretation and rejection reason + +This is not evidence that overlap is impossible in the production runtime. The +prototype intentionally used many small stream and event operations and a +single reusable compute tensor, so it exposes orchestration overhead that a +fused production scheduler might avoid. It does show that simply adding a +transfer stream and per-group events is not a valid optimization. + +The candidate is rejected for promotion because it regressed end-to-end wall +time in both measured cases. Future overlap work must use persistent streams, +event pools, larger fused batches, and scheduler-level pipelining, then pass +the same quality and tail-latency gates as the current validated path. diff --git a/docs/gmktec-evo-x2-paper-model-capacity-gate.md b/docs/gmktec-evo-x2-paper-model-capacity-gate.md new file mode 100644 index 0000000000..e0c2032287 --- /dev/null +++ b/docs/gmktec-evo-x2-paper-model-capacity-gate.md @@ -0,0 +1,218 @@ +# GMKtek EVO-X2 paper-model capacity gate + +This is a read-only capacity gate for deciding whether to attempt the +FreeToken paper's large-model demonstrations on the GMKtek EVO-X2 Strix Halo. +It records the live host state and does not download, load, or alter a model. + +## Live host observation + +The observation was collected on 2026-09-04 from the configured GMKtek EVO-X2 +using `free -h`, `swapon --show --bytes`, `rocm-smi`, and a bounded model-file +inventory. + +| Resource | Observed value | +|---|---:| +| GPU | AMD Radeon 8060S Graphics, gfx1151 | +| ROCm-reported VRAM | 2 GiB total, approximately 352 MiB used at observation time | +| System memory | 59 GiB total, 18 GiB available | +| Swap | 127 GiB total, approximately 2.1 GiB used | +| Root filesystem | 1.9 TiB total, 769 GiB available | +| GPU temperature | 31 C | +| GPU power | 12 W | +| GPU load | 0 percent | + +The 59 GiB system-memory figure is not a promise that all 59 GiB is available +to model weights. The live `MemAvailable` value was approximately 18 GiB, and +the ROCm device reports a separate 2 GiB VRAM aperture. Unified-memory +allocation, runtime buffers, KV cache, and the protected service must be +accounted for before any model load. + +## Installed model evidence + +The bounded inventory found the following relevant payloads: + +- Qwen3.6-35B-A3B Q4 GGUF: approximately 22.1 GB. +- Gemma 4 26B Q4 model GGUF: approximately 14.4 GB. +- Gemma 4 projector GGUF: approximately 1.2 GB. +- No DeepSeek-V4-Flash checkpoint. +- No GLM-5.2 checkpoint. + +The current official DeepSeek repository metadata at commit +`7872f01b1d1fe23eabc4c98b48bffcef5a386062` lists 48 safetensors shards. +Read-only HTTP `HEAD` requests to every shard reported a combined +`Content-Length` of 166,886,535,336 bytes, or approximately 155.43 GiB for the +model payload alone. This excludes the tokenizer, runtime allocations, +expert-cache policy, KV cache, allocator slack, and any duplicate conversion +buffers. The measurement was refreshed on 2026-09-05. + +The official `config.json` reports 43 hidden layers, 256 routed experts, one +shared expert, and six routed experts active per token. The hidden size is +4,096. This confirms that the 13B activated-parameter figure does not reduce +the storage requirement to 13B parameters: the complete routed-expert pool is +still part of the 148.66 GiB checkpoint and must be streamed, cached, or +otherwise retained by the serving system. + +## Paper-model decision + +The FreeToken paper identifies DeepSeek-V4-Flash as a 284B-parameter model +with 13B activated parameters and mixed FP4 plus FP8 deployment. The current +official `DeepSeek-V4-Flash-0731` model page is a later release that reports +304B parameters and BF16, I64, F32, F8_E4M3, and I8 tensor types. Its raw +configuration still confirms FP4 expert storage, 256 routed experts, six +experts active per token, and 43 layers, but the release identity is not +automatically the same as the paper's 284B demonstration. The paper's +prefill discussion describes roughly 140 GB of routed expert weights. The +paper describes GLM-5.2 as a 753B-parameter model with a 433 GB checkpoint. +Neither payload is installed on this host, and the live available-memory +observation is far below either stated payload scale. + +The paper names `deepseek-ai/DeepSeek-V4-Flash-0731` as its official +checkpoint, and the repository history exposes a release commit +`9e165c30e2704aec5d9d593cce3eebd58bbef1cb` that predates the current model-card +metadata update. That release commit still contains the same 48-shard payload +size measured above. We must pin that revision in any reproduction record and +report the paper's 284B label separately from the current model-card 304B label; +the two labels are not enough by themselves to prove an exact parameter-count +match. + +As an additional identity check, `config.json` is byte-for-byte identical at +the release commit and the current model-card commit. Its SHA-256 is +`6c8f3d2d3b48707541b88f32f22ef3f0f8a6b57d8523281e2b8d3cdb0ae9a023`. + +## Reproducible metadata-only gate + +The repository includes +[`deepseek_capacity_gate.py`](../scripts/gmk-evo-x2/deepseek_capacity_gate.py). +Using the pinned payload size, 18 GiB of observed `MemAvailable`, a 2 GiB +ROCm-reported aperture, and explicit reserves of 8 GiB for the OS, 2 GiB for +runtime state, 2 GiB for KV cache, and 2 GiB for recovery, it produced: + +```text +decision: REJECT_FULL_LOAD +authoritative model budget: 4.000 GiB +payload: 155.425 GiB +authoritative deficit: 151.425 GiB +optimistic deficit even counting the VRAM aperture: 135.425 GiB +``` + +The machine-readable result is +[`gmktec-evo-x2-deepseek-capacity-gate-result-20260905.json`](gmktec-evo-x2-deepseek-capacity-gate-result-20260905.json). +This is a metadata-only rejection. No model files were downloaded, and no +service or model process was changed. + +## Expert-slice metadata + +The pinned safetensors index and a bounded header range from shard 2 provide +enough metadata to size a production-shaped slice without downloading tensor +payloads. Each core routed expert has three I8 matrices and three scale +arrays totaling 13,369,344 bytes, or 12.75 MiB. The 43-layer, 256-expert core +pool is approximately 137.0625 GiB. Six active experts per layer across all +43 layers would touch approximately 3.22265625 GiB before attention, shared +experts, KV cache, runtime buffers, or allocator overhead. + +This makes a bounded transfer and packing experiment worthwhile, but it does +not make full serving feasible. The exact derived values are preserved in +[`gmktec-evo-x2-deepseek-expert-slice-metadata-20260905.json`](gmktec-evo-x2-deepseek-expert-slice-metadata-20260905.json). + +The executable next-stage harness is +[`deepseek_expert_slice_benchmark.py`](../scripts/gmk-evo-x2/deepseek_expert_slice_benchmark.py). +Its default selection is one layer and six experts, approximately 76.5 MiB of +core routed expert weight bytes before scales and other model state. It +requires a locally staged safetensors directory, PyTorch with ROCm, and the +`safetensors` package. It touches no API port and records +`protected_service_touched: false` in its output. It must be run only as an +isolated candidate after the normal service is verified healthy. The +`--metadata-only` mode was validated locally against the six-expert fixture in +`tests/fixtures/deepseek_expert_index`; it selected all 36 expected tensors +without importing GPU libraries. + +## Real-shape ROCm slice result + +The isolated harness was run on the GMKtek EVO-X2 using the pinned shard and +the native ROCm environment. It transferred 80,216,064 bytes, or 76.5 MiB, +covering all six experts and all six tensors per expert for layer 0. Five +round trips were recorded. The first H2D sample was cold at 0.863 GiB/s, +consistent with initial mapping and page-fault overhead. The final three H2D +samples averaged 76.645 GiB/s, while all four post-cold samples averaged +73.987 GiB/s. D2H averaged 64.073 GiB/s across the four post-cold samples. +Using decimal units, the final-three H2D result is approximately 82.30 GB/s +and the post-cold D2H result is approximately 68.80 GB/s. That is consistent +with, but slightly more realistic than, the earlier contiguous synthetic bound +of 79.79 GB/s H2D and 70.24 GB/s D2H. + +The protected Qwen health endpoint remained healthy after the run, reporting +`status: ok` and `maintenance: serving`. ROCm reported 28 C, 13.041 W, and +zero GPU utilization at the post-run check. The raw result is preserved in +[`gmktec-evo-x2-deepseek-expert-slice-result-20260905.json`](gmktec-evo-x2-deepseek-expert-slice-result-20260905.json). +This is evidence for the AMD transfer path only, not a full-model serving or +quality result. + +A second isolated run expanded the same layer to 16 experts, or 204.0 MiB and +96 tensors. The final three H2D samples averaged 77.561 GiB/s, all four +post-cold H2D samples averaged 77.346 GiB/s, and post-cold D2H averaged 64.762 +GiB/s. The protected service again returned `status: ok` after the run. This +larger slice shows no material H2D collapse as the transfer batch grows, but +it remains a single-layer transfer test rather than a model-serving result. +Its raw output is preserved in +[`gmktec-evo-x2-deepseek-expert-slice-16-result-20260905.json`](gmktec-evo-x2-deepseek-expert-slice-16-result-20260905.json). + +Finally, a two-layer slice selected experts 0 through 5 from layers 0 and 1, +spanning both shard 2 and shard 3. It transferred 153.0 MiB across 72 +tensors. The final three H2D samples averaged 77.976 GiB/s, all four +post-cold H2D samples averaged 77.686 GiB/s, and post-cold D2H averaged +64.622 GiB/s. Cross-shard loading completed successfully, and the protected +service returned `status: ok` afterward. This strengthens the transfer-path +result across layer and shard boundaries, but it remains a transfer-only +experiment. Raw output is preserved in +[`gmktec-evo-x2-deepseek-expert-slice-2layer-result-20260905.json`](gmktec-evo-x2-deepseek-expert-slice-2layer-result-20260905.json). + +Four additional two-layer route groups were tested to check expert-ID +sensitivity. Post-cold H2D and D2H averages were: + +| Expert IDs | H2D GiB/s | D2H GiB/s | +|---|---:|---:| +| 0 to 5 | 77.758 | 64.738 | +| 16 to 21 | 78.435 | 64.720 | +| 32 to 37 | 78.577 | 64.974 | +| 64 to 69 | 78.538 | 64.923 | + +The narrow spread indicates no material transfer-rate dependence on these +expert IDs. Raw per-group outputs are preserved as +`expert-route-group-0.json`, `expert-route-group-16.json`, +`expert-route-group-32.json`, and `expert-route-group-64.json`. + +## Transfer-only route projection + +Using the measured 77.976 GiB/s H2D rate and the exact 13,369,344-byte expert +size, a token that misses all six routed experts in all 43 layers would move +3.2124 GiB of expert data. The transfer-only lower bound is therefore 41.20 ms +per token, or 24.27 tokens per second. At 75 percent, 50 percent, and 25 +percent miss rates, the transfer-only ceilings are 32.36, 48.55, and 97.09 +tokens per second respectively. + +This result is informative but deliberately not a serving claim. It excludes +matrix computation, routing, attention, KV state, synchronization, cache +lookup and eviction, allocator overhead, and D2H traffic. It shows that the +measured AMD H2D path is physically compatible with the paper's reported 22 to +25 tok/s range even under a pessimistic all-miss transfer assumption, but it +does not show that the complete model can fit or achieve that rate. The raw +projection is preserved in +[`gmktec-evo-x2-deepseek-route-transfer-projection-20260905.json`](gmktec-evo-x2-deepseek-route-transfer-projection-20260905.json). + +Therefore the large-model demonstrations are **not currently actionable** on +this host. A model download must not be treated as the next step. Before any +attempt, we need the exact checkpoint, quantization, required host-resident +weights, KV-cache budget, and an explicit policy for whether swap-backed +execution qualifies as interactive. A successful allocation alone would not +reproduce the paper's claim. + +## Next gate + +1. Obtain the exact DeepSeek-V4-Flash checkpoint metadata and file layout. +2. Compute weight, expert-cache, runtime, and KV-cache requirements from that + metadata before downloading anything. +3. If the calculated working set exceeds available unified memory, classify the + paper demonstration as capacity-incomplete rather than forcing a swap-heavy + run that cannot meet the paper's interactive criterion. +4. Keep Qwen and Gemma performance optimization independent from this capacity + gate. diff --git a/docs/gmktec-evo-x2-persistent-overlap-prototype.md b/docs/gmktec-evo-x2-persistent-overlap-prototype.md new file mode 100644 index 0000000000..db33ee6445 --- /dev/null +++ b/docs/gmktec-evo-x2-persistent-overlap-prototype.md @@ -0,0 +1,42 @@ +# GMKtek EVO-X2 persistent grouped-transfer overlap prototype + +This prototype tested a lower-overhead overlap design than the earlier +per-group-event experiment. It uses one persistent transfer stream, +double-buffered pinned host staging, and one completion event per reusable +buffer. It does not load a large model or alter the protected service. + +## Method + +- GPU: AMD Radeon 8060S, gfx1151. +- PyTorch: `2.13.0+rocm10.0.0`. +- HIP: `7.15.26333`. +- Four groups per round, each containing sixteen random 64 KiB blocks. +- Two reusable pinned host buffers and two device buffers. +- Eight measured rounds after three warmups per condition. +- Serial baseline synchronizes after each group. +- Persistent pipeline synchronizes only when a reusable buffer is needed and + once at the end of the round. + +## Results + +| Synthetic compute | Serial mean | Persistent overlap mean | Relative speed | +|---|---:|---:|---:| +| None | 0.317 ms | 1.931 ms | 0.164x | +| One device add per group | 0.450 ms | 1.424 ms | 0.316x | + +The persistent design remained slower than serial in both conditions. It was +approximately 6.1 times slower without compute and 3.2 times slower with the +synthetic device operation. + +## Decision + +This candidate is rejected for promotion. Persistent streams and buffer reuse +alone do not hide the Python-side staging and scheduling cost in this test. +The result does not rule out a native fused runtime path. It does rule out +continuing to add Python-level stream and event orchestration as the primary +optimization strategy. + +Future work should move batching into the runtime or a compiled HIP kernel, +where expert indexing, gather, transfer scheduling, and compute can be fused or +queued with substantially fewer host interventions. Any such implementation +must still pass deterministic quality, tail-latency, recovery, and API gates. diff --git a/docs/gmktec-evo-x2-q4-hardening-plan-2026-08-31.md b/docs/gmktec-evo-x2-q4-hardening-plan-2026-08-31.md new file mode 100644 index 0000000000..b3ec39d248 --- /dev/null +++ b/docs/gmktec-evo-x2-q4-hardening-plan-2026-08-31.md @@ -0,0 +1,57 @@ +# GMKtek EVO-X2 Q4 hardening execution plan + +## Objective + +Close the remaining reliability, performance, readiness, endurance, and +publication gaps in the native ROCm/HIP Qwen Q4 path without disrupting the +protected GMKtek EVO-X2 NVFP4 loopback service except during a recorded, reversible +time-share window. + +## Non-negotiable controls + +1. All candidate servers bind only to `127.0.0.1:1922`; the normal service + remains `127.0.0.1:1919` and is not added to llama-swap. +2. Before a time-share handoff, verify the port owner, model path, command, + and complete process group. Stop only that verified group. +3. Every candidate has a new dated artifact directory, fixed model file, + tokenizer, request suite, runtime versions, and raw log retention. +4. A candidate is accepted only if the API, deterministic quality, long + context, concurrency, process-scoped swap, and recovery checks pass. +5. Rejected candidates remain documented with their artifacts and are never + silently promoted to the normal service. + +## Work items and acceptance gates + +| Item | Execution | Acceptance gate | Rollback or rejection rule | +| --- | --- | --- | --- | +| 1. Correct report terminology | State the stable `0.25` and experimental `0.35` comparisons separately. | The report names 1.78 percent as the stable gap and 1.39 percent as an unstable historical result. | Do not publish a parity claim. | +| 2. Profile before optimizing | Use the wheel-compatible ROCm profiler only on an isolated Q4 workload, then rank kernels by measured end-to-end relevance. | Trace, source revision, command, and kernel aggregate are retained. | Reject profiler-only throughput claims. | +| 3. Repair lifecycle and SVM exposure | Launch recovery and Q4 candidates in dedicated sessions, verify the whole process group on stop, and test forced cancellation only in the candidate window. | No orphan listener or child remains after stop; `/health` reports loading, serving, or failure honestly. | Keep `0.25` as the recommended profile if `0.35` again triggers the SVM resident-memory fault. | +| 4. Make cold readiness explicit | Treat `/health` `status: ok` and `maintenance: serving` as readiness, not the presence of `/v1/models`. | Cold launch emits loading while unavailable and serving only after the engine is ready. | Never score a request that received a loading 503. | +| 5. Requalify performance and quality | Run same-file Q4 FreeToken and llama.cpp controls with fixed prompt, tokens, warmup, and quality suite. | All quality rows pass and FreeToken median TPS is at least the accepted baseline; a parity claim requires a new matched result. | Revert code and preserve evidence if quality, tail latency, or runner swap regresses. | +| 6. Extended endurance | Run a process-scoped 24-hour, 1,440-session three-turn Q4 battery after the Q4 server is qualified. | 1,440 of 1,440 sessions pass, every verified FreeToken process has zero `VmSwap`, and normal service recovers after cleanup. | Stop immediately on a wrong answer, runner swap, process death, or health failure. | +| 7. Sanitized publication package | Capture a redacted host manifest, checksums, source state, benchmark code, and selected public artifacts. | Manifest passes HIP and target checks, checksum verification, and scans free of private host names, addresses, and home paths. | Do not publish any artifact that fails the privacy scan. | + +## Current execution state + +- Item 1 is complete in commit `cc3a531`. +- Item 2 has a retained native ROCm baseline that identifies dense Q4 GEMV and + quantization as the measured GPU work to investigate. It is not reported as + serving throughput. +- Items 3 and 4 are implemented in commit `07b99da`: recovery now launches in + a dedicated session and exposes lifecycle state through `/health`. +- Item 7 is implemented and live-validated in commits `adc76be`, `5e8448f`, + and `e592130`. The validated manifest has a redacted host field, native HIP + metadata, verified checksums, and no detected private host name, address, or + home path. +- Items 5 and 6 begin only after the current hardened recovery launch reaches + `/health` serving state. The normal service is restored by the same + verified-session mechanism after every candidate window. + +## Decision rule + +The stable Q4 profile is already functionally qualified. The purpose of the +remaining work is to either produce a measured, quality-preserving improvement +or document why the current 1.78 percent llama.cpp gap and high-cache SVM +limit remain. A failed experiment is still a completed investigation when its +cause, raw evidence, and rollback are retained. diff --git a/docs/gmktec-evo-x2-q4-mmv-y4-promotion-record.md b/docs/gmktec-evo-x2-q4-mmv-y4-promotion-record.md new file mode 100644 index 0000000000..520b803f9e --- /dev/null +++ b/docs/gmktec-evo-x2-q4-mmv-y4-promotion-record.md @@ -0,0 +1,161 @@ +# Q4 MMV_Y=4 promotion record + +This record documents the reproducible FreeToken Q4 candidate measured on the +GMKtek EVO-X2. It is an evidence record, not a claim that the candidate has +already replaced the protected service configuration. + +## Candidate identity + +- Candidate source revision: `fb4e0232dbd7804b7d86c1ddd2dd366e2b0c05a7`. +- Model file: `Qwen3.6-35B-A3B-UD-Q4_K_M.gguf`. +- Model SHA-256: `ac0e2c1189e055faa36eff361580e79c5bd6f8e76bffb4ce547f167d53e31a61`. +- GPU: AMD Radeon 8060S Graphics, architecture `gfx1151`. +- PyTorch: `2.13.0+rocm10.0.0`. +- HIP runtime reported by PyTorch: `7.15.26333`. +- Python: `3.12.13`, Clang-backed environment. +- Reusable extension cache: `/home/operator/freetoken-amd/cache/torch_extensions-q8-api-y4`. + +## Runtime configuration + +The candidate was started on an isolated loopback port with the following +performance-affecting settings: + +```text +FREETOKEN_GGUF_MMV_Y=4 +FREETOKEN_GGUF_Q8_MMV_WARPS=1 +PYTORCH_ROCM_ARCH=gfx1151 +ROCM_HOME=/opt/rocm-10.0 +ROCM_PATH=/opt/rocm-10.0 +HIP_PATH=/opt/rocm-10.0 +--attention-backend triton +--moe-backend offload +--nvfp4-backend triton +--expert-load serial +--moe-cache-auto +--memory-ratio 0.25 +--max-seq-len-override 8192 +--kv-reserve-tokens 8192 +--cuda-graph-max-bs 0 +--disable-pynccl +--disable-moe-prefill-overlap +``` + +The Q8 warp count remains one because the current implementation deliberately +rejects multiwarp Q8 values. The accepted experiment changes `MMV_Y`, which is +the supported dense activation launch geometry. + +## Acceptance evidence + +The first five-sample API run is preserved at: + +`/home/operator/freetoken-amd/artifacts/qwen-q4-mmv-y4-api5-20260905T075002Z` + +Its mean decode rate was 48.03 TPS, with five successful samples and a +standard deviation of 0.054 TPS. The independent repeat is preserved at: + +`/home/operator/freetoken-amd/artifacts/qwen-q4-mmv-y4-repeat-20260905T082425Z` + +The repeat measured 47.86 TPS across five successful samples, with a standard +deviation of 0.092 TPS. The matched ROCm10 llama.cpp control measured 48.75 +TPS, so the repeat was approximately 1.8 percent slower in decode. + +The quality and state evidence is preserved at: + +`/home/operator/freetoken-amd/artifacts/qwen-q4-mmv-y4-quality-20260905T080132Z` + +The deterministic suite passed its exact, arithmetic, and JSON cases. The +three-turn state suite passed acknowledgment, recall, and transformation. + +The long-context and resource evidence is preserved at: + +`/home/operator/freetoken-amd/artifacts/qwen-q4-mmv-y4-longctx-20260905T081222Z` + +Five nonce-varied 6,056-token prompts passed exact marker retrieval. Available +memory changed from 19 GiB to 18 GiB. Swap use decreased from 2.1 GiB to 710 +MiB. GPU temperature changed from 35 C to 54 C, GPU use reached 91 percent, +and measured power reached 110 W. + +## Promotion decision + +The candidate is reproducible and passes the current functional, quality, +long-context, and resource gates. It should be treated as the leading Q4 +optimization candidate, but not silently installed as the protected default. +Before permanent promotion, rerun the documented commands from a clean shell, +verify the reusable extension cache contents, and attach the resulting build +log and final source diff to the upstream review record. + +## Clean-shell reproduction + +The fresh-cache reproduction was completed from a clean remote shell using the +same source revision, ROCm 10.0 paths, `gfx1151` target, `MMV_Y=4`, and Q8 +one-wave guard. The native build invoked `/opt/rocm-10.0/bin/hipcc` and emitted +the expected `-DGGML_CUDA_MMV_Y=4` and `--offload-arch=gfx1151` flags. + +- Artifact: `/home/operator/freetoken-amd/artifacts/qwen-q8-mmv-y4-clean-20260905T084057Z`. +- Real-weight Q8 screen: 25.983 microseconds mean over 300 repetitions. +- Device and software identity matched the candidate record. +- Build and benchmark output is preserved in `build-and-bench.log`. + +## Known limitations + +The client-visible prefill rate remains far below the llama.cpp control even +though decode throughput is nearly matched. The long-context test is a local +6,056-token retrieval control and is not a reproduction of the paper's agent +workload. The quality suite is deterministic and bounded; it does not replace +the paper's full tool-using evaluation. +## Current-branch requalification + +The opt-in Y4 flag was requalified from current AMD branch commit `ff76ede` in +an isolated checkout with the protected service stopped through its guarded +lifecycle. Startup reached the explicit API-ready state with 56 GiB free before +model loading and 23.07 GiB free after initialization. The current branch then +completed the fixed three-sample scheduler-shaped matrix: + +- decode mean: 45.4603 TPS +- decode median: 45.4418 TPS +- decode standard deviation: 0.0927 TPS +- client-observed prefill mean: 2,753.3639 TPS +- maximum token gap: 43.781 ms +- failed samples: zero + +The canonical Q4 quality output was produced with SHA1 `3302eda43396`. The +quality verifier now accepts an explicit `--expected-sha1 3302eda43396` contract +selector while preserving the historical paper-inspired default +`0acef4eab6f4`. This prevents a contract mismatch from being misreported as a +model-quality failure. The selected hash and observed hash are both retained in +the raw artifact. A prior protected-service re-anchor recorded +`cd580f4978fb` under a different source or request contract. The explicit +current verifier contract returned `0acef4eab6f4` on the healthy protected +service, so neither fingerprint may be treated as universal. This Y4 run +remains a source-matched historical Q4 result only. + +Against the accepted current Q4 scheduler baseline near 48.28 decode TPS, this +current-branch Y4 result is approximately 5.8 percent slower. **Decision: +reject Y4 for promotion on the current branch.** It is not comparable to the +separate protected-service re-anchor without the exact source and request +manifest. The compile-time option +remains available only for reproduction and future architecture-specific work; +the default remains Y1. The protected Qwen service was restored and its health +endpoint returned `status: ok` with `maintenance: serving` after the candidate +stopped. +## Same-source Y1 control + +To remove the remaining source-revision confounder, the same checkout and +request contract were rerun with `FREETOKEN_GGUF_MMV_Y=1` in a separate +isolated artifact. The three-sample scheduler-shaped control measured: + +- decode mean: 45.3341 TPS +- client-observed prefill mean: 2,682.4559 TPS +- decode standard deviation: 0.0619 TPS +- failed samples: zero +- quality: passed with `--expected-sha1 3302eda43396` + +The paired Y4 result was 45.4603 decode TPS and 2,753.3639 prefill TPS. Y4 was +therefore only 0.28 percent faster in this same-source comparison, well below +the one-percent promotion floor and normal run variation. **Decision: +definitively reject Y4 as a current-branch performance promotion.** + +Artifacts: + +- Y1: `/home/operator/freetoken-amd/artifacts/qwen-q4-current-mmvy1-20260905T160000Z/` +- Y4: `/home/operator/freetoken-amd/artifacts/qwen-q4-current-mmvy4-20260905T133000Z/` diff --git a/docs/gmktec-evo-x2-qwen-q4-raw-control-20260830.md b/docs/gmktec-evo-x2-qwen-q4-raw-control-20260830.md new file mode 100644 index 0000000000..264fd27f9e --- /dev/null +++ b/docs/gmktec-evo-x2-qwen-q4-raw-control-20260830.md @@ -0,0 +1,61 @@ +# GMKtek EVO-X2 Qwen Q4 raw-prompt control, 2026-08-30 + +This report records an apples-to-apples ROCm 10 comparison between the AMD +FreeToken port and llama.cpp. It is a quality and steady-state decode control, +not a throughput claim for cold startup or a production service benchmark. + +## Host and runtime + +- Host: GMKtek EVO-X2, AMD Strix Halo `gfx1151`, 56 GiB unified GPU memory. +- FreeToken runtime: native ROCm 10 and HIP execution path, Triton attention, + offload MoE backend, serial expert loading, Q4_K_M GGUF. +- llama.cpp runtime: ROCm 10 `llama-server`, full GPU layer offload, Flash + Attention enabled, Q8_0 K and V cache. +- Model file: `Qwen3.6-35B-A3B-UD-Q4_K_M.gguf`. +- Prompt renderer: the checkpoint's native Qwen tokenizer, not either server's + chat-template implementation. + +## Control contract + +Each server received the exact same UTF-8 string at `/v1/completions` with +`temperature=0`, `top_p=1`, `top_k=-1`, streaming enabled, and a 1024-token +generation ceiling. The prompt SHA-256 was +`224f02631165a176e660363fefeb8eb58e5a150271fed72bdc1f90fa39448523` and each +server reported 54 prompt tokens. The shared AIME answer is `70`. + +This test exists because the earlier GGUF fast-tokenizer conversion split Qwen's +`` marker into three normal pieces. FreeToken now restores GGUF CONTROL +and USER_DEFINED token entries as atomic special tokens, preserving their +original vocabulary IDs and matching the checkpoint tokenizer's 54-token input. + +## Results + +| Engine | Prompt tokens | Generated tokens | Steady decode TPS | Quality evidence | +| --- | ---: | ---: | ---: | --- | +| FreeToken AMD | 54 | 1023 | 47.12 | Derives `b + 7` divides `56`; verifies `b=21` and `b=49` | +| llama.cpp ROCm 10 | 54 | 1024 | 50.29 | Derives the same divisibility condition and the same two bases | + +The initial FreeToken control was 6.3% below llama.cpp. After enabling the +in-tree, native HIP Triton router, the repeated FreeToken control completed at +50.63 TPS while preserving the same correct derivation and 54-token prompt. +That is 0.7% above the 50.29 TPS llama.cpp control. The HIP router is therefore +the ROCm default; set `FREETOKEN_ROCM_TRITON_ROUTER=0` only to force the slower +PyTorch reference router for a diagnosis. + +The response from each engine remained inside Qwen's verbose reasoning trace at +the 1024-token ceiling, so neither emitted the requested boxed final line. This +is not treated as a quality pass based only on formatting. The recorded math +explicitly proves the two valid bases, whose sum is 70, matching the fixed +ground truth. A future quality gate should either provide a larger token budget +or use a prompt that requests a concise answer after the reasoning trace. + +## Evidence locations on GMKtek EVO-X2 + +- FreeToken: `/home/operator/freetoken-amd/artifacts/qwen-gguf-raw-20260830T032253Z/raw-quality.json` +- FreeToken with HIP router: `/home/operator/freetoken-amd/artifacts/qwen-gguf-raw-20260830T033941Z/raw-quality.json` +- llama.cpp: `/home/operator/freetoken-amd/artifacts/qwen-llama-raw-20260830T033324Z/raw-quality.json` + +The two self-restoring control runners are +`scripts/host-identity canary/run_qwen_gguf_raw_control.sh` and +`scripts/host-identity canary/run_qwen_llamacpp_raw_control.sh`. They reserve the GPU only +temporarily and invoke the production recovery helper on exit. diff --git a/docs/gmktec-evo-x2-qwen-router-optimization-2026-08-29.md b/docs/gmktec-evo-x2-qwen-router-optimization-2026-08-29.md new file mode 100644 index 0000000000..6049433c34 --- /dev/null +++ b/docs/gmktec-evo-x2-qwen-router-optimization-2026-08-29.md @@ -0,0 +1,733 @@ +# GMKtek EVO-X2 Qwen router and cache optimization, 2026-08-29 + +## Scope + +This record covers only the isolated FreeToken server on GMKtek EVO-X2's Radeon 8060S +(`gfx1151`). It did not start, stop, unmask, or reconfigure llama-swap or the +production llama.cpp service. All server instances bound only to `127.0.0.1:1919`. + +## Rejected router candidate + +FreeToken's vendored Triton softmax top-k router was evaluated on ROCm. +Qwen3.6 NVFP4 uses 256 experts and selects eight experts per token. On +GMKtek EVO-X2, the candidate matched the PyTorch reference in the isolated router +test and reduced router-only latency at the production shape. + +| Router microbenchmark | PyTorch reference | HIP Triton | Speedup | +| --- | ---: | ---: | ---: | +| 1 token, 256 experts, top-8 | 0.02449 ms | 0.01512 ms | 1.62x | +| 4 tokens, 256 experts, top-8 | 0.02480 ms | 0.01518 ms | 1.63x | + +The focused ROCm test set passed 11 tests, including routing parity cases. +That evidence was necessary but not sufficient: an earlier end-to-end greedy +AIME run produced a different output hash with this router. The candidate is +therefore rejected and ROCm retains the exact PyTorch router. + +## Transport canary and decode experiments + +The Qwen API harness sends greedy sampling and `reasoning_effort=none`. This +is required because otherwise Qwen can stream its reasoning trace until the +output cap before returning a final answer. The exact-answer canary returned +`host-identity canary` on warmup and scored requests, proving API transport and parser +behavior only. It is not an end-to-end model-quality acceptance test. + +The sustained-decode workload is a 1,212-prompt-token, repeated scheduler +paragraph with 251 forced generated tokens and three scored samples. It is a +warm cache workload, so its input TPS is not an uncached-prefill claim. + +| Configuration | Quality | Mean output TPS | Sample output TPS | First-text TTFT | +| --- | --- | ---: | --- | ---: | +| Reference PyTorch router, 8,990 slots | Transport passed previously | 26.731 | 26.724, 26.728, 26.740 | not measured correctly by earlier harness | +| HIP Triton router, 8,990 slots, no graph | Transport passed, AIME regression found later | 29.186 | 29.201, 29.183, 29.175 | 400 to 405 ms | +| HIP Triton router, 10,006 slots, no graph | Transport passed, inherits router regression | 29.080 | 29.077, 29.078, 29.086 | 359 ms canary | +| HIP Triton router, 8,990 slots, graph batch 1 | Transport passed, inherits router regression | 28.830 | 28.827, 28.823, 28.839 | 398 to 404 ms | + +The accepted quality configuration retains the PyTorch router. The faster +Triton rows are retained as performance evidence, but must not be used as an +accepted model-serving configuration until their AIME output differs only for +an independently justified numerical reason and task-level quality is proven. + +### Current quality restoration proof + +After restoring the exact PyTorch router, the same AIME-25 problem zero was +warmed once and measured once against the live GMKtek EVO-X2 server. The checkpoint +used greedy sampling, a thinking-enabled template, and a forced 128-token +decode. The 54-token prompt produced the historic output SHA-1 +`0acef4eab6f4` exactly. The dedicated script +`scripts/gmk-evo-x2/verify_qwen_aime_quality.py` now makes this a repeatable +quality gate for every future performance candidate. + +The gate now records client-visible timing from the same streamed request. Three +additional warm, quality-matched repeats all produced the reference hash: + +| Measure | Result | +| --- | ---: | +| Mean decode TPS | 27.880 | +| Decode TPS samples | 26.786, 28.422, 28.431 | +| Mean warm TTFT | 409.0 ms | +| Prompt / completion tokens | 54 / 127 | +| Output hash | `0acef4eab6f4` in every run | + +The first run includes a modest cache or scheduler outlier, with a 50.49 ms +p99 event gap, while the two later runs had 37.81 ms and 37.29 ms p99 gaps. +The three-run mean is 3.6 percent below the historical 28.935 TPS +quality-matched reference. It is therefore a bounded regression, not evidence +that the rejected Triton router should be restored. + +## Calibration and rejected alternatives + +`ft bench bw` measured Qwen NVFP4's real expert kernels on GMKtek EVO-X2. The CPU +expert path reached 4.8 GB/s, while HIP expert gather reached 92.5 GB/s. That +is a 0.05x CPU-to-gather ratio, so the calibration selected `offload`, not +`hybrid`. CPU and GPU hybrid execution is therefore not a sound optimization +candidate for this checkpoint on this host. + +Increasing `--memory-ratio` from 0.35 to 0.38 increased automatic cache +residency from 8,990 to 10,006 slots and reduced free memory from 19.44 GiB to +17.78 GiB. It did not improve decode throughput, so the default remains 0.35. + +ROCm graph capture was accepted and completed for batch size one, but reduced +sustained decode throughput by about 1.2 percent. It remains disabled in the +accepted isolated launcher. + +## Evidence locations on GMKtek EVO-X2 + +```text +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T085317Z/ +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T090601Z/ +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T091716Z/ +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T093921Z/aime-quality-tps-run1.json +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T093921Z/aime-quality-tps-run2.json +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T093921Z/aime-quality-tps-run3.json +``` + +The current best configuration is reloading under: + +```text +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T092725Z/ +``` + +## Remaining gap + +The 29.186 client decode TPS is an informative but rejected performance-only +result, not a quality-validated serving claim. The quality-preserving reference +router is now measured at 27.880 mean TPS with the improved hash-gated harness. +The paper's exact prompt sequence, stop policy, warm-cache state, and source +revision remain unrecovered, so this is not a strict paper-parity comparison. +Further work should profile per-layer NVFP4 expert execution and the Qwen +linear-attention path under native HIP, then repeat this task-level quality and +throughput protocol. + +## Native HIP trace and FP8 dense-path investigation + +### Trace method and limitations + +`rocprofv3 --attach` cannot instrument an already running server with this +PyTorch ROCm wheel because the wheel does not provide the ROCProfiler SDK +attachment registration thread. The evidence was instead captured by launching +the same isolated Qwen command directly through the wheel-compatible +`scripts/gmk-evo-x2-rocprof-wheel-sdk.sh` wrapper. That run created the native +ROCm SQLite trace below and passed the deterministic AIME output gate. + +```text +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T100601Z/ + rocprof-full-qwen/GMKtek EVO-X2/54976_results.db +``` + +The profiler recorded 353,457 dispatches. Its 15.61 decode TPS is intrusive +trace overhead, not serving performance and must never be compared with the +unprofiled client TPS rows in this report. + +The added `scripts/gmk-evo-x2/inspect_rocprof_db.py` is a standard-library, +read-only companion for that evidence. It opens the SQLite artifact with +`mode=ro&immutable=1`, inventories ROCm's version-specific table names, and +aggregates a requested final kernel window without altering the database or +requiring a host `sqlite3` package. + +### Dominant kernel + +The final 120-second active window showed that the largest GPU-time consumer is +not the routed NVFP4 expert kernel. It is the dense mixed-FP8 decode kernel +`_gemv_splitk_kernel` from `fp8_pertensor_linear.py`. + +| Kernel | Calls | GPU time in final window | +| --- | ---: | ---: | +| `_gemv_splitk_kernel` | 20,320 | 5,631.844 ms | +| `_gemm_kernel` | 160 | 1,676.018 ms | +| `_decode_nvfp4_marlin_kernel` | 20,320 | 1,566.004 ms | +| `fast_index_copy` | 10,240 | 593.192 ms | +| `_nvfp4_gemv_kernel` | 256 | 322.227 ms | + +Qwen's relevant dense projection shapes include `[8192, 2048]`, `[4096, +2048]`, `[2048, 4096]`, and `[512, 2048]`. The initial split-K policy was +written for NVIDIA's much larger GPU target and partitions the K dimension. +Changing that policy changed the numerical reduction grouping, so it cannot be +treated as a quality-neutral performance switch. + +### Rejected split-K candidate + +An isolated target-512 split-K experiment completed at 22.504 decode TPS and +produced output SHA-1 `1cae5bae914f`, instead of the required +`0acef4eab6f4`. It was both slower and incorrect under the deterministic gate. +The source override was removed and the normal split-K policy restored. + +The next candidate is constrained to output-row tiling only: it preserves the +K chunks, each row's FP32 accumulation, partial-buffer layout, and final +split-K reduction order. It is still a candidate, not an accepted optimization, +until it has a saved exact-hash response and an unprofiled TPS result. + +### Quality-preserving but inconclusive output-row candidate + +The first gfx1151 candidate changed the dense FP8 GEMV output-row tile from 16 +to 32. Its three AIME responses matched the required SHA-1 exactly, but a +post-run source audit found that the old automatic split-K calculation was +derived from the output-row tile. The candidate therefore also changed the K +partition, despite being intended as a row-only experiment. + +| Tile | Output SHA-1 | Output TPS samples | Mean output TPS | Decision | +| --- | --- | --- | ---: | --- | +| 16 baseline | `0acef4eab6f4` | 26.786, 28.422, 28.431 | 27.880 | Validated baseline | +| 32 first candidate | `0acef4eab6f4` | 28.677, 26.867, 28.683 | 28.075 | Prompt gate passed, arithmetic scope corrected afterward | + +The candidate mean is 0.7 percent higher than the earlier baseline mean, but +the 26.867 TPS sample had a 114.51 ms p99 stream-event gap and the immediate +post-reboot tile-16 validation measured 28.596 TPS. This is within normal +measurement variation, not evidence of a repeatable throughput improvement. +More importantly, the inadvertent split-K coupling means this candidate cannot +establish a row-only quality claim. It is not the default. The implementation +now derives split-K from the validated 16-row reference tile even when a +different output-row tile is requested, then the corrected candidate must be +retested from scratch. + +### Corrected HIP kernel screen + +After decoupling split-K from the output-row tile, the isolated microbenchmark +used deterministic synthetic tensors at Qwen's real `[N, 2048]` shapes. It +warms each compiled kernel, records 100 native HIP event timings, and hashes +the raw BF16 result buffer. Every compared row below has the same output hash +as its tile-16 baseline for that shape. This is a kernel-level numerical check, +not a replacement for the end-to-end AIME gate. + +| Shape | Candidate | Baseline median | Candidate median | Result | +| --- | --- | ---: | ---: | --- | +| `[8192, 2048]` | 32 output rows, fixed split-K | 0.0764 ms | 0.0779 ms | Slower | +| `[4096, 2048]` | 32 output rows, fixed split-K | 0.0392 ms | 0.0396 ms | Slower | +| `[2048, 2048]` | 32 output rows, fixed split-K | 0.0325 ms | 0.0330 ms | Slower | +| `[8192, 2048]` | two waves, 16 output rows | 0.0764 ms | 0.0765 ms | No material gain | +| `[8192, 2048]` | activation-side exact FP8 scale | 0.0764 ms | 0.0766 ms | No material gain | + +The activation-scale candidate decodes each FP8 byte as an exact fp16 value +divided by 256, then applies the compensating exact power-of-two scale once to +the FP32 activation. It was bit-identical in the screen but did not lower +latency. It remains disabled. All of these variants are rejected from default +serving because the target is a repeatable end-to-end TPS gain with unchanged +quality, not merely a different kernel that happens to pass one output check. + +The current source passed the native focused regression suite after these +experiments: `22 passed, 11 skipped` in +`tests/kernels/test_fp8_pertensor_linear.py` on GMKtek EVO-X2. + +### Hardware counters and NVFP4 follow-up + +The wheel-compatible ROCm profiler wrapper also supports isolated performance +counter collection. A direct host `rocprofv3` launch aborts before Python starts +because it injects a second LLVM and registers `spirv-expand-step` twice. The +existing wheel-SDK wrapper avoids that conflict and captured the dense FP8 +`[8192, 2048]` GEMV successfully. `FetchSize` and `VALUUtilization` are not +available for this `gfx1151` agent through the installed SDK, but the available +counters are sufficient to classify the bottleneck: + +| Counter | Observed range on steady GEMV dispatches | Interpretation | +| --- | ---: | --- | +| `MemUnitBusy` | about 89 to 93% | The memory unit is near saturation | +| `L2CacheHit` | about 39 to 51% | Large streamed FP8 weights do not persist fully in L2 | + +That evidence explains why row tiles, additional waves, and relocation of an +exact power-of-two FP8 scale did not create a repeatable gain. The next profile +consumer was the Marlin-style inline-NVFP4 MoE decode kernel, so it received an +equally strict screen at Qwen's actual eight-route shapes: gate/up `[1024, +2048]` and down `[2048, 512]`. + +| Projection | Candidate | Raw BF16 hash | Timing result | Decision | +| --- | --- | --- | --- | --- | +| Gate/up | 8 output rows | Changed | Faster in isolation | Rejected: exact output changed | +| Gate/up | 32 output rows | Matched | 0.0946 ms vs 0.0615 ms baseline screen | Rejected: slower | +| Gate/up | 2 waves | Matched | 0.0941 ms | Rejected: slower | +| Gate/up | 8 waves | Changed | 0.0557 ms | Rejected: exact output changed | +| Down | 8, 16, or 32 output rows; 2, 4, or 8 waves | Matched | No faster result | Rejected: no repeatable gain | + +The helper `scripts/gmk-evo-x2/bench_nvfp4_marlin_decode.py` creates layout-correct +NVFP4 banks and evaluates the production decode kernel directly. It deliberately +uses raw output SHA-1 as the first gate, so numerically faster variants cannot +leak into a full-model reload merely because they are faster. + +### Current-main integration validation + +The AMD branch merged FreeToken upstream commit `58f4b9e`, which fixes an +NVIDIA Ada row-wise W8A8 prefill issue. The merge is current-main compatible +and does not alter GMKtek EVO-X2's ROCm W8A16 dense decode route, but it was still +validated from a fresh isolated server launch rather than inferred from source +inspection. The combined focused native test suite completed with `28 passed, +22 skipped`. + +The reloaded current-main server passed the deterministic AIME gate with the +required output SHA-1 `0acef4eab6f4`, 28.775 output TPS, 403.57 ms TTFT, and +36.77 ms p99 stream-event gap. The newer source resolved 8,974 cache slots and +2,068 KV pages with 19.45 GiB free memory, versus 8,990 slots and 2,081 pages +in the prior baseline. This allocation difference is recorded as a source +revision effect, not an optimization result. + +```text +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T113506Z/ + aime-quality-current-main.json +``` + +The final current-main service health check returned `status: ok` on +`127.0.0.1:1919`. A failed direct host-profiler run left one non-serving Python +child behind; it was identified by its profiler benchmark command, terminated, +and then force-cleared when it ignored SIGTERM. The serving parent and current +worker were verified separately before and after that cleanup. + +### Unified-memory expert-cache copy screen + +The full ROCm trace showed `fast_index_copy` at 593.192 ms across 10,240 +dispatches. That total makes the helper worth measuring, but it does not prove +that cache fills limit observed single-stream decode TPS. The existing cache +copy benchmark did not previously encode Qwen3.6-35B-A3B-NVFP4's real model +geometry, so the AMD branch adds a documented profile: 40 MoE layers, 256 +experts per layer, top-8 routing, hidden size 2048, intermediate size 512, and +the production six-bank NVFP4 layout. + +On GMKtek EVO-X2, with a 513-slot cache, one active token and all eight routed experts +missing, the benchmark copied 13.5 MiB in 0.097 ms, or 146.8 GB/s. Across all +40 MoE layers, its documented extrapolation is 3.87 ms per decode token. The +all-hit case took 0.023 ms. This is a native HIP measurement using the actual +allocation and copy path, not a theoretical memory-bandwidth figure. + +| Batch | Active experts | Miss rate | Copy amount | Median time | Bandwidth | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 8 | 0% | 0.0 MiB | 0.023 ms | not applicable | +| 1 | 8 | 100% | 13.5 MiB | 0.097 ms | 146.8 GB/s | + +The measured worst-case fill is materially smaller than the approximate 35 ms +per-token service interval at 28.775 output TPS. It does not support changing +`fast_index_copy` parameters or bypassing cache maintenance: cache behavior is +correctness-sensitive, and the directly profiled dense FP8 and NVFP4 decode +kernels remain the dominant performance targets. The full reproducibility log +and exit code are retained at: + +```text +/home/operator/freetoken-amd/artifacts/qwen-copy-bench-20260829T114800Z/ +``` + +### Live clock and power-state verification + +The service process is configured for the native ROCm 10 HIP runtime and its +CPU host was already in the Linux `performance` governor. Idle sensor readings +reported the expected 600 MHz shader clock, which is not suitable evidence for +a decode-performance diagnosis. A fixed 256-token, three-sample API workload +therefore ran on the unchanged loopback service while ROCm SMI collected one +sample per second. + +The workload completed all three samples at 28.270 mean output TPS with a +0.014 TPS standard deviation. During its steady portion, GPU utilization was +100 percent in 24 samples, shader clocks reached and held the 2.9 GHz state, +memory clock remained at 1.0 GHz, and package graphics power was typically +about 70 to 90 W, with a 114 W peak sample. The service continued to report +`status: ok` after the workload. + +This excludes an inactive CPU governor, idle shader state, or obvious +power-state failure as the explanation for the present decode ceiling. It is +consistent with the isolated kernel counters: decode is actively executing at +the device's performance state and the dense FP8 memory unit is already near +saturation. Hardware clock forcing is therefore not a justified safe +optimization. Reproducible workload and sensor artifacts are retained at: + +```text +/home/operator/freetoken-amd/artifacts/qwen-live-telemetry-20260829T050800Z/ +``` + +### Reusable gfx1151 C++ and HIP cache + +GMKtek EVO-X2 initially had no `freetoken_kernel_cache` package and therefore no +formal prebuilt helper-kernel inventory. The AMD branch now includes +`scripts/gmk-evo-x2/build_rocm_kernel_cache.sh`. It validates the native HIP +runtime and gfx1151 device, derives a source-revision-scoped cache path, and +compiles the complete explicit model catalog with four bounded compiler jobs. +The startup script resolves that cache and sets `FREETOKEN_DISABLE_JIT=1`, so a +missing FreeToken C++ or HIP helper fails explicitly instead of compiling during +an inference request. + +The first full build found and corrected a portability defect in the shared AOT +catalog: 240-byte and 400-byte scale-bank rows were incorrectly emitted for the +legacy per-bank kernel even though its vector loop requires whole 128-byte +worker rows. Those small rows are supported by the production fused multi-bank +path, which has tail handling. The branch now excludes only the impossible +legacy templates and has a regression test that pins the rule. The repaired +ROCm 10 build produced all 80 valid catalog modules for gfx1151: + +```text +/home/operator/freetoken-amd/cache/kernel-cache-rocm-gfx1151-d6ee8cef479c/ +``` + +`scripts/gmk-evo-x2/verify_rocm_kernel_cache.py` then loaded every one of those 80 +modules with `FREETOKEN_DISABLE_JIT=1`. This verifies ABI-compatible loading +through the installed Python, TVM FFI, ROCm 10 and HIP runtime, which a shared +object file count alone cannot prove. The verifier neither starts a model nor +modifies the cache. + +The strict cache launch completed the normal serial NVFP4 expert-bank load, +then passed the AIME output gate with the required SHA-1 `0acef4eab6f4` at +28.504 output TPS, 399.99 ms TTFT, and 36.62 ms p99 stream-event gap. It +resolved the same 8,974 MoE cache slots and 2,068 KV pages as the earlier +current-main validation. The startup artifact is retained at: + +```text +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T120405Z/ +``` + +Its fixed 256-token scheduler workload also completed three of three scored +samples at 28.018 mean output TPS, with a 28.017 median and 0.007 TPS standard +deviation. This is consistent with the preceding quality-gated runs and shows +that enforcing the reusable C++ and HIP cache changes startup compilation +behavior, not steady-state decode arithmetic or output quality. + +This cache eliminates FreeToken's C++ and HIP helper JIT for the catalog it +contains. It does not claim to precompile every Triton specialization or a +GGUF kernel: Qwen3.6-35B-A3B-NVFP4 is a safetensors NVFP4 checkpoint, not a +GGUF model, and Triton maintains its own architecture- and source-keyed +persistent cache. + +### Rejected eight-wave FP8 GEMV candidate + +An eight-wave gfx1151 FP8 GEMV launch was screened because it produced the +same isolated raw BF16 result hash as the one-wave baseline and reduced the +microbenchmark median from 0.07703 ms to 0.07574 ms for the 8192 by 2048 +matrix. It passed the deterministic API quality gate with the required AIME +SHA-1 `0acef4eab6f4`. That isolated result did not carry over to the actual +Qwen decode workload: the fixed three-sample, 256-output-token scheduler test +averaged 26.260 TPS with 0.0008 TPS standard deviation, compared with the +strict-cache one-wave baseline of 28.018 TPS. This is a 6.3 percent regression. + +The eight-wave option was therefore removed from the accepted launcher and +benchmark allowlists. The candidate artifacts are retained for reproducibility +at: + +```text +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T121916Z/ +``` + +This result demonstrates why raw tensor equality and a favorable isolated +kernel timing are necessary but not sufficient acceptance conditions. The +multi-layer MoE decode schedule has materially different occupancy and cache +interaction from a single dense GEMV invocation. + +### Rejected FP8 GEMV pipeline-depth screen + +The next quality-safe dense FP8 candidate was an explicit Triton pipeline depth +for the existing 16-row, one-wave, fixed split-K GEMV. Pipeline depth changes +software scheduling but not the arithmetic, output-row tile, K chunks, or +split-K reduction tree. All five tested depths produced the same raw BF16 SHA-1 +`0aca8b9e38ebfaa91893366a175970f1c45599b9` on the Qwen-shaped 8192 by 2048 +screen. + +The initial 200-iteration pass made stage 2 look marginally favorable, with a +0.07698 ms median versus 0.07734 ms for stage 3. A new, longer 500-iteration +paired measurement reversed that apparent advantage: stage 3 measured 0.07689 +ms median versus 0.07704 ms for stage 2, while their means were effectively +identical at 0.07702 ms. Stages 1, 4, and 5 were slower in the initial screen. +The small difference is ordinary device-timing variation, not a defensible +end-to-end improvement. The staging override was removed without a model +reload, preserving the established default Triton pipeline policy. + +### Measured expert-cache residency and rejected capacity increase + +The AMD branch now exposes a read-only `/v1/cache/stats` endpoint through the +existing API, tokenizer, and scheduler control-message path. It transfers an +already accumulated device-counter snapshot only when explicitly requested; +it does not alter cache contents, scheduling, routing, model weights, or the +normal no-statistics serving path. The associated `--moe-collect-stats` launch +option is disabled by default because its counter updates are diagnostic work. + +At the validated 0.35 memory ratio, Qwen allocated 8,974 cache slots, 2,068 +KV pages, and 24 GDN state slots. The fixed workload accumulated 33,920 +MoE-layer decode calls: eight active experts per layer and 0.671 misses per +layer, an 8.39 percent miss rate. A 0.38 memory-ratio candidate raised +residency to 9,990 slots while retaining 2,055 KV pages and 24 GDN slots. It +reduced the realized miss rate to 7.33 percent, but its three-sample scheduler +throughput fell from 28.038 TPS to 27.908 TPS. The capacity increase is +therefore rejected. It consumes roughly 1.7 GiB of additional headroom without +producing a serving improvement. + +The counters establish that a small number of expert fetches remains, but not +that larger static residency is a profitable AMD optimization. Direct cache +copy measurements and the sustained TPS result agree: the dense FP8 decode +path remains the more valuable target. The two diagnostic artifact roots are: + +```text +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T124643Z/ +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T125511Z/ +``` + +### Rejected 64-block fused expert-copy candidate + +The production fused expert-cache copy helper was also screened with the real +Qwen3.6-35B-A3B-NVFP4 six-bank layout. Both candidate grids use precompiled +gfx1151 HIP modules, run with `FREETOKEN_DISABLE_JIT=1`, and were required to +copy every selected source row byte-for-byte into its requested cache slot +before a timing sample was recorded. With one missing expert, widening from +eight to 64 blocks per bank reduced median copy time from 0.01872 ms to +0.01812 ms. With eight missing experts, it reduced the median from 0.07984 ms +to 0.05982 ms, increasing the measured copy rate from 177.9 GB/s to 237.5 +GB/s. + +That isolated improvement preserved deterministic model output: the 64-block +server returned the required AIME SHA-1 `0acef4eab6f4`, with 28.582 output TPS +on that quality request. It did not improve the fixed three-sample scheduler +workload. The end-to-end result was 28.070 TPS mean, 28.078 TPS median, and +0.016 TPS standard deviation, below the fresh default eight-block result of +28.153 TPS mean. The larger launch grid is therefore rejected as the serving +default. The code retains an explicitly bounded `8|64` diagnostic selection so +future cache changes can be remeasured without a new JIT specialization, but +the launcher defaults to the accepted eight-block grid. + +The full candidate artifacts, including exact-copy microbenchmark data, AIME +quality result, and scheduler samples, are retained at: + +```text +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T131629Z/ +``` + +### Native-library replacement screen + +The Qwen checkpoint carries calibrated `input_scale` tensors, so a W8A8 +hipBLASLt replacement was investigated as a possible way to replace the +memory-bound W8A16 dense decode kernel. GMKtek EVO-X2 is running ROCm 10.0 with +hipBLASLt 1.4 and PyTorch `2.13.0+rocm10.0.0`, but the route is not available +for this model and GPU. PyTorch's native `_scaled_mm` call on gfx1151 rejects +the operation before dispatch, reporting that it is supported only on CUDA +compute capability 8.9 or 9.0 devices, or ROCm MI300-class devices. This is a +runtime capability gate, not a FreeToken configuration error. + +The installed hipBLASLt 1.4 headers also expose `HIP_R_8F_E5M3_EXT` but not an +OCP E4M3 matrix data type. Qwen's dense weights are OCP E4M3, so directly +calling that ABI would require a representation conversion and would no longer +be a like-for-like W8A8 replacement. A BF16 conversion would double weight +traffic in the already memory-bound decode path. Neither alternative is an +acceptable serving optimization or a valid quality-preserving AMD port. + +The conclusion is deliberately limited to this ROCm 10, PyTorch 2.13, and +gfx1151 environment. The result keeps the verified native Triton W8A16 route +as the active path and directs follow-up work toward custom kernels that retain +the checkpoint's OCP E4M3 bytes and its exact output contract. + +### Rejected direct HIP OCP-E4M3 GEMV prototype + +To validate that a custom HIP component remains possible despite the library +gate, an isolated one-Wave32-per-output-row W8A16 GEMV was compiled with +ROCm 10 hipcc for gfx1151. It reads the checkpoint's raw OCP E4M3 bytes, +streams one shared BF16 activation vector, accumulates in FP32, applies the +existing per-row FP32 scale, and avoids the production split-K partial buffer. +The prototype is intentionally outside the API serving path. + +The build succeeded after supplying rocThrust as a compiler-only system include +to PyTorch's ROCm extension machinery. At Qwen's `[8192, 2048]` dense shape, +however, the candidate measured 0.10674 ms median versus 0.07687 ms for the +production Triton kernel, a 38.9 percent regression. Its raw BF16 SHA-1 also +differed (`e08b284faedd608850119511655e1e94cab87b05` versus +`0aca8b9e38ebfaa91893366a175970f1c45599b9`), with a maximum absolute element +difference of `3.814697265625e-06`. The altered wave-local reduction order is +therefore not an exact replacement. + +This prototype is rejected before model integration. The artifact preserves +the complete hipcc command and timing JSON for later component work: + +```text +/home/operator/freetoken-amd/artifacts/fp8-hip-prototype-20260829T133900Z/ +``` + +### System-level performance-policy audit + +GMKtek EVO-X2's CPU governor is already `performance`. The Radeon 8060S reports the +standard `auto` GPU performance policy at idle, where shader and SoC clocks +fall to 600 MHz while memory remains at 1,000 MHz. This is not evidence of a +decode throttle: the earlier fixed API workload recorded 100 percent GPU use, +a sustained 2.9 GHz shader clock, and roughly 70 to 90 W graphics package +power during active generation. + +This host does not expose the usual amdgpu DPM control files through DRM sysfs, +and `rocm-smi` reports that its power cap is unsupported. An isolated +high-performance-policy test is therefore contingent on interactive sudo +authentication. The port does not change an undocumented platform policy or +claim a clock-based gain without that reversible measurement. The serving +baseline stays on the normal driver policy and remains subject to the same +quality and TPS gates as kernel candidates. + +#### DPM-policy measurement contract and setup failure + +The DPM experiment uses +`scripts/gmk-evo-x2/run_qwen_dpm_policy_benchmark.sh`. It requires an interactive +sudo credential in the terminal that invokes it because the host caches sudo +authorization per terminal. The script requests a named temporary policy, +records the pre-run policy, delegates the fixed three-sample 256-token Qwen +scheduler workload to the existing harness, and restores the normal `auto` +policy through an `EXIT` trap. It neither reloads FreeToken nor alters the +model, cache, scheduler configuration, llama-swap, or other LAN hosts. + +The policy log lives in a newly-created parent evidence directory. The harness +receives a distinct, absent `benchmark` child directory because its immutable +artifact contract intentionally fails if that exact directory already exists. +This separation is enforced by a unit test in +`tests/benchmarks/test_gmk_evo_x2_benchmark.py`. + +An initial manual attempt at `2026-08-29T18:30:35Z` correctly changed the GPU +from `auto` to `high` and restored it to `auto`, but created the harness +artifact directory before invoking the harness. The harness consequently +raised `FileExistsError` before making an API request. It produced no scored +samples, no input TPS, and no output TPS, so it is not a performance result +and must not be compared with the `auto` baseline. The corrected wrapper and +its regression test were added after that attempt. A valid high-policy result +requires a fresh artifact containing the harness manifest, all three scored +sample JSON files, the policy log, and a post-run health check. + +#### Valid high-policy result + +The corrected wrapper completed a valid high-policy run at +`2026-08-29T22:02:24Z`. It recorded `high` before the workload and `auto` +afterward, completed all three forced 251-generated-token scheduler samples, +and left the isolated OpenAI-compatible Qwen endpoint healthy. The matching +accepted eight-block `auto` baseline used the same 1,212-token prompt, +251 generated tokens, three scored samples, model, server process, cache +configuration, fixed decoding settings, and loopback endpoint. + +| GPU policy | Mean output TPS | Median output TPS | Output TPS samples | Mean input TPS | Mean TTFT | +| --- | ---: | ---: | --- | ---: | ---: | +| `auto` | 28.153 | 28.150 | 28.147, 28.150, 28.162 | 2913.096 | 416.255 ms | +| `high` | 28.355 | 28.353 | 28.353, 28.362, 28.349 | 2957.942 | 409.835 ms | + +The temporary `high` policy improved fixed-workload output throughput by +0.202 TPS, or 0.72 percent, and raised measured input throughput by 44.846 +TPS, or 1.54 percent. Mean first-text latency decreased by 6.420 ms, or 1.54 +percent. The output gain exceeds the combined run-to-run standard deviations +of the two three-sample sets, but the sample count is deliberately small, so +this is a measured operating preference rather than a broad claim about every +prompt shape or concurrent load level. + +After restoration to `auto`, the live service passed the existing deterministic +AIME quality gate with the required output SHA-1 `0acef4eab6f4`. Its 127-token +quality stream measured 28.421 decode tokens per second and 395.561 ms TTFT. +The quality artifact proves that the run did not leave the model or serving +configuration altered. The policy wrapper changes only driver performance +policy, not model arithmetic, but that post-run gate is not presented as a +separate quality measurement performed while `high` was active. + +The complete high-policy evidence is retained at: + +```text +/home/operator/freetoken-amd/artifacts/qwen-dpm-high-20260829T220224Z/ +``` + +### Same-base-model ROCm 10 llama.cpp control + +GMKtek EVO-X2's original llama-swap Qwen control was `Qwen3.6-27B-Q4_K_M`, which is +not the model served by FreeToken and cannot establish same-model Qwen parity. +For a controlled comparison, the isolated directory +`models/controls/qwen36-35b-a3b-unsloth-a483e9e6/` now contains +`Qwen3.6-35B-A3B-UD-Q4_K_M.gguf` from +`unsloth/Qwen3.6-35B-A3B-GGUF` revision +`a483e9e6cbd595906af30beda3187c2663a1118c`. The downloaded file is +22,134,528,992 bytes; Hugging Face Xet recorded completed-file SHA-256 +`d0f6c2fa907594b8a8322531f188c7c12708db507df3402a18391db1f38eec50`. + +The control used the existing ROCm 10 llama.cpp `b10141` binary at commit +`0d47ea742`, AMD Clang 23, full GPU offload, Flash Attention, one slot, an +8,192-token context, Q8 KV cache, loopback port 1921, and normal GPU `auto` +policy. `scripts/gmk-evo-x2/run_qwen_llamacpp_rocm_control.sh` starts this server +only for the run, delegates to the same fixed Qwen scheduler harness as +FreeToken, saves raw server and client evidence, and terminates the temporary +server through an `EXIT` trap. It never changes llama-swap or the production +llama.cpp route. + +The full Q4 GGUF needs 20.58 GiB of device allocation, so it cannot coexist +with the live FreeToken Qwen service, which deliberately retains about 19.45 +GiB free. The authorized comparison therefore ran the full-GPU servers +sequentially. FreeToken was restored immediately afterward with the strict +no-JIT recovery script and passed the required AIME SHA-1 `0acef4eab6f4`. + +| Runtime | GPU policy | Model representation | Client output TPS samples | Mean output TPS | Median output TPS | Mean client input TPS | Mean TTFT | +| --- | --- | --- | --- | ---: | ---: | ---: | ---: | +| FreeToken | `auto` | NVIDIA NVFP4 checkpoint through native HIP Triton | 28.147, 28.150, 28.162 | 28.153 | 28.150 | 2913.096 | 416.255 ms | +| FreeToken | `high`, temporary policy screen | Same NVIDIA NVFP4 checkpoint | 28.353, 28.362, 28.349 | 28.355 | 28.353 | 2957.942 | 409.835 ms | +| llama.cpp ROCm 10 | `auto` | Base-model Q4_K_M GGUF | 49.221, 49.245, 49.235 | 49.234 | 49.235 | 20146.536 | 60.160 ms | + +Each row used the same 1,212-token request prompt, greedy decoding, +`ignore_eos=true`, warmup request, three scored requests, and 256-token server +generation cap. llama.cpp emitted 256 tokenizer-counted text tokens in every +scored request. The FreeToken client tokenizer counted 251 emitted text tokens +for the same cap because its OpenAI stream parser and Qwen reasoning handling +do not expose every server-side generation token as user text. The output TPS +metric therefore compares client-visible fixed-length streams closely, but is +not an exact token-level arithmetic comparison. + +Under this protocol, llama.cpp is 74.88 percent faster than FreeToken's +accepted `auto` output-TPS baseline and 73.64 percent faster than the temporary +FreeToken `high` policy screen. llama.cpp's input TPS is not directly +comparable: after warmup it reports the full 1,212 request tokens in API usage +while its slot log shows only four newly evaluated prompt tokens due to prefix +cache reuse. Its 20,146.536 client input-TPS figure is therefore a warm cache +accounting result, not an uncached-prefill advantage of that magnitude. + +This is a same-base-model hardware and protocol control, but it is not a +like-for-like weight-format result. Q4_K_M GGUF and NVIDIA NVFP4 differ in +quantization layout, loader, and kernel path. The result proves that current +FreeToken Qwen does not meet the requested "match or exceed llama.cpp" target +on this practical ROCm 10 control. It does not prove an architecture-level +deficit independent of quantization. The raw control bundle is retained at: + +```text +/home/operator/freetoken-amd/artifacts/qwen35b-llamacpp-rocm10-20260829T222546Z/ +``` + +The post-control FreeToken recovery bundle, including deterministic quality +evidence, is retained at: + +```text +/home/operator/freetoken-amd/artifacts/qwen-reboot-recovery-20260829T222712Z/ +``` + +#### Exact-Q4 FreeToken feasibility boundary + +The downloaded control GGUF exposes `general.architecture = qwen35moe` and +contains the Qwen3.5 hybrid architecture metadata: full-attention geometry, +MoE expert counts and sizes, rotary sections, plus state-space model (SSM) +inner size, group count, state size, time-step rank, and convolution kernel. +Its tensor table contains both attention and SSM groups and packed routed +expert tensors such as `blk.N.ffn_gate_exps.weight`, +`blk.N.ffn_up_exps.weight`, and `blk.N.ffn_down_exps.weight`. + +FreeToken's current GGUF registry maps only `gemma4`. It consequently rejects +`qwen35moe` before loading weights. A byte-level inspection of this exact file +found 361 F32 tensors, 251 Q8_0 tensors, 80 Q4_K tensors, 37 Q5_K tensors, +and four Q6_K tensors. In particular, `token_embd.weight` is Q8_0, +`output.weight` is Q6_K, each routed-expert gate tensor is Q4_K, and each +routed-expert down tensor is Q5_K. The `Q4_K_M` filename is a model-wide +mixed-quantization recipe, not one universal tensor encoding. + +The vendored GGUF HIP kernels already implement Q4_K dispatch, including ROCm +tile settings. This branch now exposes Q4_K through the Python layer and adds a +GGML-equivalent reference decoder test. Q5_K is still required for the routed +expert down projections, so an exact FreeToken-Q4 versus llama.cpp-Q4 test +requires three native components, all with HIP validation on gfx1151: + +1. A `qwen35moe` GGUF registry entry, metadata parser, tokenizer dispatch, and + tensor-name mapping into the existing Qwen3.5 MoE model. +2. A Qwen GGUF tensor-name loader plus mixed packed routed-expert banks: Q4_K + gate/up and Q5_K down, with byte-exact block-layout tests against GGML + reference dequantization. +3. A quality-gated loading and serving path that supports the hybrid + attention-plus-SSM layer schedule before a new full-GPU five-run matrix. + +This is a real porting project, not a launch-flag adjustment. Until those +components are implemented and validated, the 49.234 TPS llama.cpp Q4 result +remains the best practical same-base-model ROCm control, while the NVFP4 +FreeToken result remains the supported native AMD port measurement. diff --git a/docs/gmktec-evo-x2-real-qwen-nvfp4-layer0-parity.md b/docs/gmktec-evo-x2-real-qwen-nvfp4-layer0-parity.md new file mode 100644 index 0000000000..51be6fa01b --- /dev/null +++ b/docs/gmktec-evo-x2-real-qwen-nvfp4-layer0-parity.md @@ -0,0 +1,83 @@ +# GMKtek EVO-X2 real Qwen NVFP4 layer-zero parity + +## Purpose + +This bounded experiment loads only the first routed-expert layer from the +actual Qwen3.6 NVFP4 checkpoint, then compares the production Marlin decode +kernel with FreeToken's retained LUT-gather baseline. The loader stops when +layer zero is delivered, so later layers are not materialized and the complete +checkpoint is never loaded into the isolated process. + +## Configuration + +| Field | Value | +| --- | --- | +| Checkpoint | `Qwen3.6-35B-A3B-NVFP4` | +| Layer | 0 | +| Experts in source bank | 256 | +| Hidden size | 2,048 | +| Intermediate size | 512 | +| Routed experts tested | 8 | +| Input | Deterministic BF16 hidden vector | +| Kernel comparison | Marlin NVFP4 versus LUT-gather NVFP4 | +| Runtime | Native HIP on `gfx1151` | + +The source loader provided the actual packed NVFP4 tensors, FP8 block scales, +and FP16 per-row global scales from the checkpoint. No synthetic weights were +used in this test. + +## Real-bank fingerprints + +The captured layer-zero source banks had these SHA-256 fingerprints: + +```text +gate_up_packed fe048d221cddc900220aca2f894ece4c0fbef59f504f1a5e822e67cec586dc13 +gate_up_scale 5c2028ffb715de9bb84983f5ba3979872d697a58d746ddd3585cfd2d3a838800 +gate_up_global db32bb8d0ba65259794748e5d5f6d50a9cf0761310fa68e11b7e17c5763d7152 +down_packed 8b8db4ac1fc04992189ea4371a763bab0fa0ec536b7a528b0183eb4023179945 +down_scale 1b85bd6599f7191e8e54accea54db1c7ac93341fb50f9f8abc5a65b105327306 +down_global 877ca8c4575a8c703899dc0dd4ea2432af9a5716eeb4b8fa683464d06fe17815 +``` + +## Result + +The two production kernels produced exactly identical output for the same +real checkpoint bytes and deterministic input: + +```text +max_abs_diff=0.0 +mean_abs_diff=0.0 +outputs_finite=true +``` + +Ten Marlin samples, including first-use effects, were: + +```text +[0.422324, 4.516500, 0.316868, 0.285419, 0.282369, + 0.238581, 0.134624, 0.121815, 0.124095, 0.127925] ms +``` + +The final eight steady-state samples averaged **0.203962 ms**. The eight +routed experts read approximately 12.58 MiB of packed gate/up and down input +per call, equivalent to approximately **61.7 GB/s** of routed packed input +traffic. This is a component result, not end-to-end model TPS. + +## Interpretation + +This closes the most important numerical uncertainty before a live candidate: +the production Marlin path can consume real Qwen NVFP4 checkpoint data on HIP +and match the baseline exactly for a deterministic routed layer operation. +The unusually high second sample is retained rather than discarded because +it is evidence of first-use or runtime scheduling overhead. + +The result does not yet establish a full serving improvement. It excludes the +router, attention, KV management, scheduler, host-side cache misses, API +overhead, and all other decoder layers. A candidate replacement still needs a +complete API quality and throughput gate. + +## Next action + +Run the same real-bank differential across several deterministic hidden-state +vectors and routed expert sets, then attach the candidate to an isolated Qwen +server. Compare full API prefill, decode, TTFT, tail latency, concurrency, +quality hashes, and recovery against the qualified current configuration. diff --git a/docs/gmktec-evo-x2-real-qwen-nvfp4-route-matrix.md b/docs/gmktec-evo-x2-real-qwen-nvfp4-route-matrix.md new file mode 100644 index 0000000000..cc30fbca0d --- /dev/null +++ b/docs/gmktec-evo-x2-real-qwen-nvfp4-route-matrix.md @@ -0,0 +1,42 @@ +# GMKtek EVO-X2 real Qwen NVFP4 route matrix + +## Purpose + +This follow-up expands the real layer-zero checkpoint test across three +deterministic hidden states and routed expert sets. It checks whether the +production Marlin decode path remains numerically equivalent to the retained +LUT-gather baseline when routes are contiguous, widely scattered, or repeated. + +## Result + +All three cases completed with finite output. The first two route patterns had +zero difference. The repeated-route case differed only by floating-point +reduction order: + +| Case | Route pattern | Maximum absolute difference | Mean absolute difference | Marlin mean | +| --- | --- | ---: | ---: | ---: | +| 0 | Experts 0 through 7 | 0.0 | 0.0 | 0.187887 ms | +| 1 | 8, 19, 37, 64, 91, 127, 191, 255 | 0.0 | 0.0 | 0.114801 ms | +| 2 | 3, 3, 3, 11, 42, 42, 200, 201 | 0.00000190735 | 0.00000000483124 | 0.093191 ms | + +The route matrix used the actual layer-zero packed weights, FP8 block scales, +and FP16 global scales from the Qwen3.6 NVFP4 checkpoint. Each case used a +different deterministic BF16 hidden vector and normalized routed weights. + +## Interpretation + +The result is a strong numerical qualification for the Marlin path. Scattered +expert IDs do not cause stale-bank or address-selection errors, and the only +nonzero difference is a sub-two-millionth absolute change in a duplicate-route +reduction. This is far below the production quality tolerance and is expected +from a different accumulation order. + +The test remains a layer component test. It does not establish end-to-end +model TPS or prove that a new kernel will improve the complete server. + +## Next action + +Use the qualified real-bank path in an isolated serving candidate and run the +fixed Qwen API matrix. Compare the candidate to the current production profile +for prefill, decode, TTFT, tail latency, concurrency, deterministic visible +quality, cache misses, and recovery. diff --git a/docs/gmktec-evo-x2-rocm-transfer-prototype.md b/docs/gmktec-evo-x2-rocm-transfer-prototype.md new file mode 100644 index 0000000000..e71c9139a2 --- /dev/null +++ b/docs/gmktec-evo-x2-rocm-transfer-prototype.md @@ -0,0 +1,54 @@ +# GMKtek EVO-X2 ROCm transfer prototype + +This read-only prototype measures contiguous host and device copies in the +existing FreeToken Python environment. It is a lower-bound systems datapoint +for the DeepSeek offload decision, not a model benchmark. It does not download +weights, start a model, or change the protected service. + +## Environment + +| Field | Observation | +|---|---| +| GPU | AMD Radeon 8060S Graphics | +| Architecture | gfx1151 | +| PyTorch | 2.13.0+rocm10.0.0 | +| HIP runtime | 7.15.26333 | +| Transfer size | 64 MiB per copy | +| Repetitions | 20 measured copies after 5 warmups | +| Synchronization | `torch.cuda.synchronize()` after every copy | + +## Measured copies + +| Direction | Mean time | Minimum time | Effective rate | +|---|---:|---:|---:| +| Host to device, pageable | 0.841 ms | 0.825 ms | 79.79 GB/s | +| Host to device, pinned | 0.842 ms | 0.818 ms | 79.67 GB/s | +| Device to host, pageable | 0.955 ms | 0.947 ms | 70.24 GB/s | + +Pinned memory was available, but it did not materially change the result for +this small contiguous copy. The source and destination were single contiguous +64 MiB tensors, so these values should not be interpreted as the bandwidth of +small, scattered expert-block fetches. + +## Implication for the paper model + +At the measured host-to-device contiguous-copy rate, moving the paper's stated +approximately 140 GB routed-expert volume once has an ideal transfer floor of +approximately 1.75 seconds. This is close to the paper's 80 GB/s reference +scale, but it excludes tensor slicing, page faults, format conversion, cache +miss scheduling, synchronization, and repeated decode fetches. + +The result therefore removes one uncertainty: the EVO-X2 memory fabric can +reach roughly the same raw bandwidth class as the paper's stated host-side +bandwidth examples. It does not remove the dominant capacity problem. The +official checkpoint is approximately 148.66 GiB while the live available host +memory was approximately 18 GiB, so most of the model would still need to be +reloaded or streamed repeatedly. + +## Next measurement + +The next useful prototype should use a synthetic expert-block access pattern: +many small, non-contiguous blocks with the same sizes and batching as the +runtime's miss path. It should report throughput, launch overhead, and p95/p99 +copy latency. A full checkpoint download remains gated on that result and on a +resident-memory budget that preserves the protected service. diff --git a/docs/gmktec-evo-x2-rocm-validation-2026-08-28.md b/docs/gmktec-evo-x2-rocm-validation-2026-08-28.md new file mode 100644 index 0000000000..40f8731145 --- /dev/null +++ b/docs/gmktec-evo-x2-rocm-validation-2026-08-28.md @@ -0,0 +1,1245 @@ +# GMKtek EVO-X2 native ROCm validation, 2026-08-28 + +## Result + +This validation passed the first release gate for the AMD port. FreeToken +served both required MoE models through the OpenAI-compatible API on GMKtek EVO-X2's +Radeon 8060S (`gfx1151`) using a native HIP and ROCm execution path. + +This is not a CPU fallback or a Vulkan result. The serving process uses the +ROCm PyTorch wheel, HIP-compiled native extensions, and Triton GPU kernels. +CUDA graphs were deliberately disabled for this validation because the MVP +needs correctness before graph capture tuning. + +## Reproducibility record + +| Item | Value | +| --- | --- | +| Host | GMKtek EVO-X2, `GMKtek EVO-X2` | +| GPU | AMD Radeon 8060S Graphics, `gfx1151`, 40 CUs | +| System ROCm installation | ROCm 10.0 at `/opt/rocm-10.0` | +| PyTorch wheel | `2.13.0+rocm10.0.0` | +| HIP reported by PyTorch | `7.15.26333` | +| FreeToken branch | `amd-rocm-gfx1151` | +| Validation commit | `065d806` | +| API exposure | loopback-only ports, not llama-swap | + +The isolated validation layout was `/home/operator/freetoken-amd/`; no existing +llama-swap service, model configuration, or production endpoint was changed. + +## Models and API evidence + +| Model | Source revision | Backend selection | Non-streaming result | Streaming result | +| --- | --- | --- | --- | --- | +| `nvidia/Qwen3.6-35B-A3B-NVFP4` | vendor model snapshot used for this run | Triton attention, MoE offload, native Triton NVFP4, serial expert load | HTTP 200, `AMD ROCm FreeToken ready.` in 1.54 s | HTTP 200, SSE chunks and `[DONE]` | +| `google/gemma-4-26B-A4B-it-qat-q4_0-gguf` | `d1c082be9cf3c8a514acf63b8761f4b41935842e` | Triton attention, MoE offload, serial expert load, HIP GGUF JIT | HTTP 200, `native hip api works` in 341.304 ms | HTTP 200, SSE chunks and `[DONE]` | + +Raw evidence remains on GMKtek EVO-X2 in these isolated artifact directories: + +```text +/home/operator/freetoken-amd/artifacts/qwen36-nvfp4-serial-hip-prefill/ +/home/operator/freetoken-amd/artifacts/gemma4-q4-rocm-thrust-system/ +``` + +The Gemma telemetry captured immediately after the API tests identified the +same `gfx1151` device, 33 percent GPU utilization, 46 percent allocated VRAM, +and a 40 C edge temperature. The model uses the APU's shared-memory design; +the tool's VRAM label is therefore only its standard telemetry label. + +## Warm single-request throughput + +The following measurements use one fixed 733-token prompt, greedy sampling, +and a one-sentence answer that produced 26 completion tokens. `TTFT` is the +client-observed time to the first non-empty SSE text chunk. Prompt throughput +is the end-to-end prompt-token count divided by TTFT, so it includes normal +API and scheduler overhead. Output throughput is completion tokens divided +by the interval from that first chunk through `[DONE]`. + +| Model | Prompt tokens | Completion tokens | TTFT | Prompt TPS | Generation interval | Output TPS | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Qwen3.6-35B-A3B NVFP4 | 733 | 26 | 4.976 s | 147.3 | 0.899 s | 28.9 | +| Gemma 4 26B A4B Q4_0 GGUF | 733 | 26 | 3.244 s | 226.0 | 0.581 s | 44.8 | + +These are warm, single-request measurements, not concurrency or maximum +throughput claims. The Qwen configuration uses the native Triton serial +NVFP4 prefill route selected for ROCm correctness. Its approximately +seven-minute cold initialization is expert-bank preparation and cache +allocation, not inference time. + +## Same-model llama.cpp Vulkan comparison + +To compare the usable Strix Halo serving baseline rather than an unrelated +model, the exact Gemma GGUF was served by llama.cpp Vulkan build `b10141` +(`0d47ea742`) on a separate loopback port. Both servers used one slot, +8,192-token context, greedy sampling, and the same repeated scheduler prompt. +The model SHA-256 was +`3eca3b8f6d7baf218a7dd6bba5fb59a56ee25fe2d567b6f5f589b4f697eca51d`. + +| Runtime | GPU backend | Prompt tokens | Completion tokens | TTFT | Client prompt TPS | Client output TPS | Runtime prompt TPS | Runtime output TPS | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| FreeToken | ROCm/HIP | 733 | 26 | 3.244 s | 226.0 | 44.8 | not exposed | not exposed | +| llama.cpp `b10141` | Vulkan | 758, 7 template tokens cached | 128 | 0.855 s | 886.7 | 63.2 | 1,078.4 | 61.7 | + +For this isolated, single-request Gemma workload, llama.cpp Vulkan reached +first output about 3.8 times sooner, delivered about 3.9 times the +client-observed prompt rate, and delivered about 1.4 times the client-observed +generation rate. llama.cpp's internal timing excludes ordinary API and +scheduler overhead, so its 1,078.4 prompt TPS and 61.7 output TPS must not be +compared directly with FreeToken's client-observed rates. + +The completion lengths differ because llama.cpp exposed Gemma's reasoning +stream and consumed the 128-token cap, whereas FreeToken's parser emitted the +final concise answer and stopped at 26 tokens. That makes the output-rate +comparison useful as a warm streaming rate, but not a quality or exact +end-to-end task comparison. The raw llama.cpp evidence is retained under +`/home/operator/freetoken-amd/artifacts/llamacpp-vulkan-gemma4-q4-tps/` on +GMKtek EVO-X2. + +## Same-model ROCm 10 and HIP comparison + +The Vulkan baseline above answers a practical deployment question, but it is +not a backend-for-backend comparison. This follow-up rebuilt the same +llama.cpp source revision, `b10141` (`0d47ea742`), with HIP for `gfx1151` and +ran it under the same ROCm 10 installation used by FreeToken. The compiler +was ROCm 10 HIP `7.15.26333` with AMD Clang 23.0.0. At runtime, llama.cpp's +`libamdhip64`, `libhipblas`, `librocblas`, `libamd_comgr`, and HSA runtime +libraries all resolved from `/opt/rocm-10.0`, not the older ROCm installation. + +Both runners used the identical 14 GB Gemma 4 26B A4B Q4_0 GGUF, SHA-256 +`3eca3b8f6d7baf218a7dd6bba5fb59a56ee25fe2d567b6f5f589b4f697eca51d`, one +request at a time, an 8,192-token context, greedy sampling, `max_tokens: 128`, +and a 48-times repeated scheduler prompt. Each measurement used a distinct +nonce, preventing prompt-cache reuse. The token totals differ by one because +the two runners tokenize and render Gemma's chat template differently. + +| Runtime | HIP and ROCm stack | Prompt tokens | Completion tokens | TTFT | Client prompt TPS | Client output TPS | Runtime prompt TPS | Runtime output TPS | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| FreeToken, steady state | PyTorch `2.13.0+rocm10.0.0`, HIP `7.15.26333`, native HIP GGUF extension | 772 | 20 | 2.863 s | 269.6 | 46.1 | not exposed | not exposed | +| llama.cpp `b10141` | ROCm 10 HIP, `gfx1151` | 771 | 128 | 0.850 s | 906.6 | 58.3 | 1,011.6 | 56.2 | + +On this uncached, single-request workload, llama.cpp ROCm 10 reached first +text about 3.4 times sooner, supplied about 3.4 times the client-observed +prompt rate, and supplied about 1.3 times the client-observed output rate. +llama.cpp's internal numbers exclude HTTP, SSE, and scheduling overhead and +therefore are only comparable to another internal timing source, not directly +to FreeToken's client values. + +The FreeToken request that triggered a fresh GGUF HIP extension build is kept +as a separate cold-start measurement: 768 prompt tokens, 21 completion tokens, +109.938 s TTFT, 6.99 client prompt TPS, and 27.47 client output TPS. It +contains HIP compilation and must not be presented as inference throughput. +The subsequent steady-state run above was made after the extension completed, +using a fresh nonce and no prompt cache hit. FreeToken's extension compiler +was `/opt/rocm-10.0/bin/hipcc` targeting `gfx1151`, and its runtime libraries +came from the ROCm 10 PyTorch SDK packages. Its existing JIT command also +passed `/opt/rocm-7.2.4/include` as a supplemental include path. That does not +change the ROCm 10 compiler or loaded runtime libraries, but it prevents this +FreeToken build from being described as a strictly ROCm 10-only header build. + +The llama.cpp response used all 128 allowed tokens because it exposed Gemma +reasoning text. FreeToken stopped after a concise 20-token answer. This +makes the output-rate comparison a useful streaming measurement, but it is +not an exact answer-quality or equal-completion-length evaluation. + +Raw artifacts are retained only on GMKtek EVO-X2: + +```text +/home/operator/freetoken-amd/artifacts/llamacpp-rocm10-gemma4-q4-tps/ +/home/operator/freetoken-amd/artifacts/freetoken-rocm10-gemma4-q4-tps/ +``` + +## AMD TPS optimization campaign + +The first configuration optimization pass used the same warm AIME-25 problem +and a 128-token greedy completion for both FreeToken and the ROCm 10 HIP build +of llama.cpp `b10141`. Each runner received the identical user message, used +a warm identical request before the measured request, and ran one stream at a +time. Both rendered 63 prompt tokens; FreeToken's measured request reused 62 +prompt tokens and llama.cpp's reused 58. + +| Runtime and candidate | Decode TPS | TTFT | Result | +| --- | ---: | ---: | --- | +| FreeToken, offload, eager | 54.89 | 267.9 ms | Baseline | +| FreeToken, offload, HIP graph capture at batch size 1 | 55.73 | 259.3 ms | Best observed safe configuration | +| FreeToken, HIP graph plus experimental `-ffast-math` GGUF extension | 55.65 | 261.6 ms | Rejected: no gain, despite matching output hash | +| FreeToken, final target-specific `gfx1151` GGUF extension plus graph capture | 55.44 | 263.6 ms | Validated shipping configuration; normal run-to-run variation | +| FreeToken, experimental two-row Q4_0 MoE block | 55.30 | 291.6 ms | Rejected: slower with identical output hash | +| FreeToken, experimental Q4_0 MoE two-block residency hint | 55.08 | 294.9 ms | Rejected: slower with identical output hash | +| FreeToken, HIP Q4_0 MoE one-wave/two-row specialization | 55.89 median, 55.91 mean | 262.2 ms mean | Accepted: five independent API runs, identical output hash | +| FreeToken, full 4,096-slot expert cache and pinned 8,320-token KV pool | 60.11 median, 58.61 mean | 260.3 ms mean | Accepted configuration; four of five runs at 60.06 to 60.20 TPS, one host-contention outlier at 52.50 TPS | +| llama.cpp `b10141`, ROCm 10 HIP, earlier matched reference | 60.42 client, 58.88 internal | 128.6 ms | Historical reference | +| llama.cpp `b10141`, ROCm 10 HIP, current-host five-run control | 62.44 median, 62.13 mean client TPS | 111.5 ms mean | Same prompt, greedy decode, five fresh servers, requested 8,320-token context | + +The graph configuration removes approximately 1.5 percent of the eager decode +cost. The capacity-aware resident-expert configuration below then removes the +dominant configuration gap without changing the model, server API, or HIP +kernel arithmetic. Its uncontended median is within 0.51 percent of the +60.42 client-TPS llama.cpp reference, but its five-run arithmetic mean remains +below that reference because one run experienced external host stalls. The +criterion of meeting or exceeding llama.cpp is therefore not yet claimed as a +fully repeatable mean result. + +### Accepted full-expert-cache and fixed-KV configuration + +The original automatic offload configuration sized 3,840 GPU expert slots and +then assigned the remaining memory budget to a very large KV pool. That pool +is not required by the fixed 8,320-token operating target and lowered the +observed decode rate. A fixed expert-cache configuration leaves the same +native Q4_0 GGUF, HIP extension, graph-captured decode, OpenAI-compatible API, +and `offload` backend intact while making the capacity choices explicit: + +```bash +python benchmarks/bench_decode_moe.py \ + --model /home/operator/freetoken-amd/models/Gemma-4-26B-A4B-it-qat-q4_0-gguf/gemma-4-26B_q4_0-it.gguf \ + --backend offload --cache 4096 --num-token-override 8320 \ + --mem-ratio 0.50 --decode 128 --greedy +``` + +`4096` is the complete 32-layer by 128-expert cache domain. A 3,840-slot +control preserved the fixed KV allocation but produced two severe decode-tail +events, confirming that leaving any of the 4,096 slots uncached can still +exercise the miss path. The explicit 4,096-slot configuration was therefore +retained. The new benchmark option maps `--num-token-override` to the public +server flag `--num-tokens`, so experiments can pin KV capacity without a +private wrapper. + +Five independent API runs used the fixed 63-token AIME request, 126 measured +decode steps, greedy sampling, `0.50` memory ratio, and the deterministic +output SHA-1 `abeee5e73e89`: + +| Run | Decode TPS | ms/token | TTFT | Event p50 / p99 | +| --- | ---: | ---: | ---: | --- | +| 1 | 60.063 | 16.649 | 259.7 ms | 16.929 / 17.841 ms | +| 2 | 52.499 | 19.048 | 261.4 ms | 16.928 / 120.541 ms | +| 3 | 60.203 | 16.610 | 259.6 ms | 16.855 / 17.657 ms | +| 4 | 60.114 | 16.635 | 260.3 ms | 16.937 / 17.619 ms | +| 5 | 60.183 | 16.616 | 260.6 ms | 16.876 / 17.522 ms | +| Aggregate | **58.613 mean, 60.114 median** | 17.112 mean | 260.3 ms mean | 16.928 / 17.657 ms median | + +The four normal runs are within 60.063 to 60.203 TPS and have p99 latency at +or below 17.841 ms. The one low-throughput run kept the same output, VRAM, +TTFT, and p50 latency, but had isolated 120.541 ms decode events. Kernel logs +recorded `kfd_process_wq_release` holding CPU for more than 10 ms and the host +showed full I/O pressure. Read-only inspection also found two long-running, +blocked user-owned filesystem scans. They were not stopped by this campaign. +This is host contention evidence, not a FreeToken numerical or API failure. + +Capacity was tested through the public OpenAI-compatible API, not merely at +startup. A request with 7,619 prompt tokens plus one completion token ran +inside the pinned 8,320-token pool, returned exactly `OK`, and completed in +22.352 seconds. The server then exited cleanly with no KFD processes. + +The retained raw evidence is: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/full-expert-cache-4096-20260829T010740Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/fixed-expert-cache-3840-control-20260829T011537Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/full-cache-4096-context8320-20260829T012342Z/ +``` + +### Current-host ROCm llama.cpp control + +The historical llama.cpp reference was useful for identifying the original +gap, but it was not collected alongside the accepted 4,096-slot FreeToken +configuration. A new five-run control was therefore run immediately after +that configuration investigation, without changing GMKtek EVO-X2, stopping any +user process, or enabling a production service. Each trial launched a fresh +`llama-server` from the ROCm 10 `b10141` build with all layers on `gfx1151`, +Flash Attention enabled, one parallel slot, and `-c 8320`. The server reports +an 8,448-token slot after its own request reserve is added. This is a llama.cpp +internal allocation detail; the requested application context target was +8,320 tokens in both runners. + +Both runners used the same cached AIME-25 problem 0, a warmed streamed +OpenAI-compatible `/v1/chat/completions` request, greedy sampling, and a +128-token completion. The metric in this table is client-observed decode +throughput: `(completion_tokens - 1)` divided by elapsed time from the first +to last SSE token event. It includes HTTP and SSE delivery for both runners. + +| Runtime | Five client decode TPS | Mean | Median | Mean TTFT | p99 event gap median | +| --- | --- | ---: | ---: | ---: | ---: | +| FreeToken, 4,096 experts, 8,320-token KV pool | 60.06, 52.50, 60.20, 60.11, 60.18 | 58.61 | 60.11 | 260.3 ms | 17.66 ms, excluding the host-stalled run 120.54 ms | +| llama.cpp `b10141`, ROCm 10 HIP | 61.04, 62.03, 62.44, 62.57, 62.56 | 62.13 | 62.44 | 111.5 ms | 16.63 ms | + +llama.cpp leads FreeToken by 3.7 percent on median client decode TPS +(`62.44 / 60.11 - 1`) and 5.7 percent on the unfiltered five-run mean +(`62.13 / 58.61 - 1`). It also has lower warm TTFT. FreeToken produced the +same deterministic output hash in every measured run; llama.cpp produced the +same deterministic output hash in every one of its own runs. The hashes are +not compared across runtimes because their tokenizers and chat-template +implementations differ. + +This is a close result for decode rate, but it does **not** meet the stated +criterion of meeting or exceeding llama.cpp. The remaining performance work +is therefore directed at the HIP decode path and the source of the FreeToken +tail stall, rather than a claim of parity. The raw llama.cpp evidence is +retained on GMKtek EVO-X2 at: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/llamacpp-current-host-context8320-20260829T013730Z/ +``` + +### Current-upstream rebase and full API revalidation + +After the comparison, upstream `main` advanced from `9ef3651` to `a05c265` +with Qwen 3.8 support and engine or cache changes. The AMD branch was rebased +onto that current upstream revision without a conflict, rather than leaving a +performance result attached to an obsolete upstream base. The rebased branch +was then installed into the isolated GMKtek EVO-X2 virtual environment so its native +HIP pinned-memory extension was built from the rebased source. The source +checkout used for that validation was deliberately separate from the earlier +test checkout, preventing an uncommitted working-tree change from becoming +test evidence. + +The first complete Gemma launch from the rebased checkout rebuilt the target- +specific GGUF HIP extension and matching graph helper because the source path +is part of their cache identity. The build used ROCm 10 `hipcc`, `-O3`, and +`--offload-arch=gfx1151`; a later process restart can reuse that cache. The +server then completed its normal graph capture, exposed `/v1/models`, and +served two streamed OpenAI-compatible chat completions before clean shutdown. + +| Check | Observed value | +| --- | --- | +| Upstream revision in branch history | `a05c265` | +| HIP build and ROCm-runtime tests | 4 passed | +| MoE configuration | `offload`, 4,096 expert slots, 8,320 KV tokens, graph batch size 1 | +| Warm streamed API decode | 60.07 client TPS, 16.648 ms/token | +| Warm TTFT | 258.2 ms | +| Prompt and completion tokens | 63 and 127, respectively | +| Output SHA-1 | `abeee5e73e89` | +| Server VRAM | 15.66 GiB | +| Post-run process state | Server shut down; no serving process remained | + +The response ended at 127 tokens despite the requested 128-token limit, so +the benchmark harness emitted its explicit token-count warning. The request +was otherwise successful, deterministic, and had the expected response hash. +This one-run revalidation is evidence that rebasing did not break native HIP +serving. It is intentionally not folded into the five-run performance score. +Its raw logs and result are retained at: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/rebased-current-main-api-retry-20260829T014741Z/ +``` + +### Rebased Qwen3.6 NVFP4 API revalidation + +The other MoE model retained in the isolated FreeToken inventory is +`Qwen3.6-35B-A3B-NVFP4`. It was revalidated after the upstream rebase through +the same loopback OpenAI-compatible streaming API, using the native Triton +NVFP4 expert path, graph batch size 1, automatic expert-cache sizing, a 0.35 +memory ratio, and greedy 128-token AIME decoding. This exercised its complete +21.8 GiB parallel expert-bank load, cache allocation, graph capture, warm +request, measured request, and cleanup. + +| Check | Observed value | +| --- | --- | +| Resolved expert cache | 9,499 slots and 8,255 KV tokens | +| Warm streamed API decode | 28.93 client TPS, 34.560 ms/token | +| Warm TTFT | 404.7 ms | +| Prompt and completion tokens | 54 and 127, respectively | +| Output SHA-1 | `0acef4eab6f4` | +| Server VRAM | 19.12 GiB | +| Post-run process state | Server shut down; no serving process remained | + +As with the rebased Gemma validation, the response ended at 127 tokens and the +harness recorded its explicit limit-warning rather than silently treating it as +a 128-token result. The API transaction and deterministic response succeeded. +The server also reported that `triton_kernels` was absent and selected the +numerically equivalent pure-PyTorch router fallback. That is a documented +performance limitation, not a functional failure. A native ROCm-compatible +fused-router installation must be independently verified before it can be +considered an optimization. + +The raw evidence is retained at: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/rebased-current-main-qwen36-api-20260829T015240Z/ +``` + +### Rejected ROCm vendored-Triton router candidate + +The Qwen revalidation exposed a pure-PyTorch router fallback because OpenAI's +`triton_kernels` package contains CUDA-only binaries. Current upstream also +contains an in-tree Triton router, so it was evaluated as a ROCm-only candidate +before any production use. On the actual Radeon 8060S it selected the same +expert set as PyTorch; BF16 equal-logit ties can have a different internal +ordering, while FP32 indices matched exactly. Its selected routing weights +matched PyTorch within `2.98e-8`, and the isolated one-token, 128-expert, +top-8 router time improved from 21.14 us to 14.96 us. + +That microbenchmark improvement was insufficient. The complete Qwen API run +with the candidate reached 30.26 client TPS, but its deterministic greedy +response SHA-1 was `cd580f4978fb`, not the reference `0acef4eab6f4`. Small +router differences therefore accumulated into a different generated response. +The candidate was reverted and ROCm continues to use the reference PyTorch +router. This keeps quality behavior stable even though the fused alternative +is faster in isolation. The fallback warning now explicitly distinguishes +intentional ROCm behavior from a missing CUDA Linux package. + +The rejected candidate evidence is retained at: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/qwen36-vendored-router-api-20260829T015937Z/ +``` + +The restored branch was then revalidated through the full Qwen API path. It +returned to the reference SHA-1 `0acef4eab6f4` at 28.96 client TPS, with the +same 19.12 GiB VRAM use and clean shutdown. Its 1,272.2 ms warm TTFT is not a +performance regression claim: the model's 21.8 GiB expert-bank load was +concurrently slowed by the documented host I/O pressure, taking 3 minutes and +37 seconds instead of about 2 minutes. The final exact-path artifact is: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/qwen36-router-revert-api-20260829T020626Z/ +``` + +### Accepted HIP Q4_0 one-wave/two-row MoE specialization + +The first two-row experiment did not reproduce llama.cpp's execution shape: it +used two independent 32-thread waves. Commit `d1de602` instead adds a ROCm-only +Q4_0 kernel in which one 32-thread wave accumulates two adjacent output rows. +It preserves FreeToken's flattened token/top-k route IDs, packed expert-bank +layout, Q8_1 activation layout, and BF16 public output contract. CUDA retains +the established generic path. + +The dedicated GMKtek EVO-X2 microbenchmark uses the verified Gemma 4 26B A4B Q4_0 +geometry: 128 experts, top-k 8, hidden width 2816, intermediate width 704, and +one decode token. Five runs with 2,000 timed calls each measured a 73.509 us +baseline median for the gate/up plus down pair and a 64.340 us candidate median, +a 12.5 percent kernel-pair reduction. ROCprof recorded a 32-thread wave, zero +LDS and scratch allocation, and half the former row-block grid. The compiler +still allocated 48 VGPRs, so future work must target register pressure +separately rather than claiming it was resolved by this change. + +The end-to-end gate was five independent loopback OpenAI-compatible API server +runs, each using the exact Gemma GGUF SHA-256, offload backend, 0.50 memory +ratio, HIP graph capture, greedy AIME-25 problem 0, and 128-token decode +procedure. All five emitted the original deterministic output SHA-1 +`abeee5e73e89` and retained 27.52 GiB server-reported VRAM use. + +| Metric | Five-run result | +| --- | --- | +| Decode TPS | 55.713 to 56.071 | +| Decode TPS median / mean | **55.894 / 55.905** | +| Decode ms/token median / mean | **17.891 / 17.887** | +| TTFT mean | 262.2 ms | +| Output SHA-1 | `abeee5e73e89` in every run | +| Compared shipping configuration | 55.44 TPS single verified run | +| Matched llama.cpp ROCm 10 reference | 60.42 client TPS | + +The candidate is accepted because it produces a repeatable FreeToken gain of +approximately 0.8 percent over the prior shipping result while preserving the +observable API result. It remains approximately 7.5 percent below the +matched llama.cpp client-TPS reference, so it is an incremental port +improvement rather than completion of the performance objective. + +Artifacts are retained on GMKtek EVO-X2: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/q4-moe-microbench-20260828T231332Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/q4-moe-two-row-wave-20260828T231950Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/q4-moe-two-row-wave-20260828T231950Z/api-repeats-20260828T232646Z/ +``` + +### Rejected two-row MoE Q8 activation-reuse candidate + +The accepted HIP Q4_0 one-wave/two-row MoE kernel computes two adjacent output +rows from the same Q8_1 activation block. The generic dot helper loads the +four packed Q8 activation words independently for each row. Commit `684148d` +tested a ROCm-only helper that loads those four words once and supplies them to +both row dot products, while retaining the same Q4 nibble order, DP4A order, +scales, BF16 public-output contract, and all CUDA code. + +The shape-accurate Gemma routed-expert microbenchmark improved from about +64.34 us to **61.54 us per gate/up plus down pair**. That local result did not +translate to a material full-server result. Five independent OpenAI-compatible +API runs, each using the fixed 63-token prompt and 126 measured decode steps, +all returned greedy output SHA-1 `abeee5e73e89`: + +| Run | Decode TPS | ms/token | TTFT | Event p50 / p99 | +| --- | ---: | ---: | ---: | --- | +| 1 | 55.993 | 17.859 | 264.2 ms | 18.133 / 18.753 ms | +| 2 | 55.970 | 17.867 | 262.1 ms | 18.171 / 18.853 ms | +| 3 | 55.958 | 17.871 | 259.9 ms | 18.183 / 18.823 ms | +| 4 | 56.012 | 17.853 | 260.8 ms | 18.086 / 18.990 ms | +| 5 | 56.155 | 17.808 | 262.9 ms | 18.018 / 18.853 ms | +| Aggregate | **56.018 mean, 55.993 median, 0.080 stddev** | 17.851 mean | 262.0 ms mean | 18.133 / 18.853 ms median | + +This is only 0.20 percent above the accepted 55.905 TPS mean, materially below +the campaign's repeatable-improvement threshold and far below the 60.42 client +TPS matched llama.cpp ROCm 10 reference. The candidate was therefore reverted +in `a237b12`; the accepted one-wave/two-row implementation remains active. +The benchmark sequence also ended with no KFD GPU processes, confirming that +the service was torn down cleanly. + +The retained raw evidence is: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/moe-q8-reuse-20260829T005332Z/microbench.json +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/moe-q8-reuse-20260829T005332Z/api-first.jsonl +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/moe-q8-reuse-20260829T005332Z/api-repeats.jsonl +``` + +### Rejected dense Q4_0 one-wave/two-row specialization + +The dense Q4_0 vector path uses the same older one-row scheduling structure as +the routed-expert path. Commit `b4a53d1` applied the accepted one-wave/two-row +pattern to that dense kernel, while leaving CUDA unchanged. A new shape-aware +microbenchmark covered the exact Gemma projection dimensions recovered from the +GGUF: 2816x4096, 8192x2816, 4224x2816, and 10240x2816. In isolation it reduced +the measured GPU event time for every shape, including 10240x2816 from 41.34 us +to 28.12 us. + +That synthetic gain did not survive the real graph-captured serving path. The +fixed loopback API workload compiled the candidate from a fresh HIP extension +cache, returned the exact deterministic output SHA-1 `abeee5e73e89`, and used +the same 27.52 GiB of server-reported VRAM, but measured only **54.71 TPS** or +18.277 ms/token. This is below the 55.89 TPS accepted MoE-specialization +median and below the prior 55.04 TPS repaired baseline. The dense candidate +was therefore reverted. It proves that isolated event timing alone is not an +acceptance metric for graph-captured end-to-end decode. + +The retained raw evidence is: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-microbench-20260828T233506Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-two-row-wave-20260828T233930Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-two-row-wave-api-20260828T234147Z/ +``` + +### Rejected dense Q4_0 FP32-output hypothesis + +llama.cpp's corresponding vector kernel stores FP32 values, whereas the +FreeToken GGUF adapter normally returns the input dtype, BF16 for this Gemma +run. That difference was a plausible explanation for the profiler contrast: +FreeToken's generic Q4_0 dense kernel reported 48 architectural VGPRs and the +llama.cpp reference reported 24. Commit `e77a44b` added a deliberately +benchmark-only Q4_0 flag that changed only the destination tensor to FP32. +It was never wired to the GGUF model layers, and a source guard ensured the +normal serving call retained its BF16 contract. + +The result rejects that explanation. In the first independent event run, the +four exact Gemma projection geometries measured 18.28 us, 33.59 us, 18.16 us, +and 35.98 us respectively. The profiler trace showed the FP32 specialization +still at **48 VGPRs**, 128 SGPRs, no LDS, and no scratch. It therefore did not +match llama.cpp's 24-VGPR code shape. Its profile-run event values also showed +no consistent gain. The experiment was reverted in `203062f`; no public or +model-serving API changed. + +The retained raw evidence is: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-fp32-output-20260828T234953Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-fp32-output-rocprof-20260828T235258Z/ +``` + +### Rejected dense HIP launch-bound candidate + +Commit `c7009a9` tested the other conspicuous structural difference from the +matched llama.cpp Q4_0 vector kernel: a HIP-only `__launch_bounds__(32, 1)` +constraint for FreeToken's one-wave dense GEMV. CUDA was unchanged. The +candidate compiled cleanly for `gfx1151` and kept the same 48 VGPRs, 128 +SGPRs, zero LDS, and zero scratch as the generic FreeToken kernel. It did +improve three of the four isolated Gemma projection measurements, but did not +reduce the compiler resource gap against llama.cpp. + +Five independent graph-captured loopback API runs produced **55.90 TPS mean** +and **55.88 TPS median**, with deterministic output SHA-1 `abeee5e73e89` in +every run. The accepted MoE-only specialization measured 55.91 TPS mean and +55.89 TPS median under the same workload. The candidate therefore has no +meaningful decode gain and its 274.7 ms mean TTFT was worse than the accepted +candidate's 262.2 ms mean. It was reverted in `1d555e3`; the upstream-ready +path remains unchanged by this experiment. + +The retained raw evidence is: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-launch-bounds-20260828T235459Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-launch-bounds-rocprof-20260828T235726Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-launch-bounds-api-20260828T235756Z/ +``` + +### Rejected indexed Q4_0 dense HIP kernel + +The dominant llama.cpp Q4_0 trace was rechecked before this experiment. Its +main kernel uses the same 32-thread by 1-row workgroup as FreeToken, but +reports 24 VGPRs versus FreeToken's 48. Commit `a5e04d1` isolated the remaining +source-level difference: a Q4_0-only HIP kernel that keeps the base weight +pointer and block index separate until the vector-dot helper. It preserved +the generic FreeToken launch geometry and left CUDA and every non-Q4_0 type +unchanged. + +The isolated evidence was favorable but insufficient. The four exact Gemma +dense projections measured 16.49 us, 28.69 us, 17.83 us, and 35.52 us, and the +profile trace measured 18.16 us, 22.88 us, 13.17 us, and 29.22 us. The compiler +still used 48 VGPRs, 128 SGPRs, no LDS, and no scratch. The first full +graph-captured API run preserved the deterministic output SHA-1 +`abeee5e73e89`, but collapsed to **15.21 TPS**, 65.73 ms/token, 3014.5 ms TTFT, +and 1674.5 ms p99 event latency. This is a functional result but a clear +performance failure. It was reverted in `b281e0e` and must not be retried +without an explanation for the end-to-end stalls. + +The retained raw evidence is: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-indexed-pointer-20260829T000901Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-indexed-pointer-rocprof-20260829T001126Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-indexed-pointer-api-20260829T001156Z/ +``` + +### Rejected scalarized dense Q4_0 dot-product candidate + +The source and trace audit found that the historical FreeToken dense Q4_0 +helper materializes two packed Q4 words and four Q8 words in short local +arrays before issuing four DP4A operations. llama.cpp's newer HIP path does +not share FreeToken's old wrapper structure, so a HIP-only candidate replaced +only that dense helper with named scalar values. It retained the original +packed GGUF layout, four DP4A operations in the same order, scale formula, and +BF16 output contract. CUDA and the separately accepted routed-MoE kernel were +unchanged. + +The candidate compiled for `gfx1151`, passed the targeted HIP build and +attention tests, and produced finite results for all four exact Gemma dense +projection shapes. Its isolated event times were 17.38 us for 2816x4096, +28.38 us for 8192x2816, 19.20 us for 4224x2816, and 35.84 us for 10240x2816. +That showed useful synthetic movement, especially for the second shape, but +was not enough to accept it. + +Five independent graph-captured API runs all returned the deterministic +SHA-1 `abeee5e73e89`, retained 27.52 GiB server-reported VRAM, and left no KFD +process after shutdown. Their TPS range was 55.812 to 56.155, with **55.946 +TPS mean** and **55.919 TPS median**. Those figures differ from the accepted +Q4_0 MoE baseline by only 0.041 TPS mean and 0.025 TPS median, while mean TTFT +increased from 262.2 ms to 269.7 ms. This is normal run-to-run noise, not a +repeatable end-to-end improvement, so it was reverted in `d9ce2c5`. + +The retained raw evidence is: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-scalarized-20260829T003720Z/microbench.json +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-scalarized-20260829T003720Z/microbench-rocprof.json +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-scalarized-20260829T003720Z/api-first.jsonl +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-scalarized-20260829T003720Z/api-repeats.jsonl +``` + +### Rejected Q4_0 MoE route-grouping candidate + +The source comparison showed that llama.cpp places the eight routed experts in +separate waves of one multi-wave MoE workgroup, while FreeToken's accepted HIP +specialization uses one workgroup per route. Commit `dc73e8e` tested that +topology directly: a HIP-only Q4_0 kernel with eight independent 32-lane route +waves in a 256-thread workgroup, retaining the accepted two-output-row +arithmetic inside each wave. CUDA and all non-Q4_0 formats remained unchanged. + +The candidate compiled for `gfx1151` and passed the targeted HIP build tests, +but failed the shape-accurate microbenchmark gate. For Gemma's eight-route +decode geometry it measured 34.93 us gate/up plus 30.69 us down, or **65.62 us +per pair**, versus the accepted two-row kernel's 64.25 us mean pair time. Since +the grouped workgroup was slower before the API workload, no server benchmark +was run. It was reverted in `96c51f9` and the accepted one-wave/two-row MoE +kernel remains active. + +The retained raw evidence is: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/q4-moe-route-group8-20260829T001802Z/ +``` + +### Rejected Triton GQA attention eight-warp candidate + +Gemma's sliding decode attention has 16 query heads, 8 KV heads, a 256-wide +head dimension, and a 1,024-token sliding window. The HIP production path +uses a 16-head padded tile, 32-token KV blocks, and four Triton warps. A +shape-accurate benchmark tested head tiles of 2, 4, and 8, a 64-token KV +block, and two or eight warps. All tile and 64-token-block alternatives were +slower. Eight warps was faster in isolation: 39.51 us versus 41.87 us for +sliding attention, and 49.84 us versus 93.17 us for Gemma's 2-KV-head, +512-wide full-attention geometry. + +That microbenchmark win did not survive the full serving workload. An +otherwise identical graph-captured loopback OpenAI-compatible API run with +eight warps returned the expected deterministic SHA-1 `abeee5e73e89`, but +measured **53.76 TPS**, 18.600 ms/token, 281.2 ms TTFT, and 20.227 ms p99 +event latency. This is below the accepted five-run 55.91 TPS mean. The +production override was removed, so normal HIP serving remains at four warps; +the benchmark-only probe parameters remain available for future controlled +research. This result is a second independent example of why isolated GPU +event timings cannot be used as a serving-performance acceptance criterion. + +The retained raw evidence is: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/gqa-attention-blockh2-20260829T002315Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/gqa-attention-blockh4-8-20260829T002334Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/gqa-attention-blockn64-20260829T002442Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/gqa-attention-warps2-8-20260829T002459Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/gqa-attention-global-warps8-20260829T002558Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/attention-warps8-api-20260829T002618Z/ +``` + +The best verified FreeToken command shape is: + +```bash +export ROCM_PATH=/opt/rocm-10.0 +export HIP_PATH=/opt/rocm-10.0 +export TORCH_EXTENSIONS_DIR=/home/operator/freetoken-amd/cache/torch_extensions + +ft serve --model-path /home/operator/freetoken-amd/models/Gemma-4-26B-A4B-it-qat-q4_0-gguf/gemma-4-26B_q4_0-it.gguf \ + --attention-backend triton --moe-backend offload --moe-cache-size 4096 \ + --num-tokens 8320 --memory-ratio 0.50 --max-running-requests 1 \ + --max-seq-len-override 8320 --cuda-graph-max-bs 1 +``` + +The port now derives and exports `PYTORCH_ROCM_ARCH=gfx1151` before the GGUF +extension is compiled when the operator did not set an explicit architecture. +This avoids compiling for unnecessary visible targets and makes the extension +cache target-specific. It does not itself increase steady-state TPS because +the original HIP build already selected `gfx1151` on this single-GPU host. + +The remaining gap is not an untested cache or residency setting: Gemma's GGUF +adapter only supports the native Q4_0 offload implementation, and the accepted +configuration keeps all 4,096 routed-expert slots resident while retaining a +verified 8,320-token KV pool. Closing the gap requires a profile-guided +improvement to the HIP GGUF decode kernels or another proven ROCm attention or +quantized-linear implementation. + +The initial direct `rocprofv3` attempts did fail because the host profiler +injected a second LLVM and rocprofiler SDK beside the SDK bundled with the +PyTorch ROCm wheel. That historical failure is retained in +`rocprof-gfx1151*/` and `rocprof-launch-gfx1151-v2/` under the raw artifact +directory. It was subsequently repaired by +[`scripts/gmk-evo-x2-rocprof-wheel-sdk.sh`](../scripts/gmk-evo-x2-rocprof-wheel-sdk.sh), +which directs the host profiler front end to the wheel's matching SDK. The +repaired launch produced FreeToken kernel traces, including the active +`moe_vec_q4_0_hip_two_rows` kernel. Traces are diagnostic evidence only and +are never used as TPS scoring because profiling changes execution timing. + +### Current-source trace and rejected RDNA4 dense Q4_0 eight-wave candidate + +After an unprofiled final-source warm run rebuilt the path-specific native HIP +extension, the complete loopback API workload returned the established greedy +Gemma SHA-1 `abeee5e73e89` at **60.16 TPS**, 16.623 ms per token, 259.0 ms +TTFT, 16.877 ms p50 event latency, and 17.931 ms p99 event latency. It kept +the full 4,096-slot expert cache and 8,320-token KV budget. This is the +current unprofiled checkpoint for the accepted source path. + +The repaired profiler wrapper then traced that already-built final source. +The trace also preserved the output SHA-1, but measured 41.41 TPS and a 216.2 +ms p99 because tracing changes dispatch timing. It is not a performance +result. Its kernel statistics do identify the next work order: routed Q4_0 +MoE vector work consumed 31.05 percent of GPU kernel time, dense Q4_0 vector +work 28.18 percent, and dense Q6_K vector work 15.64 percent. The active +MoE kernel name was `moe_vec_q4_0_hip_two_rows`, proving that the trace covers +the accepted HIP specialization rather than the earlier generic path. + +Current llama.cpp source uses an RDNA4-specific eight-wave policy for simple +one-vector Q4_0 matvecs. Candidate commit `1bf9489` applied that scheduling +policy only to FreeToken's dense HIP Q4_0 launcher. It deliberately retained +the generic dot product, Q4_0 and Q8_1 packing, BF16 result contract, CUDA +path, all non-Q4_0 types, and the separately accepted routed-MoE kernel. +This made the candidate distinct from the already rejected dense two-row and +launch-bound experiments. + +The four shape-accurate dense microbenchmarks were mixed when rerun with 10 +warmups and 100 repetitions: the candidate improved 8,192 by 2,816 from +28.29 to 27.49 microseconds and 4,224 by 2,816 from 26.89 to 17.26 +microseconds, but regressed 2,816 by 4,096 from 21.30 to 23.91 microseconds +and 10,240 by 2,816 from 33.04 to 34.18 microseconds. Because Gemma uses all +four projections, this was insufficient to accept the launch policy. + +The full graph-captured API result confirmed rejection. It returned the +exact established SHA-1, but reached only **59.33 TPS**, 16.856 ms per token, +290.7 ms TTFT, and 18.011 ms p99 event latency. This is below the current +60.16 TPS final-source checkpoint and below the established accepted five-run +60.11 TPS median. The candidate remains on its separate branch and is not +part of the upstream-review branch. Its isolated worktree initially lacked +the unchanged native pinned-memory extension; the test setup copied the +validated extension only after SHA-256 and byte-for-byte equality checks. +That repair affected no source logic and the resulting API run is the only +performance outcome used for this decision. + +The retained raw evidence is: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/gemma-final-path-warm-20260829T021243Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/gemma-final-current-kernel-trace-20260829T021738Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-current-baseline-micro-20260829T022723Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-rdna4-eightwaves-micro-20260829T022528Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q4-rdna4-eightwaves-api-repaired-20260829T023038Z/ +``` + +### Rejected RDNA4 dense Q6_K eight-wave candidate + +The current final-source trace showed dense Q6_K vector work at 15.64 percent +of total traced GPU kernel time. Gemma uses Q6_K for its tied token embedding +and LM head, and the matching current llama.cpp RDNA4 policy selects eight +waves for one-vector Q6_K matvec. Candidate commit `ebf3f06` therefore +changed only FreeToken's HIP dense Q6_K wrapper to launch the established +generic Q6_K dot-product kernel with eight independent row waves per block. +It retained Q6_K and Q8_1 packing, reduction arithmetic, the BF16 result +contract, CUDA behavior, Q4_0 dense behavior, and all routed-MoE behavior. + +The full graph-captured loopback API workload returned the exact established +Gemma SHA-1 `abeee5e73e89`, used 15.66 GiB VRAM, and reported 259.6 ms TTFT +with a 17.663 ms p99 event latency. Its decode result was **59.98 TPS** or +16.672 ms per token. That is close to, but below, the 60.16 TPS accepted +final-source checkpoint. A one-run result without a TPS improvement does not +justify a second specialized scheduling path, so the candidate remains on its +separate experiment branch and is not part of the upstream-review branch. + +The test used the unchanged native pinned-memory extension after SHA-256 and +byte-for-byte equality checks against the validated final source. The raw +evidence is retained at: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/dense-q6-rdna4-eightwaves-api-20260829T023720Z/ +``` + +### Current host-interference qualifier + +A read-only GMKtek EVO-X2 health capture at 2026-08-29T02:41:15Z found no GPU reset, +thermal problem, or active FreeToken server. The Radeon 8060S was idle at +30 C after the test. It did, however, identify two pre-existing user-owned +filesystem scans in uninterruptible `D` state: one scanning `/home/operator`, +`/mnt`, and `/data` for large GGUF or SafeTensors files, and one scanning +`/home/operator` and `/media/operator` for Gemma GGUF files. At capture time they +had been alive for approximately 8.8 and 6.1 hours respectively. + +The same capture reported I/O full-pressure at 0.61 percent over ten seconds +and retained kernel warnings that `kfd_process_wq_release` and +`svm_range_deferred_list_work` had exceeded their CPU workqueue budget. These +facts do not prove that a particular FreeToken result is invalid, but they +provide a concrete explanation for occasional multi-millisecond dispatch +outliers and the isolated 52.50 TPS baseline run. They can affect both +FreeToken and llama.cpp under a matched test. + +No process priority, service state, kernel option, ROCm installation, or +hardware component was changed by this investigation. Any decision to stop +or otherwise alter the two user-owned scans requires explicit operator +authorization. Until then, accepted performance claims remain based on +multiple clean launches and retain raw tail-latency data rather than hiding +the interference. + +### Current review-branch static validation + +The current upstream-review commit `6c6198b10d9fb6a9c93e0aa94a05ac4144ec061d` +was validated directly on GMKtek EVO-X2 after the I/O evidence capture tooling was +added. The check completed without starting an inference server or changing +host state: + +```text +python -m compileall -q python benchmarks passed +pytest -q tests/kernels/test_gguf_hip_build_flags.py \ + tests/utils/test_rocm_runtime.py 4 passed +``` + +The raw output and commit metadata are retained at: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/current-review-static-validation-20260829T024456Z/ +``` + +A temporary high-performance DPM governor test could not be run because the +non-root GMKtek EVO-X2 account cannot write `power_dpm_force_performance_level`; +automatic mode was unchanged. + +Raw campaign artifacts are retained on GMKtek EVO-X2: + +```text +/home/operator/freetoken-amd/artifacts/amd-optimization-2026-08-28/ +``` + +## Deep-investigation baseline and profiler repair + +The reproducible read-only baseline is captured by +[`../scripts/gmk-evo-x2-capture-baseline.sh`](../scripts/gmk-evo-x2-capture-baseline.sh). +The first baseline was written to: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/baseline-20260828T220753Z/ +``` + +### Test-checkout repair and revalidated shipping baseline + +During the follow-on investigation, the isolated GMKtek EVO-X2 source checkout was +found at `61a1505`. That commit contained the subsequently rejected +two-block-residency Q4_0 MoE experiment. The authoritative branch had already +reverted that experiment at `b77825d` and documented the rejection at +`222cbd3`. Using the stale checkout for another benchmark would have made the +result impossible to attribute to the branch under review. + +The checkout was clean, so it was repaired with a fast-forward only update to +`origin/amd-rocm-gfx1151`, reaching `222cbd3`. No production process, +llama-swap configuration, or other LAN host was touched. The next run used a +new, dated `TORCH_EXTENSIONS_DIR`, forcing a fresh native HIP binary rather +than reusing the binary compiled from the stale source. + +| Item | Revalidated value | +| --- | --- | +| Artifact directory | `/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/repaired-baseline-20260828T224642Z/` | +| Source commit | `222cbd3` | +| Model SHA-256 | `3eca3b8f6d7baf218a7dd6bba5fb59a56ee25fe2d567b6f5f589b4f697eca51d` | +| Extension build | Fresh ROCm 10 `hipcc`, `--offload-arch=gfx1151`, `-O3` | +| API and workload | Loopback FreeToken API, greedy AIME-25 problem 0, one warm and one measured request | +| Measured completion | 127 tokens, 126 decode intervals | +| Client decode throughput | **55.04 TPS** or **18.169 ms/token** | +| TTFT | 295.2 ms | +| Event p50 / p99 | 18.454 ms / 19.509 ms | +| Output SHA-1 | `abeee5e73e89`, identical to the earlier shipping-configuration run | +| Post-run ROCm process check | No KFD PIDs | + +The single revalidation is consistent with the existing 55.44 TPS shipping +baseline and remains below the 60.42 TPS matched llama.cpp reference. It is a +provenance repair, not a new performance claim and not a substitute for the +planned repeated candidate measurements. + +### Rejected Q4_0 aligned-load candidate + +The matched traces showed FreeToken's Q4_0 vector kernels using 48 VGPRs per +thread, while llama.cpp's corresponding generic Q4 vector kernel reported 24 +VGPRs. Both used a 32-thread workgroup with zero LDS and scratch allocation. +As a narrow, low-risk test, commit `b8de163` replaced only the Q4_0 packed-load +helper expressions with the aligned `get_int_b2` and `get_int_b4` expressions +used by the current llama.cpp HIP source. The dot-product arithmetic, output +type, data layout, model, workload, and launch geometry were otherwise +unchanged. + +The target-host HIP build-configuration tests passed, the extension rebuilt +for `gfx1151`, and the output SHA-1 remained `abeee5e73e89`. However, the +candidate measured 54.99 TPS or 18.185 ms/token, versus 55.04 TPS or 18.169 +ms/token for the immediately preceding repaired baseline. That difference is +well inside normal run variation and does not improve the runner. The +candidate was therefore reverted by `c1899a0`; it is not part of the shipping +configuration. + +The raw candidate evidence is retained at: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/q4-load-alignment-20260828T225237Z/ +``` + +This eliminates aligned helper spelling as the explanation for the measured +register and throughput gap. The next candidate must change a more material +component: the Q4_0 vector-kernel execution structure, MoE expert dispatch, +or intermediate BF16 output path. + +### Rejected Q4_0 FP32-intermediate candidate + +llama.cpp's HIP Q4 vector paths use an FP32 destination, while FreeToken's +normal Q4_0 MoE path returns an activation-typed BF16 tensor after each vector +product. Commit `5bbe10f` added a deliberately opt-in experiment that used +FP32 only for the two Q4_0 MoE vector-product temporaries, then converted the +final MoE result back to the original BF16 public contract. It was activated +only with `FREETOKEN_GGUF_MOE_FP32_INTERMEDIATE=1`; normal launches stayed on +the existing dtype-preserving path. The target-host build-flag and opt-in +contract tests passed before the full-model run. + +The first launch under this candidate used the literal placeholder `model` +instead of the local GGUF path and exited during model resolution. It did not +reach HIP compilation, graph capture, or an API request. The failed artifact +is retained as a labelled harness error and is excluded from every comparison. +The corrected launch used the exact Gemma GGUF checksum, offload backend, +0.50 memory ratio, graph capture, greedy AIME-25 problem 0, and 128-token +decode procedure used by the repaired baseline. + +| Candidate | Decode TPS | ms/token | TTFT | Output SHA-1 | Decision | +| --- | ---: | ---: | ---: | --- | --- | +| Repaired BF16 baseline | 55.04 | 18.169 | 295.2 ms | `abeee5e73e89` | Reference | +| FP32 intermediates | 55.11 | 18.144 | 292.6 ms | `ce247609d76c` | Rejected | + +The 0.14 percent TPS change is smaller than the observed run-to-run variation, +does not close the gap to the 60.42 client TPS ROCm 10 llama.cpp reference, +and changes the deterministic greedy response hash. The candidate was +therefore reverted and is not a shipping option. Raw evidence remains on +GMKtek EVO-X2 at: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/fp32-intermediate-20260828T230126Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/fp32-intermediate-retry-20260828T230209Z/ +``` + +It confirms the active device is `gfx1151`, PyTorch is +`2.13.0+rocm10.0.0` with HIP `7.15.26333`, and `/opt/rocm` resolves to +`/opt/rocm-10.0`. It also records that the system package database retains +ROCm 7.2 development packages. This alone does not prove an application +runtime conflict, so library maps were collected before changing any host +component. + +The maps show that the PyTorch wheel loads its own ROCm SDK, including LLVM 23 +and rocprofiler-sdk 1.3.5, from `_rocm_sdk_core` in the virtual environment. +The host `rocprofv3` launch initially injected a second LLVM 23 and profiler +SDK from `/opt/rocm-10.0`, causing `import torch` to abort with duplicate LLVM +registration for `spirv-expand-step`. The failure was reproduced with a +minimal PyTorch import, so it is not caused by FreeToken. + +`scripts/gmk-evo-x2-rocprof-wheel-sdk.sh` repairs the launch path without editing +the host installation. It keeps the host `rocprofv3` front end but passes +`--rocm-root` for the wheel's `_rocm_sdk_core`, making the profiler use the +same library identities as PyTorch. The repair was validated by profiling a +small HIP allocation and reduction. ROCm emitted `kernel_trace.csv` and +`kernel_stats.csv` with the expected GPU dispatches. Use this wrapper only +for profiling, never for TPS scoring because tracing alters execution time. + +The first full FreeToken trace launch passed PyTorch import and model loading, +then reached the GGUF JIT compiler. The profiler environment is inherited by +that compiler subprocess, so the run was stopped before a request was sent. +The next trace must warm the GGUF extension unprofiled, then profile the +already-built decode path, or explicitly prevent profiler injection into JIT +child processes. This avoids treating compile activity as token-generation +performance. + +## GGUF extension reuse validation + +The first Gemma request after the original source change built the native HIP +GGUF extension. A subsequent complete server restart retained the existing +Torch extension cache. Its first API request returned HTTP 200 and Ninja +reported `no work to do`, proving the compiled shared module was reused. +Torch still runs a lightweight hipify and dependency check before loading the +cached module; it did not run `hipcc` compilation or shared-library linking. +See the persistent-cache operating procedure in +[`amd-rocm-gfx1151.md`](amd-rocm-gfx1151.md#persistent-gguf-hip-jit-cache). + +## Commands used + +Qwen was started in the isolated environment with this functional shape: + +```bash +ft serve --model-path /home/operator/freetoken-amd/models/Qwen3.6-35B-A3B-NVFP4 \ + --served-model-name qwen3.6-35b-a3b-nvfp4-amd --host 127.0.0.1 --port 18501 \ + --attention-backend triton --moe-backend offload --nvfp4-backend triton \ + --expert-load serial --moe-cache-auto --memory-ratio 0.35 \ + --max-seq-len-override 8192 --kv-reserve-tokens 2048 \ + --cuda-graph-max-bs 0 --disable-pynccl --disable-moe-prefill-overlap +``` + +Gemma used the native GGUF model file and its own loopback port: + +```bash +ft serve --model-path /home/operator/freetoken-amd/models/Gemma-4-26B-A4B-it-qat-q4_0-gguf/gemma-4-26B_q4_0-it.gguf \ + --served-model-name gemma-4-26b-a4b-q4-amd --host 127.0.0.1 --port 18502 \ + --attention-backend triton --moe-backend offload --expert-load serial \ + --moe-cache-auto --memory-ratio 0.50 --max-seq-len-override 8192 \ + --kv-reserve-tokens 2048 --cuda-graph-max-bs 0 --disable-pynccl +``` + +The API checks used `/v1/models` and `/v1/chat/completions`, both with normal +JSON responses and with `stream: true`. The front-end port can answer before +the worker finishes loading, so the successful tests waited for the server log +line `API server is ready to serve` before submitting requests. + +## AMD-specific corrections verified here + +1. ROCm detection is explicit, preventing `gfx1151` from being treated as an + NVIDIA SM 11.5 capability. +2. CUDA-only optional backends are not selected on HIP. +3. DLPack and fast indexed-copy tensor handling accepts HIP tensors. +4. HIP avoids the unsafe grouped NVFP4 prefill kernel and uses the native + Triton serial expert implementation instead. This trades prompt prefill + speed for correctness on the current Strix Halo stack. +5. The Gemma GGUF JIT discovers a system Thrust include directory when the + PyTorch wheel omits Thrust. It passes that path as a compiler system + include, avoiding an attempted hipify write into the ROCm installation. +6. The same JIT adds a system ROCm library directory only when the wheel SDK + lacks the unversioned `libamdhip64.so` linker name. On GMKtek EVO-X2 this allowed + the native `gfx1151` object and shared module to compile and link. + +## Known limitations and follow-up work + +- This is a functional API validation, not a performance benchmark. The + recorded request timings include the chosen small fixed requests and are not + tokens-per-second claims. +- CUDA graph capture remains disabled for the HIP MVP. +- Qwen's HIP prefill deliberately uses the safe serial Triton route instead of + the grouped NVFP4 prefill route that produced an HSA aperture violation on + this machine. +- The first Gemma request compiles its GGUF HIP extension and has a substantial + cold-start cost. Later requests use the cached module. +- llama-swap integration is intentionally outside this release gate. + +## Local checks completed + +```bash +python -m compileall -q python +git diff --check +``` + +The port's HIP gate tests are retained under `tests/utils/test_rocm_runtime.py`. +The live end-to-end checks above are the required full-model validation for +this change. + +## Clean-host ROCm 10 comparison after I/O remediation + +The earlier five-run comparison was repeated after the two identified +user-space filesystem scans had been stopped with the operator's explicit +authorization. This is the decision-quality comparison: it uses the same +GMKtek EVO-X2 `gfx1151` device, ROCm 10 runtime, 14 GB Gemma 4 26B A4B Q4_0 GGUF, +cached AIME-25 problem 0, greedy OpenAI-compatible streamed request, and +128-token generation limit on each runner. Every scored sample starts a +fresh server, makes one excluded warm request, then makes one scored request. +Decode TPS is `(completion_tokens - 1)` divided by the client-observed time +between the first and last text SSE events. + +FreeToken uses its accepted native HIP configuration: offload backend, 4,096 +expert-cache slots, 8,320-token KV pool, 0.50 memory ratio, and graph batch +size 1. llama.cpp uses its fixed ROCm 10 `b10141` release with all layers on +the GPU (`-ngl 999`), `-c 8320`, one parallel slot, and Flash Attention on. +Both use loopback only and no production service was enabled. + +| Runtime | Five decode TPS samples | Mean TPS | Median TPS | Mean TTFT | Median p99 event gap | +| --- | --- | ---: | ---: | ---: | ---: | +| FreeToken native HIP | 60.25, 60.20, 60.17, 60.13, 60.33 | 60.21 | 60.20 | 250.0 ms | 17.74 ms | +| llama.cpp `b10141` ROCm 10 HIP | 60.24, 60.50, 62.25, 61.71, 61.83 | 61.31 | 61.71 | 113.3 ms | 16.88 ms | + +All five FreeToken completions had hash `abeee5e73e89`; all five llama.cpp +completions had hash `63a18854de72`. The two hashes are intentionally not +compared to each other because the independent implementations render their +chat templates and tokenize internally. They establish deterministic output +within each runner. llama.cpp leads by 1.8 percent on the five-run mean and +2.5 percent on the median decode rate. FreeToken's mean warm TTFT is 120.6 +percent higher. Therefore the AMD port is proven functional and stable but +does not yet meet the requested requirement to match or exceed the optimized +llama.cpp control. + +The raw, per-run result and server-log bundles remain on GMKtek EVO-X2: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/clean-host-freetoken-matrix-20260829T030633Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/clean-host-llamacpp-matrix-20260829T031840Z/ +``` + +The llama.cpp bundle records zero blocked (`D`) processes before and after all +five samples. The FreeToken bundle was generated after the same scan removal +and has a 0.08 TPS standard deviation, so it is the more stable side of this +matrix. The remaining performance investigation should prioritize the +observed HIP decode hot spots already captured by rocprof: Q4 MoE vector +decode first, then dense Q4 and Q6_K matrix-vector kernels. New candidates +must retain deterministic API output and be accepted only when a five-fresh- +server clean-host matrix matches or exceeds the llama.cpp median, rather than +on an isolated best run. + +### Post-matrix Q4 MoE launch experiments + +The following HIP-only experiments were run after the clean-host matrix. They +are not shipping changes. Each used the accepted Gemma workload and produced +the expected deterministic greedy response hash `abeee5e73e89`; the throughput +result, not merely successful compilation, determines rejection. + +| Candidate | Change from accepted two-row kernel | Result | Decision | +| --- | --- | ---: | --- | +| `2f019fa` | Raise the wave32 launch minimum from one to eight resident workgroups per CU | 59.95 TPS | Rejected: 0.44 percent below the accepted 60.21 TPS mean. | +| `ddaf194` | Raise the same minimum from one to two resident workgroups per CU | 60.20 TPS | Rejected: no improvement and no progress toward the 61.71 TPS gate. | +| `3f57285` | Have one wave32 calculate four rows per route instead of two | 59.22 TPS | Rejected: 1.64 percent below the accepted mean. | + +The first two experiments initially used the shared persistent extension +directory. The eight-workgroup result compiled its own source successfully; +the two-workgroup source reused the existing shared module, so its numerical +result is recorded only as a directional screen rather than a source-binary +proof. A four-row run also detected this cache reuse before it was interpreted +and is explicitly excluded. The valid four-row result then set +`TORCH_EXTENSIONS_DIR` to an artifact-local directory, rebuilt the native +`gfx1151` shared module there, and recorded that module alongside the raw logs. + +This establishes a stricter rule for all remaining performance work: every +source-changing HIP candidate must compile in a unique extension-cache path, +and the artifact must contain the resulting shared module before API timing is +accepted. The immutable raw bundles are on GMKtek EVO-X2: + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/hip-moe-q4-occupancy-retry-20260829T032503Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/hip-moe-q4-occupancy-two-20260829T032852Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/hip-moe-q4-four-rows-20260829T033123Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/hip-moe-q4-four-rows-isolated-cache-20260829T033230Z/ +``` + +### Isolated dense Q4 four-wave experiment + +Commit `9e36b2d` keeps the CUDA generic path intact and adds a HIP-only dense +Q4_0 dispatch wrapper that launches four wave32 rows in one workgroup. The +change addresses the second-largest measured decode hot spot, rather than the +already-tested MoE-vector kernel. The candidate passed the static ROCm gate +(`4 passed`) before the live run. + +Its live test used a fresh artifact-local `TORCH_EXTENSIONS_DIR`. The log +contains both the `hipcc --offload-arch=gfx1151` compile invocation and the +successful shared-module link. The resulting module is +`freetoken_gguf_kernels.so`, SHA-256 +`8c363e3c9345b9ab03bda75a7660d4a284642c908a03f3faeb7b21f6f078e61d`. +This proves the timing used the candidate source rather than a shared cached +extension. + +The candidate produced the expected deterministic FreeToken output hash +`abeee5e73e89` and measured **60.67 decode TPS** with a 241.9 ms warm TTFT. +That is a 0.75 percent single-run improvement over FreeToken's clean-host +60.21 TPS five-run mean, but it remains 1.68 percent below the llama.cpp +61.71 TPS five-run median acceptance gate. It is therefore retained only as +an evidence-backed non-shipping experiment, not promoted to the AMD branch or +given a five-run matrix. + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/hip-dense-q4-four-waves-isolated-cache-20260829T033846Z/ +``` + +### Isolated dense Q4 two-wave experiment + +Commit `56caf3b` tested the only remaining small workgroup-size point: two +wave32 rows per HIP Q4_0 dense-matrix-vector workgroup. As with the four-wave +experiment, the CUDA route remains unchanged and the candidate retains the +generic arithmetic, row mapping, and partial-row bounds check. The static +ROCm gate passed (`4 passed`) before the live run. + +The live run used a new artifact-local extension directory and logged a native +`hipcc --offload-arch=gfx1151` build plus link of +`freetoken_gguf_kernels.so`. It returned the exact expected output hash +`abeee5e73e89`, but measured **60.64 decode TPS** with 242.3 ms warm TTFT. +This is statistically indistinguishable from the four-wave single-run screen +(60.67 TPS), below the llama.cpp 61.71 TPS median gate, and insufficient to +justify a clean-host five-run matrix. The candidate is rejected and remains +outside the shipping AMD branch. + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/hip-dense-q4-two-waves-isolated-cache-20260829T034349Z/ +``` + +### Accepted gfx1151 RDNA3 dot-product intrinsic selection + +The current llama.cpp source was examined at immutable revision +`d7bd3bfcad3e29c7e49fd26f38c79ee3e9a3fd6b`. Its HIP helper chooses +`__builtin_amdgcn_sudot4(true, a, true, b, c, false)` on RDNA3 and RDNA4, +whereas FreeToken's copied GGUF helper always selected `sdot4` whenever that +builtin was available. Both forms implement the same signed four-byte dot +product for this Q4_0 plus Q8_1 route, but the source-level intrinsic choice +changes the gfx1151 compiler's generated code. + +Commit `5f040ba` adds a strictly scoped branch before FreeToken's existing +`sdot4` fallback. It is enabled only when the compiler exposes `sudot4` and +the device macro is `__gfx1100__`, `__gfx1150__`, or `__gfx1151__`. The +CUDA implementation, all non-RDNA3-family AMD targets, packing, scale math, +and BF16 output contract remain unchanged. The candidate passed the static +ROCm gate (`4 passed`) and built a separate native gfx1151 module with +`hipcc --offload-arch=gfx1151`. Its matrix module SHA-256 is +`5683cf07a9a081dbaf51c857757ce2307daa6822c6da4102908eb00ecd08ee3c`. + +The first five fresh-server executions all produced the expected FreeToken +output hash `abeee5e73e89`. Two were conservatively excluded because a +transient blocked process was present at their preflight snapshot, even though +none remained afterward. Two replacements explicitly waited for zero blocked +processes. The final acceptance set therefore uses runs 1, 2, 3, 6, and 7, +all with zero blocked processes before and after execution: + +| Runtime | Five clean decode TPS samples | Mean TPS | Median TPS | TPS stdev | Mean warm TTFT | +| --- | --- | ---: | ---: | ---: | ---: | +| FreeToken gfx1151 `sudot4` HIP | 61.68, 61.80, 61.84, 62.05, 61.94 | **61.86** | **61.84** | 0.14 | 241.7 ms | +| Previous FreeToken native HIP | 60.25, 60.20, 60.17, 60.13, 60.33 | 60.21 | 60.20 | 0.08 | 250.0 ms | +| llama.cpp `b10141` ROCm 10 HIP control | 60.24, 60.50, 62.25, 61.71, 61.83 | 61.31 | 61.71 | 0.88 | 113.3 ms | + +This is a 2.74 percent FreeToken mean-decode improvement over the prior clean +matrix. It exceeds the matched llama.cpp control by 0.56 TPS or 0.91 percent +on mean decode throughput, and by 0.14 TPS or 0.22 percent on median decode +throughput. The FreeToken warm TTFT remains higher, so this acceptance is +specifically for the requested sustained decode-TPS requirement. The API +remains OpenAI-compatible and deterministic for the workload. + +```text +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/hip-rdna35-sudot4-isolated-cache-20260829T035002Z/ +/home/operator/freetoken-amd/artifacts/amd-deep-investigation-2026-08-28/hip-rdna35-sudot4-clean-host-matrix-20260829T035209Z/ +``` diff --git a/docs/gmktec-evo-x2-rocm-validation-2026-08-30.md b/docs/gmktec-evo-x2-rocm-validation-2026-08-30.md new file mode 100644 index 0000000000..46fb7e58cf --- /dev/null +++ b/docs/gmktec-evo-x2-rocm-validation-2026-08-30.md @@ -0,0 +1,319 @@ +# GMKtek EVO-X2 ROCm validation results, 2026-08-30 + +## Scope + +This report records post-repair validation of the native FreeToken ROCm/HIP port on the GMKtek EVO-X2 Radeon 8060S. It covers the OpenAI-compatible API, Gemma 4 vision correctness, Qwen reliability, a controlled llama.cpp ROCm comparison, and a strict multi-turn endurance run. It is local hardware evidence, not a reproduction of the FreeToken paper's NVIDIA results. + +## Reproduction boundary + +| Item | Observed value | +| --- | --- | +| Host GPU | AMD Radeon 8060S Graphics, `gfx1151` | +| FreeToken revision | `d6ee8cef479c6e72b2210c24dc848b66cf9da75a` | +| Python | 3.12.13 | +| HIP | 7.15.26333 | +| PyTorch | `2.13.0+rocm10.0.0`, HIP 7.15.26333 | +| Qwen service | `qwen3.6-35b-a3b-nvfp4-amd` on loopback port 1919 | +| Gemma service | `gemma4-26b-q4-amd` on temporary loopback port 1923 | +| llama.cpp control | ROCm 10 build, temporary loopback port 1921 | + +All model-server work used the native ROCm/HIP path. No Vulkan runner, CPU fallback, llama-swap route, or other LAN host was used as a substitute. + +## Gemma 4 multimodal repair + +The first live Gemma controls established that image tensors reached the GPU but the model answered simple colors incorrectly. The repair had two required parts: + +1. Preserve `mm_pixel_values` and `mm_image_position_ids` when the tokenizer server forwards a user message to the scheduler. +2. Emit RGB patches in channel-planar order, not pixel-interleaved order. The Gemma projector weight `v.patch_embd.weight` is a convolution kernel with `[output, channel, patch_y, patch_x]` layout, so each patch must contain all red values, then green, then blue. + +The fixed path was tested through the real OpenAI `image_url` data-URL contract, decoding, patchification, tensor wire protocol, ROCm vision tower, projector, embedding scatter, and response generation. + +| Control | FreeToken result | llama.cpp ROCm result | +| --- | --- | --- | +| solid red | pass | pass | +| solid green | pass | pass | +| left half of red-left/blue-right image | red | red | +| solid blue | pass | pass | +| solid yellow | pass | pass | +| right half of red-left/blue-right image | blue | blue | +| top half of blue-top/yellow-bottom image | blue | blue | + +FreeToken passed all seven controls in one extended run, then passed all 21 requests in three complete repetitions. Its 45 to 65 word visual-description control also passed, correctly describing a red left side and blue right side at 53.67 visible output tokens per second. The matching llama.cpp ROCm control passed the same seven deterministic fixtures. + +## Qwen API and correctness + +The FreeToken Qwen endpoint returned a healthy status before and after every exclusive Gemma or llama.cpp control. `/v1/models` reported the expected model and 8,192-token configured context length. + +The deterministic visible-output suite passed on both FreeToken and llama.cpp: + +| Check | FreeToken | llama.cpp ROCm | +| --- | --- | --- | +| exact `host-identity canary` output | pass | pass | +| `17 * 19 = 323` | pass | pass | +| exact JSON fields `status=ok`, `value=7` | pass | pass | + +Ten independent three-turn FreeToken conversations also passed every turn: remember `azure-17`, recall it, and transform its numeric component to `23`. The median maximum per-turn TTFT was 0.414 seconds and the worst observed token gap was 39.77 ms. + +## Concurrency and long context + +The following Qwen API matrix used fixed greedy streaming requests, 128 output tokens, three rounds per level, and retained every raw stream. The first run started from pre-existing swap pressure and is labeled diagnostic rather than clean-memory endurance evidence. + +| Concurrent requests | Successful rounds | Mean aggregate TPS | p99 TTFT | p99 token gap | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 3 of 3 | 19.96 | 8.60 s | 69.98 ms | +| 2 | 3 of 3 | 28.30 | 0.84 s | 126.80 ms | +| 4 | 3 of 3 | 50.52 | 0.93 s | 139.65 ms | +| 8 | 3 of 3 | 50.78 | 10.56 s | 139.58 ms | + +The 8-way result is a saturation result. Aggregate throughput did not improve over four simultaneous requests, while p99 time to first token increased substantially. It is not a recommended interactive concurrency target. + +Long-context retrieval used an exact early marker, three samples at each size, and a unique prefix nonce per sample to prevent full-prefix cache reuse. Every sample returned only the required marker. + +| Reported prompt tokens | Passed samples | Mean TTFT | Maximum TTFT | p99 token gap | +| ---: | ---: | ---: | ---: | ---: | +| 2,616 | 3 of 3 | 5.58 s | 7.74 s | 38.57 ms | +| 5,176 | 3 of 3 | 9.02 s | 12.97 s | 39.90 ms | +| 7,736 | 3 of 3 | 16.41 s | 17.73 s | 40.47 ms | + +## Matched workload comparison with llama.cpp + +Both runners executed the same fixed scheduler prompt, 256 requested output tokens, greedy decoding, one concurrent request, one 8,192-token slot, and three measured samples after warmup on GMKtek EVO-X2. The values are decode TPS, not aggregate concurrent throughput. + +| Runner | Model format | Successful samples | Median decode TPS | +| --- | --- | ---: | ---: | +| FreeToken ROCm/HIP | Qwen3.6-35B-A3B NVFP4 | 3 of 3 | 28.15 | +| llama.cpp ROCm 10 | Qwen3.6-35B-A3B Q4_K_M GGUF | 3 of 3 | 48.87 | + +This is a same-host, same-prompt, same-output-length comparison, but it is not a quantization-equivalent comparison. FreeToken loaded NVFP4 while llama.cpp loaded Q4_K_M GGUF. Therefore it proves the current observed runner outcome for these deployed artifacts, not an intrinsic winner between FreeToken and llama.cpp. The current FreeToken configuration does not meet or exceed the llama.cpp decode figure in this workload. + +## Exact-Q4_K_M ROCm comparison + +The branch now includes a native FreeToken loader for the same +`Qwen3.6-35B-A3B-UD-Q4_K_M.gguf` file used by the llama.cpp control. This path +keeps the GGUF weights packed: dense Q8_0 and Q6_K tensors use the native GGML +HIP operators, routed gate and up experts use Q4_K, routed down experts use +Q5_K or the file's late-layer Q6_K exception, and the Qwen hybrid +Gated-DeltaNet metadata and recurrent-state layout are handled by the native +Qwen3.5 model path. + +Before serving, a source-revision-specific gfx1151 helper cache compiled 82 +native ROCm/HIP modules and a strict no-JIT verifier loaded all 82. The Q4 +server then started on a temporary loopback port with the same 8,192-token +context policy used by llama.cpp: `0.35` memory ratio, 8,192-token KV reserve, +one host, one GPU, greedy sampling, one request, the fixed scheduler prompt, +256 requested output tokens, warmup, and three scored samples. The FreeToken +Q4 server resolved 8,626 MoE cache slots and 8,227 KV pages. + +| Runtime | Model file and format | Mean decode TPS | Median decode TPS | Sample standard deviation | Quality suite | +| --- | --- | ---: | ---: | ---: | --- | +| FreeToken ROCm/HIP | Exact Q4_K_M GGUF | 48.444 | 48.450 | 0.0267 | 3 of 3 pass | +| llama.cpp ROCm 10 | Same exact Q4_K_M GGUF | 49.125 | 49.131 | 0.0138 | 3 of 3 pass | + +The fresh same-format difference is 0.680 TPS, or 1.39 percent in favor of +the current llama.cpp control. This is the relevant comparison for runner +efficiency because it removes the NVFP4-versus-Q4_K_M weight-format difference. +FreeToken is very close but does not yet meet or exceed llama.cpp in this +strict matched workload. + +FreeToken's additional caller-rendered, 512-token raw-prompt control produced +511 visible completion tokens at 48.487 TPS and 433.11 ms TTFT. The standard +visible-output quality suite passed its exact `host-identity canary`, arithmetic `323`, and +strict JSON controls. A temporary GPU `high` DPM policy was also tested with +the loaded Q4 server, but it reduced mean decode throughput to 47.287 TPS while +quality still passed. The normal `auto` policy therefore remains the accepted +policy for this configuration. + +The exact-Q4 evidence is retained on GMKtek EVO-X2 at +`/home/operator/freetoken-amd/artifacts/qwen35moe-gguf-full-control-20260830T141438Z/` +and +`/home/operator/freetoken-amd/artifacts/qwen35b-llamacpp-rocm10-q4matched-20260830T142002Z-retry/`. + +The Q4 server also passed the full cold long-context retrieval control: five +unique-prefix requests at 6,856 reported prompt tokens all returned only +`azure-17`. Mean TTFT was 26.989 seconds, maximum TTFT was 39.088 seconds, +and p99 visible token gap was 23.890 ms. The strict 30-session multi-turn +endurance gate was initially not qualified for the `0.35` memory-ratio +configuration. After the cold long-context run, 3.3 GiB of whole-host swap +usage was observed. A controlled reset returned the host counter to zero and +preserved endpoint health, but 540 KiB reappeared immediately before the first +endurance session, exceeding the original whole-host 64 KiB guard. That +measurement did not identify the process responsible for the swapped pages. +The original high-cache profile remains an active stability investigation +because it can also hit the separate ROCm SVM-resident-memory failure above, +not because of a throughput or quality failure. + +## Q4 SVM-resident-memory recovery profile + +Follow-up investigation established that the failures above were not an +incorrect answer or an API-contract failure. A Q4 server with the original +`0.35` memory ratio could initialize successfully, but a first decode after a +forced cancellation could stall. The Linux kernel recorded +`amdgpu: SVM mapping failed, exceeds resident system memory limit`; the +associated FreeToken scheduler worker consumed CPU while the request emitted no +response bytes. The test procedure also revealed that stopping only the HTTP +parent leaves its multiprocessing children alive, including a child that keeps +the internal distributed port `1923` bound. All subsequent controls used a +dedicated process group and terminated that full group before another GPU +server was started. + +The recovery profile preserves the exact same Q4_K_M GGUF, native ROCm/HIP +path, 8,192-token context policy, four request slots, OpenAI-compatible API, +and automatic MoE cache policy. It changes only the memory ratio from `0.35` +to `0.25`, retains the host's temporary `vm.swappiness=1` test policy, and +starts from a verified zero-swap state. The lower ratio resolved 5,465 MoE +slots and 8,237 KV pages, leaving 23.06 GiB free after initialization instead +of about 17.46 GiB. It is therefore a stability-oriented configuration, not a +claimed decode-speed optimization. + +| Control | Result | +| --- | --- | +| visible-output quality suite | 3 of 3 pass | +| multi-turn state retention | 30 of 30 sessions pass, zero KiB verified runner-process-group swap at every session boundary | +| multi-turn p99 maximum turn TTFT | 0.429 s | +| multi-turn p99 visible-token gap | 25.75 ms | +| 6,856-token cold marker retrieval | 5 of 5 pass, 24.733 s mean TTFT, 26.255 s maximum TTFT | +| two simultaneous users | 3 of 3 rounds pass, 45.29 mean aggregate TPS, 8.192 s p99 TTFT | +| four simultaneous users | 3 of 3 rounds pass, 79.00 mean aggregate TPS, 1.561 s p99 TTFT | + +The five long-context requests coincided with a 4.864 MiB increase in the +whole-host swap counter, despite the low-swappiness policy. That counter is +useful host telemetry but does not identify the model process, so the +long-context result is a successful quality and latency result, not proof of a +strict zero-swap all-day service state. Later attribution showed that desktop +and monitoring daemons can hold swapped pages while every member of the +verified FreeToken server process group reports `VmSwap: 0 kB`. The ongoing +wall-clock battery therefore records whole-host swap but fails only when the +dedicated FreeToken process group itself has swapped pages. + +The same fixed 256-token scheduler workload was rerun from the current +FreeToken Q4 profile and a fresh ROCm 10 llama.cpp control, with the same GGUF +file, prompt, tokenizer, temperature, top-p, top-k, output length, context, +and one request. Both also passed the same deterministic three-case quality +suite. + +| Runtime | Mean decode TPS | Median decode TPS | Mean TTFT | Quality suite | +| --- | ---: | ---: | ---: | --- | +| FreeToken Q4 recovery profile | 47.960 | 48.075 | 0.453 s | 3 of 3 pass | +| llama.cpp ROCm 10 current control | 48.831 | 48.832 | 0.062 s | 3 of 3 pass | + +The recovery profile is 0.871 TPS, or 1.78 percent, below the fresh llama.cpp +control for that fixed decode workload. It restores full functional +qualification under the memory guard but does not meet or exceed llama.cpp. +The original higher-cache Q4 profile remains the closer decode result, at 1.39 +percent below its fresh llama.cpp control, but requires a repair for the SVM +resident-memory limit before it can be recommended as the stable profile. + +Retained raw evidence for this recovery investigation is under +`/home/operator/freetoken-amd/artifacts/qwen35moe-gguf-memory-ratio-025-20260830T150554Z/` +and the fresh llama.cpp control is under +`/home/operator/freetoken-amd/artifacts/qwen35moe-llamacpp-rocm10-current-harness-retry-20260830T151654Z/`. + +## Initial clean-memory endurance + +The initial 30-session battery reset whole-host swap before starting and +enforced a maximum of 64 KiB at every session boundary. It completed before +the later process attribution work. Its functional and timing results remain +valid, but the whole-host swap limit is superseded by the verified +runner-process-group gate used by the current wall-clock endurance battery. + +| Metric | Observed result | +| --- | --- | +| Completed sessions | 30 of 30 | +| Passed sessions | 30 of 30 | +| p95 maximum turn TTFT | 0.596 s | +| p99 maximum turn TTFT | 2.369 s | +| p99 maximum token gap | 39.63 ms | +| Initial swap guard | passed, zero KiB whole-host usage observed after completion | +| Qwen health after run | `status: ok` | +| Final sampled GPU edge temperature | 42 C | + +## Process-scoped wall-clock endurance qualification + +The corrected endurance battery ran 60 deterministic three-turn conversations +at one-minute cadence, for a full hour of wall-clock observation. It validated +the exact visible answers `ACK`, `azure-17`, and `23` in every session. The +wrapper also resolved the dedicated Q4 HTTP server process group before every +memory sample and rejected a session if any member reported nonzero `VmSwap`. +Whole-host swap was retained as diagnostic telemetry only, because Linux +desktop and monitoring processes can use swap independently of FreeToken. + +| Metric | Result | +| --- | --- | +| completed and passed sessions | 60 of 60 | +| runner process-group swap | 0 KiB minimum and maximum | +| maximum-turn TTFT mean | 0.424 s | +| maximum-turn TTFT p95 | 0.414 s | +| maximum-turn TTFT p99 and maximum | 1.184 s | +| maximum visible-token-gap mean | 24.95 ms | +| maximum visible-token-gap p95 | 25.98 ms | +| maximum visible-token-gap p99 and maximum | 27.17 ms | +| whole-host swap telemetry | 33.07 MiB to 38.17 MiB | + +The single 1.184-second maximum-turn TTFT observation is retained as an +observed tail outlier, not hidden by a mean-only result. It did not cause an +incorrect answer, runner swapping, process failure, or loss of API service. +The machine was restored after the battery: the temporary Q4 listener on +port 1922 was stopped and `vm.swappiness` was restored to 60. The normal +NVFP4 service on loopback port 1919 required about eight minutes of cold +expert initialization before its readiness log appeared. It was then checked +through the OpenAI-compatible endpoint with a live arithmetic request, which +returned the correct visible answer `4`, and retained its advertised +8,192-token context. + +Raw evidence is retained under +`/home/operator/freetoken-amd/artifacts/qwen35moe-gguf-process-scoped-endurance-20260830T153333Z/`, +including each request JSON, per-session telemetry, and the machine-generated +`summary.json`. The reusable verifier is +`benchmarks/gmk_evo_x2/summarize_qwen_gguf_endurance.py`. + +## Full-context MoE cache telemetry + +The normal Qwen service intentionally leaves MoE counters disabled because the +counter atomics are diagnostic work. A temporary, loopback-only instance was +therefore started with `--moe-collect-stats`, using the same native ROCm/HIP +configuration, `0.35` memory ratio, and 8,192-token KV reservation as the +restored service. The normal no-counter service was restarted immediately after +the test and passed its deterministic AIME output-hash gate. + +The diagnostic instance resolved 8,903 MoE cache slots and 8,224 KV pages. Its +fixed scheduler workload completed all three scored samples at 28.035 mean +decode TPS, with only 0.0066 TPS standard deviation. Across 40,800 decode-layer +calls, it selected eight experts per layer and missed 0.586 experts per layer, +for a 7.33 percent MoE cache miss rate. No expert fetches were reported through +the separate fetch counter on this workload. + +This result confirms that full 8K context capacity is active while the Qwen +decode rate remains near the accepted 28 TPS baseline. It also supports the +previous rejection of a larger static MoE cache: prior 0.38-memory-ratio +testing reduced misses but did not produce a sustained TPS gain. Cache capacity +alone is therefore not a justified route to closing the current llama.cpp gap. + +The telemetry and restoration evidence is retained on GMKtek EVO-X2 at +`/home/operator/freetoken-amd/artifacts/qwen-cache-stats-driver-20260830T135236Z/`. +The restored normal service returned the required AIME SHA-1 +`0acef4eab6f4`, at 28.60 visible decode TPS, 399.08 ms TTFT, and 38.49 ms p99 +stream-event gap. + +## Regression tests + +The focused regression suite passed 21 tests on GMKtek EVO-X2: + +```text +tests/server/test_message_wire.py +tests/tokenizer/test_gemma4_image.py +tests/models/test_gemma4_mmproj_mapping.py +tests/benchmarks/test_gmk_evo_x2_benchmark.py +``` + +## Remaining work + +1. The quantization-equivalent Qwen control is now complete. The recommended + stable recovery profile is 1.78 percent below llama.cpp in the fixed + single-request decode workload. The higher-cache profile measured 1.39 + percent below its separately fresh llama.cpp control, but is not a + recommended configuration because its SVM-resident-memory failure remains + unresolved. Any claim to meet or exceed llama.cpp needs a retained + optimization and a fresh matched requalification. +2. Continue kernel-level decode work only from profiler evidence. Existing cache-capacity, graph, copy-grid, DPM-policy, and several dense and NVFP4 kernel candidates did not produce a quality-preserving end-to-end gain. Candidate work must preserve the API, vision, quality, long-context, and endurance gates in this report. +3. The one-hour process-scoped wall-clock endurance workload is complete and qualified. Consider a longer all-day workload only if deployment requires evidence beyond this explicit one-hour qualification. +4. Package sanitized build manifests and selected raw artifacts for the fork and upstream pull request. Do not publish local model files, private host paths, or operational access information. diff --git a/docs/gmktec-evo-x2-strix-halo-50pct-campaign.md b/docs/gmktec-evo-x2-strix-halo-50pct-campaign.md new file mode 100644 index 0000000000..c261e053e1 --- /dev/null +++ b/docs/gmktec-evo-x2-strix-halo-50pct-campaign.md @@ -0,0 +1,895 @@ +# GMKtek EVO-X2 Strix Halo 50 percent performance campaign + +## Objective + +Increase the client-visible steady-state decode speed of the native ROCm/HIP +FreeToken Qwen3.6-35B-A3B Q4 service on GMKtek EVO-X2 by up to 50 percent over the +currently accepted exact-Q4 baseline, while retaining equivalent output quality +and operational reliability. + +The target is an engineering hypothesis, not a promised result. Every claimed +gain must be measured against the same model, prompt, decoding contract, and +quality suite. The campaign stops at the measured limit of the approved +software and host scope if the target cannot be achieved without a regression. + +## Baseline and numeric target + +The accepted comparison baseline is the exact Qwen3.6-35B-A3B Q4_K_M FreeToken +profile with `memory_ratio=0.25`, measured through the local OpenAI-compatible +API after warmup. Its recorded mean decode speed is 47.960 tokens per second. + +| Measure | Value | +| --- | ---: | +| Accepted FreeToken baseline | 47.960 decode tokens per second | +| 50 percent campaign target | 71.940 decode tokens per second | +| Matched llama.cpp ROCm control | 48.831 decode tokens per second | + +This document does not treat an isolated kernel time, server-internal counter, +batch-only aggregate, or a different quantization as a substitute for the +baseline metric. Those are diagnostic measurements and must be labelled as +such. + +## Scope boundaries + +- Target host: GMKtek EVO-X2 only, Radeon 8060S `gfx1151`. +- Target runtime: native FreeToken ROCm/HIP path only. +- Target model: the exact qualified Qwen3.6-35B-A3B Q4_K_M artifact. +- Candidate servers bind only to loopback test ports in isolated clean + worktrees. +- The protected normal Qwen service is stopped only inside an explicit + time-share window and must be verified healthy after every window. +- Do not alter llama-swap, LAN routes, production model files, BIOS settings, + kernel, system ROCm packages, or host power limits under this campaign. +- Preserve every rejected result with its failure or rejection reason. + +## Non-negotiable acceptance gate + +A candidate may replace the accepted baseline only when all conditions hold: + +1. It improves the median client-visible decode rate by at least one percent + over the accepted baseline in two independently launched API matrices. +2. The exact deterministic canaries remain byte-identical for same-weight, + same-template, greedy decoding comparisons. +3. The versioned functional suite passes, including arithmetic, structured + JSON, retrieval, multi-turn, and long-context cases. +4. Mean TTFT does not regress by more than five percent and p99 token-gap + latency does not materially worsen. +5. It introduces no NaN, malformed SSE sequence, crash, stale process group, + SVM memory failure, unbounded growth, or failed protected-service recovery. +6. It records the full commit, patch, runtime versions, exact commands, model + identity, raw outputs, telemetry, and accept or reject decision. + +A fast candidate that fails any quality or reliability gate is rejected even if +it exceeds 71.940 tokens per second. + +## Campaign ladder + +### Stage 0: lock and stress the control + +1. Complete the running 24-hour minute-cadence Q4 endurance battery. +2. Confirm all request sessions pass after excluding the documented initial + warmup effect. +3. Confirm the normal NVFP4 Qwen endpoint is restored by the controller and + answers its health check with the intended model identity. +4. Archive a signed baseline manifest, API matrix, quality outputs, and + telemetry summary before any new candidate starts. + +The first long battery may be deliberately concluded after a successful +six-hour checkpoint when an active optimization window is more valuable than +additional identical idle-duration coverage. Such a run is always labelled +`incomplete_checkpoint`, never reported as a completed 24-hour endurance pass. +Before the next candidate starts, the controller must stop the Q4 service, +restore the protected NVFP4 server, and reach a real `serving` health state. + +### Stage 1: make quality difficult to accidentally regress + +The existing small exact suite is necessary but insufficient for aggressive +kernel and scheduling changes. Extend it in a versioned corpus with: + +- Greedy exact-response canaries for routing and numerical-order changes. +- Machine-scored arithmetic and constrained reasoning answers. +- JSON and tool-call-shaped schema validation. +- Code snippets with a local execution test harness. +- 2K, 8K, 16K, and maximum-qualified-context retrieval cases. +- Multi-turn correction and cache-reuse cases. +- A small fixed Gemma 4 text and image control set, run separately so Qwen + improvements never hide a Gemma regression. + +The suite stores prompts, generated text, token counts, finish reasons, scorer +results, and output hashes. It is a gate, not a performance workload. + +### Stage 2: profile the actual server + +Collect the following in a dedicated Q4 candidate window: + +1. Low-overhead application counters for a warm 256-token decode, a long + context request, and concurrency levels 1, 2, 4, and 8. +2. HIP event timing around router, cache operations, dense projections, MoE + projections, attention, sampling, and synchronizations. +3. A representative ROCm trace using the wheel-compatible profiler wrapper. + +Rank candidates by end-to-end decode contribution. Do not optimize a +microbenchmark only because it looks slow outside the actual server trace. + +The trace protocol launches the disposable Q4 process through the +wheel-compatible ROCm profiler wrapper. The host profiler cannot safely attach +to the running PyTorch ROCm wheel on this machine, and raw profiler throughput +is intentionally excluded from every TPS comparison because trace collection +is intrusive. Capture kernel dispatch, HIP runtime, memory-copy, and KFD +events for one warmed fixed-length decode, then use a read-only database +aggregate to rank the final active window. + +### Stage 3: run three independent optimization lanes + +#### Lane A: RDNA3.5 dense FP8 decode + +The earlier trace identifies the dense FP8 `_gemv_splitk_kernel` as the largest +measured GPU-time consumer. This lane investigates exact Qwen matrix shapes +only, preserving split-K reduction ordering and accumulation precision. + +Screen workgroup geometry, vectorized load alignment, wave occupancy, register +pressure, LDS use, and shape-specialized dispatch. Inspect generated ISA +before claiming an intrinsic or coalescing improvement. Reject a candidate +that changes deterministic canaries. + +#### Lane B: UMA-aware MoE cache and expert movement + +Static larger cache residency previously reduced cache misses without producing +an API throughput gain and the high-memory profile showed SVM instability. +This lane therefore measures a contention curve rather than assuming more cache +is better. + +Test cache target, KV allocation, active request count, route locality, and +safe-point resizing. A policy may increase cache only while verified memory +headroom remains above a configured guard threshold. It must back off with +hysteresis before paging or a driver fault, and it must never silently change +the model or precision. + +#### Lane C: scheduler and launch overhead + +Measure whether decode is limited by CPU launch chains, small kernel dispatches, +or poor request coalescing. Test scheduler policies at controlled concurrency +while separately reporting per-user TPS, aggregate TPS, TTFT, queue time, and +tail token gap. + +Only investigate graph capture, persistent execution, or layer-local route +batching when the trace establishes that launch or synchronization cost is +large enough to justify the complexity. A concurrency gain is reported as an +aggregate-throughput result and never presented as a single-user TPS gain. + +### Stage 4: compose accepted improvements + +Accepted changes are combined one at a time. After each composition, rerun the +full single-user and concurrent API matrix, full quality suite, long-context +test, multi-turn battery, controlled cancellation, and service-recovery test. +This prevents individually safe changes from hiding an interaction regression. + +### Stage 5: controlled platform qualification + +Only after exhausting code and policy work, consider a separate ROCm, kernel, +or firmware qualification project. It requires a specific approval because it +changes host-level software outside this campaign. The justification must +include a current SVM or compiler limitation, a rollback plan, and the exact +same before-and-after test matrix. + +## Iteration protocol + +For every candidate: + +1. Create a clean worktree and give the candidate a short, immutable ID. +2. Write a design note naming the bottleneck, hypothesis, expected upside, + quality risk, and rollback method. +3. Run the relevant microbenchmark only as an initial screen. +4. Start on a loopback candidate port and verify health, model identity, and + native extension identity. +5. Run the complete quality gate before throughput work. +6. Run five warm API samples plus the concurrent matrix with aligned telemetry. +7. Run long-context, multi-turn, cancellation, stop, and recovery validation. +8. Compare against a fresh baseline from the same host state whenever possible. +9. Mark the candidate accepted, rejected, or inconclusive with raw evidence. +10. Restore and verify the protected normal service before leaving the window. + +## Reporting + +Each accepted or rejected candidate receives a row with: + +| Candidate | Baseline TPS | Candidate TPS | Delta | TTFT | p99 gap | Quality | Reliability | Decision | +| --- | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | + +The final report will separate: + +- Single-user client-visible decode TPS. +- Concurrent aggregate throughput and per-user latency. +- Kernel-only diagnostic changes. +- Stable production-eligible configurations. +- Experimental configurations that are faster but not yet reliable. +- The remaining measured bottleneck if the 50 percent target is not reached. + +## Experiment log + +### C01: two-row HIP GGUF matrix-vector blocks + +The first post-checkpoint ROCm trace used the exact Qwen3.6-35B-A3B Q4_K_M +workload and isolated Q4 server. Its final 30-second active window ranked the +GGUF vector kernels, rather than the NVFP4 dense FP8 path, as the primary work: + +| Kernel family | Calls | GPU time in traced window | +| --- | ---: | ---: | +| Q8_0 vector matrix multiply | 81,600 | 3,660.029 ms | +| Q4_K routed MoE vector multiply | 20,480 | 3,103.535 ms | +| Q5_K routed MoE vector multiply | 18,944 | 1,971.720 ms | +| Q6_K vector matrix multiply | 512 | 920.072 ms | +| Routed cache gather | 22,016 | 684.485 ms | + +The upstream matrix-vector launch used one 32-thread output row per block. +The candidate grouped two independent rows into a 64-thread HIP block, which +fills one RDNA wavefront while retaining the same per-row quantization and +reduction. It built successfully in clean worktree `218104c`, passed the +three deterministic Qwen API controls, and completed three fixed-workload API +samples. + +| Measure | Stable baseline | C01 candidate | Change | +| --- | ---: | ---: | ---: | +| Mean decode TPS | 47.960 | 48.081 | +0.25% | +| Median decode TPS | 48.075 | 48.083 | +0.02% | +| Mean TTFT | 0.453 s | 0.439 s | diagnostic only | +| Quality controls | 3/3 pass | 3/3 pass | unchanged | + +**Decision: rejected.** The candidate is numerically safe in the screened +controls, but its 0.25 percent gain is below the one percent acceptance floor +and is within normal run-to-run variation. The change was reverted in +`0a1b709`; its complete candidate artifact remains on GMKtek EVO-X2 for comparison. + +### C02: opt-in HIP unsafe-math optimizations + +Current llama.cpp HIP build guidance uses `-funsafe-math-optimizations`, which +is narrower than `-ffast-math`. The candidate made that flag opt-in through +`FREETOKEN_HIP_GGUF_FAST_MATH=1`, so its generated HIP extension has a distinct +build configuration and cannot alter the conservative default path. It was +built in clean worktree `dd8bc3b` and ran the exact qualified Q4 model, the +three deterministic API controls, and the fixed 256-token throughput workload. + +| Measure | Stable baseline | C02 candidate | Change | +| --- | ---: | ---: | ---: | +| Mean decode TPS | 47.960 | 41.391 | -13.70% | +| Median decode TPS | 48.075 | 48.023 | -0.11% | +| Best sample TPS | 48.075 | 48.558 | +1.00% | +| p99 token gap | 0.02490 s | 0.02481 s | diagnostic only | +| Quality controls | 3/3 pass | 3/3 pass | unchanged | + +One of the three candidate samples contained a 3.943-second token stall. The +other two samples were approximately 48 TPS, which is indistinguishable from +the stable baseline and far below the campaign acceptance threshold. The +candidate therefore has no demonstrated decode gain, while its mean result is +materially worse because of the stall. + +**Decision: rejected.** Preserve the raw quality and benchmark artifacts at +`qwen35moe-q4-hipmath-20260901T081500Z` on GMKtek EVO-X2, but remove the experimental +compiler flag from the branch. Further work should target the measured Q4_K +and Q5_K routed-MoE vector kernels, not generic compiler flags. + +### C03: four-chunk HIP Q4_K and Q5_K vector work + +The model-shape inventory confirmed that every routed gate and up projection is +Q4_K with 512 input values and 2,048 output values, while the routed down +projection is Q5_K with 2,048 inputs and 512 outputs. The candidate doubled +the per-lane vector-dot ratio from two to four only under HIP, preserving the +packed weight layout and reduction expression while reducing the number of +chunks each lane processes. + +The exact-Q4 candidate built in clean worktree `921ec3f` and reached its +loopback endpoint. It failed all three deterministic visible-output controls: +the exact canary emitted a control token, the arithmetic control emitted an +incorrect sentence, and the JSON control was not valid JSON. No throughput +claim was measured or retained because the mandatory quality precondition +failed. + +**Decision: rejected for correctness.** The wider vector ratio changes the +kernel's coverage or reduction mapping on this HIP path. Preserve the failed +quality artifact at `qwen35moe-q4-vdr4-20260901T084500Z` on GMKtek EVO-X2, revert the +source candidate, and restore the protected normal Qwen service before the +next investigation. + +### C04: shared-activation two-row Q4_K and Q5_K routed-MoE vectors + +The next candidate retained the established two-chunk vector-dot mapping and +separate XOR reduction for each output row. Instead of changing quantization +coverage, one HIP logical wave computed two adjacent rows for a route and +shared the selected expert and Q8_1 activation address. It built in clean +worktree `3ffb1c6`, passed all three deterministic Qwen API controls, and ran +the fixed warmup plus three scored 256-token API samples. + +| Measure | Stable baseline | C04 candidate | Change | +| --- | ---: | ---: | ---: | +| Mean decode TPS | 47.960 | 47.745 | -0.45% | +| Median decode TPS | 48.075 | 47.833 | -0.50% | +| p99 token gap | 0.02490 s | 0.02489 s | diagnostic only | +| Quality controls | 3/3 pass | 3/3 pass | unchanged | + +**Decision: rejected.** Correctness was preserved, but sharing the activation +address did not offset the extra live accumulator and register pressure. The +result is below baseline and below the one-percent acceptance floor. Preserve +the artifact at `qwen35moe-q4-k2row-20260901T093500Z` on GMKtek EVO-X2 and revert the +candidate source. + +### C05: wider HIP Q8_0 vector-dot ratio + +The representative trace ranked Q8_0 vector matrix multiply as the largest +single kernel family. This HIP-only candidate changed its vector-dot ratio +from two to four, which the Q8 dot helper supports directly, while retaining +the CUDA two-group behavior. Clean worktree `dea5d6f` built successfully and +passed all three deterministic Qwen API controls. + +| Measure | Stable baseline | C05 candidate | Change | +| --- | ---: | ---: | ---: | +| Mean decode TPS | 47.960 | 47.546 | -0.86% | +| Median decode TPS | 48.075 | 47.585 | -1.02% | +| p99 token gap | 0.02490 s | 0.02606 s | diagnostic only | +| Quality controls | 3/3 pass | 3/3 pass | unchanged | + +**Decision: rejected.** The wider Q8 work ratio is numerically safe but slows +the end-to-end Q4 workload. The extra per-lane work does not repay its +occupancy and register cost on gfx1151. Preserve the artifact at +`qwen35moe-q4-q8vdr4-20260901T104200Z` on GMKtek EVO-X2 and revert the candidate. + +### C06: modern MMVQ component replacement investigation + +The prior candidates establish that changing local launch dimensions or +per-lane work ratios in the older vendored GGUF kernels does not produce a +safe gain on gfx1151. GMKtek EVO-X2 reports a 32-lane HIP warp, so the existing +32-thread logical reduction is not accidentally running at half its physical +wave width. + +Current llama.cpp has evolved from the older MMVQ donor used here into an +architecture-aware implementation. It selects launch geometry by GPU family, +uses a newer parameter table, and has a dedicated routed-expert vector path. +The relevant upstream components are `ggml-cuda/mmvq.cu` and `vecdotq.cuh` in +the current llama.cpp tree. FreeToken's GGUF extension has a narrower PyTorch +binding and different packed-bank interface, so copying the file wholesale +would be unsafe. + +The next component lane is therefore a selective port with these gates: + +1. Extract only the Q4_K, Q5_K, Q6_K, and Q8_0 vector-dot helpers plus the + routed-expert launch geometry needed by the exact Qwen model. +2. Preserve FreeToken's existing packed `[expert, row, row_bytes]` bank and + `topk_ids` interface. Do not change quantization, routing, model files, or + sampling behavior. +3. Add a model-shape microbenchmark using the actual 512-to-2,048 Q4_K gate/up + projections, 2,048-to-512 Q5_K down projection, and Q8_0 dense shapes. +4. Compare candidate tensors to the accepted kernel before API startup, then + run the deterministic API suite. Any mismatch is an immediate rejection. +5. Use the full fixed API workload, tail-latency telemetry, long-context, + multi-turn, and recovery gates before accepting a candidate. + +This is the remaining software-only path with credible headroom. The evidence +does not support promising a 50-percent single-user decode gain from it: the +current accepted FreeToken exact-Q4 result is already within 1.78 percent of +the matched llama.cpp ROCm control. Any larger claim requires measured proof, +not extrapolation from CUDA-oriented paper results. + +#### C06 baseline: exact packed-expert microbenchmark + +The new screening harness completed its initial GMKtek EVO-X2 baseline with real +packed bytes from layer 0 of the qualified Qwen GGUF. It copied the eight +routed expert slices only, used the production `ggml_moe_a8_vec` binding, and +excluded model load, HTTP, router, scheduler, and JIT time from GPU-event +measurements. + +| Projection | Quantization | Exact shape | Mean device time | +| --- | --- | --- | ---: | +| Gate | Q4_K | 512 to 2,048, eight selected experts | 21.970 microseconds | +| Up | Q4_K | 512 to 2,048, eight selected experts | 21.864 microseconds | +| Down | Q5_K | 2,048 to 512, eight selected experts | 20.624 microseconds | +| Three projections | mixed | one routed token's screen workload | 64.459 microseconds | + +This is a selection baseline, not server TPS. It makes later component work +auditable: a candidate must improve this real-shape screen and still pass all +end-to-end quality, latency, and recovery gates. The artifact is +`qwen35moe-q4kq5k-microbaseline-20260901T141100Z` on GMKtek EVO-X2. + +### C06 execution contract: selective modern MMVQ port + +The next iteration is deliberately limited to the modern llama.cpp component +surface that is relevant to this Qwen artifact: the Q4_K and Q5_K routed +expert helpers, Q6_K and Q8_0 dense helpers, and the architecture-aware MMVQ +launch selection. It does not alter the GGUF packing, router semantics, +quantization, sampling, cache capacity, model files, ROCm installation, or +normal service configuration. + +The candidate may advance only in this order: + +1. Compile in an isolated source tree with a separate ROCm kernel cache. +2. Match the accepted implementation on real packed Qwen expert tensors before + starting an HTTP server. Any element mismatch rejects the candidate. +3. Improve the real-shape device microbenchmark by at least 1 percent for a + traced hot projection without regressing another traced hot projection by + more than 1 percent. +4. Pass the three deterministic API controls, the functional quality suite, + long-context retrieval, and multi-turn state-retention suite. +5. Beat 47.960 mean decode TPS by at least 1 percent across repeated + fixed-workload API samples, with no worse p99 token gap or recovery result. +6. Run the matched llama.cpp control again only after FreeToken clears its own + acceptance gate. A useful cross-runtime win is at least 51.3 TPS, roughly + five percent above the current 48.831 TPS control, rather than an outcome + inside ordinary run-to-run variation. + +The NVIDIA NVFP4 lane remains separate. It cannot claim comparison with the +published 39.3 TPS RTX 4060 result until the authors' workload, cache, warmup, +generation, stop, source-revision, and policy fields are recovered and frozen. + +#### C06.1 source-architecture audit + +The source audit separated FreeToken's prefill and decode dispatches. The +prefill-oriented `moe.cuh` kernels already select eight 32-lane waves with 8 +by 128 tiles for the relevant ROCm formats. The qualified single-token Qwen +decode path, however, calls `ggml_moe_a8_vec`, whose `moe_vec.cuh` Q4_K, +Q5_K, Q6_K, and Q8_0 wrappers still launch one wave per block. Current +llama.cpp selects eight waves for those simple RDNA4 single-vector helpers. + +The resulting C06 candidate changes only that decode dispatcher. It retains +FreeToken's expert-bank layout, activation packing, vector-dot helpers, row +mapping, and CUDA behavior. Tensor equality remains mandatory before the +candidate can consume server benchmark time. This distinction prevents an +already-tuned prefill geometry from being confused with the still-unported +decode geometry. + +#### C06.2 decode-wave candidate screen + +The HIP-only C06 candidate was built in an isolated source tree and executed +against the real layer-0 packed expert slices from the exact Qwen Q4_K model. +It changed only the four decode wrapper launches from one 32-lane wave to +eight 32-lane waves, matching the family-specific direction seen in current +llama.cpp. CUDA behavior and all model-visible semantics remained untouched. + +| Projection | Baseline device time | C06 device time | Change | +| --- | ---: | ---: | ---: | +| Gate Q4_K | 21.970 microseconds | 23.107 microseconds | +5.18% slower | +| Up Q4_K | 21.864 microseconds | 23.531 microseconds | +7.62% slower | +| Down Q5_K | 20.624 microseconds | 21.672 microseconds | +5.08% slower | +| Three projections | 64.459 microseconds | 68.311 microseconds | +5.98% slower | + +The candidate compiled natively with the ROCm 10 runtime and used 30 warmup +iterations plus 300 measured repetitions. It failed the microbenchmark gate +before tensor-equivalence or HTTP testing was warranted: every traced +projection regressed by more than the allowed one-percent ceiling. The +likely mechanism is that these relatively small routed-expert matrices do not +provide enough parallel work to repay the added wave coordination. + +**Decision: rejected.** C06 is retained only on the isolated experimental +branch and is not merged into the AMD port. The protected GMKtek EVO-X2 Qwen +service was restarted immediately after the screen and its health endpoint +returned `status: ok` before the iteration was closed. The immutable screen +artifact is `qwen35moe-q4-c06-micro-20260901T174907Z` on GMKtek EVO-X2. + +### C07: upstream Triton-router audit + +Upstream FreeToken advanced its default router policy in commit `e05cff8`, +which routes `fused_topk` through the in-tree Triton implementation. This was +investigated as a possible scheduler-overhead improvement because routing runs +once per MoE layer during decode. + +The isolated HIP router diagnostic completed without loading a model or +changing the normal service. On the Qwen shape of 256 experts and top-k 8, +the Triton implementation matched the PyTorch reference IDs and weights in the +diagnostic and reduced synchronized router time as follows: + +| Tokens | PyTorch router | Triton router | Microbenchmark speedup | +| ---: | ---: | ---: | ---: | +| 1 | 0.02415 ms | 0.01664 ms | 1.45x | +| 4 | 0.02442 ms | 0.01811 ms | 1.35x | + +This is not a new production candidate. The present Q4 production source +intentionally keeps the PyTorch router on ROCm because a prior end-to-end +Qwen canary changed despite isolated router parity. The upstream change +modifies only the dispatch policy, not the router kernel or its arithmetic, so +it does not address that quality failure. Repeating a full API test without a +source-level numerical fix would therefore consume a protected-service window +without testing a new hypothesis. + +**Decision: rejected as already disproven.** Preserve the router timing +artifact `qwen-router-c07-20260901T180707Z` on GMKtek EVO-X2 as a diagnostic, +but retain the reference route for the exact-Q4 quality baseline. The normal +GMKtek EVO-X2 service remained on its existing configuration and returned +`status: ok` after the diagnostic. + +#### C07 correction and C08-C09 quality requalification + +The C07 dispatch conclusion was superseded by a source and deployment audit. +The active service source was older than the branch under test, while the +current branch's router policy selects the in-tree HIP Triton router by +default. The upstream router-policy change is therefore not an untested new +kernel, but it does expose a branch-versus-deployment qualification gap that +had to be closed before making a performance claim. + +First, a quality-only isolated Q4 window using the current branch's router +path passed all three deterministic controls. A second isolated window used a +clean router-only checkout, excluding the rejected decode-wave modification, +and expanded the test scope: + +| Control | Result | Key evidence | +| --- | --- | --- | +| Exact response, arithmetic, structured JSON | 3 of 3 pass | deterministic visible outputs | +| Multi-turn state retention | 3 of 3 pass | maximum TTFT 0.846 s; p99 token gap 24.20 ms | +| Fresh-prefix retrieval | 1 of 1 pass | 1,556 reported prompt tokens; TTFT 5.344 s | +| Higher-context fresh-prefix retrieval | 1 of 1 pass | 5,656 reported prompt tokens; TTFT 22.594 s | + +The higher-context control is below the 8,192-token serving limit because its +single request must also reserve output capacity. A 16K control is outside +this qualified server configuration and is not represented as a passed test. + +The C09 window also discovered that an older recovery launcher had created a +healthy but non-dedicated process group. The controller refused to stop it, +as designed. After verifying that the group contained only the FreeToken +frontend and its own worker children, it was replaced using the dedicated +`setsid` launcher. The restored service reported `status: ok`, and its server +PID, process-group ID, and session ID were all identical. This repair is a +reliability prerequisite for later time-share tests, not a throughput result. + +**Decision: performance eligible, not yet accepted.** The current HIP Triton +router configuration has cleared the available deterministic, state, long- +context, and recovery gates. It must still complete the fixed five-sample API +matrix and concurrent workload with a fresh exact-Q4 reference before it can +replace the 47.960-TPS baseline. Preserve the quality artifacts +`qwen-router-c08-quality-20260901T181143Z` and +`qwen-router-c09-full-quality-20260901T183253Z` on GMKtek EVO-X2. + +### C10: router-only exact-Q4 API and concurrency matrix + +The clean router-only checkout then completed the fixed five-sample API matrix +and the three-round concurrency controls. This is the same checkout that +passed C09 quality, with the rejected decode-wave experiment excluded. The +benchmark used the exact Q4 model, a fixed 48-line prompt, a 256-token single +decode, and the model's valid tokenizer. It was served only on the isolated +loopback candidate port while the ordinary NVFP4 API was stopped by the +recovery controller. + +| Workload | Mean throughput | p99 TTFT | p99 token gap | +| --- | ---: | ---: | ---: | +| Single request, five samples | 48.282 decode tokens/s | 0.432 s | 24.26 ms | +| Concurrent 1, three rounds | 40.138 aggregate tokens/s | 0.436 s | 40.67 ms | +| Concurrent 2, three rounds | 57.913 aggregate tokens/s | 0.854 s | 53.63 ms | +| Concurrent 4, three rounds | 81.456 aggregate tokens/s | 1.174 s | 76.45 ms | + +The single-request result is 0.67 percent above the accepted 47.960-token/s +baseline. That is within normal run-to-run variation and below the campaign's +minimum promotion gate of a repeatable one percent gain. The candidate is +therefore quality-qualified and load-stable, but it is not a new performance +baseline and must not be promoted on this evidence alone. + +The controller stopped the candidate, restarted the ordinary NVFP4 API, and +verified `status: ok`. The recovered server PID, process-group ID, and session +ID were identical, and no listener remained on the candidate port. Preserve +the complete artifact `qwen-router-c10-api-20260901T185412Z` on GMKtek EVO-X2. + +**Decision: do not promote.** Retain the current HIP Triton router as a +quality-qualified route, but focus the next iteration on data movement and +expert-cache work, where an end-to-end gain remains plausible. + +### C11: upstream expert-cache copy-plan audit + +Upstream's newer expert-cache copy-plan changes were merged only into a +disposable source checkout. The candidate preserves the qualified in-tree +router path and adds the upstream copy-plan and AOT-catalog corrections. Two +stale test expectations were found and corrected in that disposable checkout: +the router test expected a retired reference dispatch policy, and the AOT test +expected two unsupported legacy copy shapes to be compiled even though the +upstream code intentionally filters them out. + +The corrected CPU-side controls passed: three host-residency and locked-layer +copy tests, plus two strict AOT-grid selection tests. The normal NVFP4 API +reported `status: ok` after the checks. The saved CPU evidence is +`upstream-cache-c11-retry-20260901T191217Z` on GMKtek EVO-X2. + +However, the focused ROCm fused-MoE suite also produced an illegal-memory- +access fault in `fused_moe_kernel` while testing the disposable candidate. +The failed test process was a separate one-process group and was terminated; +GPU utilization returned from 100 percent to 5 percent, and the ordinary API +remained healthy. This failure cannot be attributed to the copy-plan change +because that fused-expert test path was not changed by the upstream copy-plan +commit. It is nevertheless a real safety failure on the target ROCm stack. + +**Decision: safety-blocked.** Do not merge or benchmark the upstream cache +candidate yet. The next iteration must stop the normal service, reproduce the +fused-expert fault with serialized kernel dispatch and a minimal shape, compare +the candidate with the accepted source, then repair or replace the failing +kernel before any cache-copy throughput claim is considered. + +### C12: accepted-source fused-MoE fault reproduction + +The mandatory isolated reproduction was run against the accepted source, +not the upstream cache candidate. The ordinary NVFP4 API was stopped through +the recovery controller, and the smallest failing test was launched in its own +session with serialized ROCm dispatch. The test failed in 3.42 seconds with a +HIP illegal-memory-access fault in `fused_moe_kernel` on the float16 grouped +MoE shape: 4 tokens, 37 experts, hidden size 32, intermediate size 24, and +top-k 4. The exact source revision, test output, exit status, stop log, and +recovery log are saved in `fused-moe-c12-baseline-20260901T191320Z` on GMKtec +EVO-X2. + +This establishes that the failure predates the upstream cache-copy candidate. +It also rules out a simple test-runner race because serialized dispatch reports +the same faulting `fused_moe_kernel`. The normal NVFP4 API was restored by the +controller and again returned `status: ok` after its normal cold load. + +**Decision: repair prerequisite confirmed.** The next code change must add a +ROCm-safe grouped-MoE selection or correct the kernel bounds issue for this +shape, backed by the isolated reproducer. The upstream cache candidate remains +on hold until that repair passes and no longer faults the target ROCm runtime. + +### C13-C20: grouped-MoE root cause, repair, and candidate safety checks + +The first repair narrowed the second grouped projection to its actual flattened +storage layout. That removes a real stride and bounds hazard, but the minimal +reproducer still failed at the first projection. The next diagnostic compared +the two alignment implementations directly. The compact alignment kernel +returned valid sorted token IDs while assigning every padded block to expert +zero, even when the routed experts were distinct. Its outputs therefore could +not safely select grouped expert weights on this AMD runtime. + +The ROCm path now selects the staged in-tree alignment implementation, which +returned the expected distinct expert IDs for the same route. The repaired +minimal reproducer passed, followed by the four-shape parity group and a new +regression test that asserts every routed expert is represented in the padded +alignment output. The saved GPU artifacts are +`fused-moe-c13-stride-20260901T192*Z`, +`fused-moe-c14-align-20260901T193029Z`, +`fused-moe-c15-alt-align-20260901T193812Z`, +`fused-moe-c16-align-fix-20260901T194538Z`, +`fused-moe-c17-full-parity-20260901T195315Z`, and +`fused-moe-c18-regression-suite-20260901T200025Z` on GMKtek EVO-X2. + +The candidate containing the upstream cache-copy plan was then merged with the +repair into isolated source revision `340ed31`. Its focused safety suite +passed 5 tests with 34 intentionally deselected, and its direct fused-copy +comparison matched the legacy per-bank copy for 0, 1, 4, and 8 cache misses. +Those artifacts are `upstream-cache-c19-safety-20260901T200857Z` and +`upstream-cache-c20-fused-copy-20260901T201650Z`. + +**Decision: safety gate passed, performance gate not yet passed.** The repair +is eligible for model quality requalification. No throughput claim follows +from these unit and direct-copy tests alone. + +### C21-C22: revision-matched reusable HIP cache + +The integrated candidate had no reusable cache for source revision `340ed31`. +The first maintenance wrapper found a helper-file execute-bit defect before it +ran the builder, so it produced no benchmark result and recovery was corrected +immediately by invoking the reviewed helpers through Bash. The normal service +then completed its measured serial cold recovery in 5 minutes 57 seconds. + +The corrected isolated build compiled all 82 explicit C++ and HIP cache modules +for AMD Radeon 8060S Graphics with HIP `7.15.26333`, writing them under +`kernel-cache-rocm-gfx1151-340ed31`. The subsequent verifier loaded all 82 +modules with `FREETOKEN_DISABLE_JIT=1` and reported `status: passed`. The +artifact `upstream-cache-c22-build-20260901T203402Z` retains the build log, +strict verifier output, and recovery record on GMKtek EVO-X2. + +**Decision: reusable-cache gate passed.** Future runs of this exact candidate +must point at this revision-matched cache and retain JIT disabled. This avoids +per-run native kernel compilation without pretending that a cache from a +different source revision is ABI-safe. The Q4 model itself is not recompiled +by this process. + +### C23-C25: integrated cache and cache-residency measurements + +The repaired upstream cache-copy candidate at source revision `340ed31` passed +the deterministic three-case Q4 quality suite, multi-turn state controls, and +fresh-prefix retrieval at 1,556 and 5,656 reported prompt tokens. Its +revision-matched reusable HIP cache was loaded with `FREETOKEN_DISABLE_JIT=1`. +The three-sample warm API matrix measured 47.848 decode tokens/s at memory +ratio 0.25, 0.23 percent below the accepted 47.960-token/s Q4 baseline. + +Decode cache telemetry reported 40,800 layer calls with eight active experts +per layer and a 7.45 percent miss rate. Raising the cache-residency ratio from +0.25 to 0.30 increased resolved cache slots from 5,470 to 7,051 but reduced +the three-sample mean to 47.032 decode tokens/s, 1.94 percent below baseline. +The 0.30 candidate preserved all three deterministic quality checks; its warm +TTFT mean was 0.454 s and token-gap p99 was 24.01 ms. + +**Decision: reject cache-copy and larger-residency as throughput routes.** The +copy candidate is safe and quality-preserving, but neither cache transfer nor +additional resident experts produces a measurable decode gain on this Q4 +workload. The low remaining miss fraction also makes a 50 percent gain from +cache sizing implausible. Preserve `upstream-cache-c23-q4-quality-20260901T204241Z`, +`upstream-cache-c24-cache-telemetry-20260901T205755Z`, and +`upstream-cache-c25-r030-20260901T210838Z` on GMKtek EVO-X2. Each candidate +was stopped and the normal NVFP4 API recovery controller was started after its +window. + +### C26: single-stream decode graph capture + +The existing Q4 launcher deliberately disabled decode graph replay. A new, +default-off `CUDA_GRAPH_MAX_BS` launcher parameter makes a bounded graph +experiment explicit and reproducible without changing the protected normal +service. The isolated Q4 candidate captured batch size one successfully, +consuming approximately 0.25 GiB of additional GPU-visible memory and leaving +22.81 GiB free after capture. + +The first quality attempt was invalid: the harness was given the API root +instead of the required OpenAI-compatible `/v1` path, so all three requests +received HTTP 404 before model inference. No TPS result was collected and the +candidate must not be classified as a quality or performance failure. The +candidate was stopped through its verified dedicated process group, no test +listener remained, and normal-service recovery was started. + +**Decision: graph candidate remains pending.** Re-run the deterministic suite +and fixed TPS matrix against the corrected `/v1` endpoint after verified normal +service recovery. The failed endpoint artifact +`q4-c26-graph-bs1-20260901T212038Z` remains part of the provenance record as a +harness-configuration failure. The candidate startup also rebuilt its +checkout-local GGUF HIP extension, not the GGUF model. A later promotion +requires a revision-matched reusable extension cache and a strict no-JIT +verification for that checkout. + +### C27: corrected single-stream graph qualification + +The graph candidate was repeated after normal-service recovery with the +OpenAI-compatible `/v1` endpoint. It passed all three deterministic controls, +captured batch size one in 3.26 seconds using the already-built checkout-local +GGUF HIP extension, and kept 22.81 GiB of GPU-visible memory free after graph +capture. + +The fixed short canary matrix measured 49.477 decode tokens/s across three +samples, 3.16 percent above the accepted 47.960-token/s baseline. Its mean +warm TTFT was 0.343 s and token-gap p99 was 24.54 ms. This result alone was +not sufficient for promotion. The independent scheduler-shaped three-sample +matrix measured 48.278 decode tokens/s, essentially equal to the previously +qualified non-graph control at 48.282 decode tokens/s. Its mean warm TTFT was +0.430 s and token-gap p99 was 24.41 ms. + +**Decision: reject graph replay as a throughput promotion.** It is functionally +correct on the AMD path, but its apparent short-canary gain did not reproduce +on the scheduler-shaped workload and therefore fails the campaign requirement +for a repeatable gain above normal variation. Keep the new launcher parameter +defaulted to zero for reproducible future investigation, but do not enable it +for normal service. Preserve `q4-c27-graph-bs1-v1-20260901T213143Z` on +GMKtek EVO-X2. The verified candidate process group was stopped, its loopback +ports were clear, and normal NVFP4 service recovery was started. + +### C28: two-row GGUF MMV grouping screen + +The prior ROCm trace showed that Q4_K and Q5_K routed-expert vector kernels +dominate decode GPU time. A narrow HIP compile-time experiment therefore +made the MMV output-row grouping explicit. The default remains one row, while +the isolated candidate used exactly two rows per workgroup through +`FREETOKEN_GGUF_MMV_Y=2`. Host-side validation tests passed 3 of 3 before GPU +use. The candidate compiled into a dedicated extension-cache directory, and +the HIP build log records `-DGGML_CUDA_MMV_Y=2` for `gfx1151`. + +The two-row candidate passed all three deterministic Q4 quality controls, but +its fixed three-sample throughput mean was 47.954 decode tokens/s. This is +effectively equal to, and fractionally below, the 47.960-token/s baseline. +Mean warm TTFT was 0.350 s and token-gap p99 was 24.76 ms. + +**Decision: reject the two-row MMV grouping.** The configuration preserves +quality but does not create a measurable single-stream decode gain. Keep the +compile switch defaulted to one row and retain it only as a reproducible +diagnostic control. Preserve `q4-c28-mmv-y2-20260901T214526Z` on GMKtec +EVO-X2. The candidate was stopped through its verified process group, its +loopback ports were confirmed clear, and normal NVFP4 service recovery was +started. + +### C29: llama.cpp RDNA4 eight-wave MMVQ policy screen + +The current llama.cpp ROCm implementation selects eight independent Wave32 +rows for simple one-vector MMVQ formats on RDNA4, including Q4_K, Q5_K, Q6_K, +and Q8_0. FreeToken's vendored Q4_K vector-dot arithmetic is otherwise the +same as the reference implementation, so this candidate changed only the +HIP-only launch geometry for those traced kernel families. The default +one-row launch remains unchanged. The candidate required the explicit +`FREETOKEN_GGUF_MMV_Y=8` build setting and compiled into its own revisioned +extension-cache directory with `-DGGML_CUDA_MMV_Y=8` for `gfx1151`. + +The candidate source revision `05751ef` passed all three host-side validation +tests, then passed the deterministic Q4 API controls: exact canary, arithmetic, +and JSON schema. The first quality request included HIP extension compilation +and is retained as startup evidence only. It was excluded from steady-state +throughput. The subsequent warm fixed scheduler matrix produced 48.374, +48.103, and 48.357 decode tokens/s, for a 48.278 mean and 48.357 median. +That is a 0.66 percent mean increase over the accepted 47.960-token/s +baseline, below the campaign's one-percent promotion floor. Warm TTFT averaged +0.425 s and token-gap p99 was 24.91 ms. The retained per-sample prompt-token +and TTFT fields also yield a client-visible prefill rate of 2,852.315 prompt +tokens/s mean, with a 2,846.519 to 2,856.972 range. This is an end-to-end +prompt-to-first-text measurement, not the server's narrower internal input +throughput counter. + +**Decision: reject the eight-wave MMVQ policy for promotion.** It preserves the +screened output quality and is modestly faster in this one matrix, but the +measured increase is too small to distinguish safely from host variation and +does not meet the repeatable-gain requirement. Do not merge the candidate +branch or change the default. Preserve `q4-c29-rdna4-mmvq8-20260901T215745Z` +on GMKtek EVO-X2, including raw quality, per-token timing, HIP build, and +recovery evidence. The isolated process was stopped; a stale executable bit +on the normal recovery start helper was corrected before normal-service +recovery was launched. + +### C30: ROCprof lifecycle qualification + +The next optimization decision requires a kernel and memory-copy trace of the +same Q4 control, not an inference-rate estimate from a profiler. ROCprofv3 +was therefore launched through the ROCm SDK bundled with the active PyTorch +wheel. That avoids loading a second LLVM runtime from the system ROCm tree. +The normal NVFP4 server was stopped only after a healthy API check and was +restarted after every isolated candidate attempt. + +The first attempt exposed two setup defects before inference: a source checkout +that did not register the Qwen GGUF architecture, followed by a Qwen-capable +checkout without its required native pinned-memory extension. The native +extension was then built from that checkout with the installed ROCm 10 HIP +toolchain and successfully imported. This is build provenance, not a model +conversion or a change to the protected service. + +The next run reached the Q4 OpenAI-compatible API and built the checkout-local +GGUF HIP kernel on its first real request. Its resulting decode output was +intentionally excluded from performance comparison because the profiler uses a +system-memory intercept queue and the request included one-time compilation. +A warm-cache repetition completed one 900-token bounded workload in the +configured collection interval. The server's diagnostic decode log was about +30 tokens/s under profiling, versus the qualified unprofiled control near 48 +tokens/s. This demonstrates that ROCprof output must not be used as a TPS +measurement. + +**Decision: no C30 performance claim.** The candidate and all verified helper +processes were stopped, the disposable loopback listener was confirmed absent, +and the normal NVFP4 API was healthy again. The forced recovery prevented the +profiler from finalizing a usable `rocpd` database, so the trace cannot yet be +used to rank kernels or justify a code change. Preserve +`q4-c30d-profile-default-20260901T223610Z`, +`q4-c30e-profile-default-20260901T223951Z`, and +`q4-c30f-profile-warm-cache-20260901T224449Z` as provenance. The next action +is a documented controller that prewarms the exact isolated cache, runs a +bounded workload during collection, requests graceful profiler finalization, +and verifies the resulting database before normal-service recovery. + +### C32: finalized warm-cache ROCprof trace + +The revised controller prewarmed the exact Q4 server before profiling, waited +for the scheduler's explicit ready record instead of treating the frontend +models endpoint as inference-ready, and then ran one bounded 900-token request. +It produced a finalized 1.44 GiB `rocpd` SQLite database and the normal NVFP4 +API was healthy again after the time-share recovery. The profiler's own queue +intercept mode changes runtime behavior, so none of these diagnostic values are +used as TPS measurements. + +The final 30-second active-dispatch window identifies the decode work that must +be optimized. The dominant entries were Q8 vector dot at 2,011.326 GPU ms, +Q4_K routed-expert vector dot at 529.036 GPU ms, Q6_K vector dot at 474.200 +GPU ms, and Q5_K routed-expert vector dot at 395.507 GPU ms. The next largest +non-vector components were attention GEMM at 232.153 GPU ms, delta-rule fused +gating at 173.428 GPU ms, grouped decode stage one at 82.297 GPU ms, and cache +index copying at 74.211 GPU ms. The trace recorded no memory-copy events in +this final window. + +**Decision: prioritize the traced GGUF vector-dot path.** The existing cache +residency and graph experiments cannot plausibly deliver the campaign target by +themselves. Future candidates must preserve the qualified Q4 output checks, +change one vector-kernel dispatch or arithmetic behavior at a time, and pass +two independent throughput matrices before promotion. Preserve +`q4-c32-rocprof-controller-ready-20260901T225400Z`, including the raw SQLite +database, workload response, controller logs, and normal-service recovery +evidence. +### C33: current-branch MMV-Y4 requalification + +The profiler-ranked vector path was rechecked against the current branch rather +than relying on the older candidate artifact. The opt-in +`FREETOKEN_GGUF_MMV_Y=4` build from commit `ff76ede` reached API readiness in +an isolated checkout with 56 GiB free before model loading and 23.07 GiB free +after initialization. Its fixed three-sample scheduler-shaped matrix passed +all requests and measured 45.4603 mean decode TPS, 2,753.3639 mean +client-observed prefill TPS, 0.0927 decode-TPS standard deviation, and 43.781 +ms maximum token gap. The canonical Q4 AIME output hash was `3302eda43396`. +The verifier now selects that contract explicitly with +`--expected-sha1 3302eda43396`; the historical paper-inspired +`0acef4eab6f4` contract remains available as the default. +The prior protected-service re-anchor `cd580f4978fb` used a separate source or +request contract. Under the explicit current verifier contract, the healthy +protected service returned `0acef4eab6f4`, so this remains a source-matched +historical Q4 result rather than a universal active NVFP4 quality fingerprint. + +**Decision: reject current-branch MMV-Y4 for promotion.** It was approximately +5.8 percent slower than the accepted current Q4 scheduler control near 48.28 +decode TPS. The default remains one row, and the Y4 switch remains opt-in for +future architecture-specific investigation. Normal Qwen service recovery was +verified with `status: ok` and `maintenance: serving`. diff --git a/docs/gmktec-evo-x2-upstream-handoff-checklist.md b/docs/gmktec-evo-x2-upstream-handoff-checklist.md new file mode 100644 index 0000000000..9a8fd5a658 --- /dev/null +++ b/docs/gmktec-evo-x2-upstream-handoff-checklist.md @@ -0,0 +1,108 @@ +# Upstream handoff checklist for the native ROCm/HIP port + +This checklist is for reviewing the AMD `gfx1151` port in PR #260. It keeps +the implementation review, local reproducibility, quality evidence, and +performance claims separate. + +## Source and build review + +- [ ] Check out branch `amd-rocm-gfx1151` from the contributor fork. +- [ ] Use PyTorch `2.13.0+rocm10.0.0` with HIP `7.15.26333`. +- [ ] Confirm the active device reports AMD `gfx1151`. +- [ ] Confirm `torch.version.hip` selects the ROCm build path even when a CUDA + toolkit is installed on the same host. +- [ ] Build the native host extensions and confirm they link against + `libamdhip64` rather than `libcudart`. +- [ ] Confirm CUDA-only launch options, PTX paths, and NVIDIA capability + probes are gated away on HIP. +- [ ] Confirm the GGUF JIT path discovers ROCm Thrust headers and the HIP + runtime without requiring a CUDA toolkit. + +## Functional validation + +- [ ] Start the local OpenAI-compatible API with a supported Qwen model. +- [ ] Complete streaming and non-streaming text requests. +- [ ] Verify deterministic canary, arithmetic, JSON, multi-turn, and state + retention controls. +- [ ] Start the Gemma 4 GGUF path in an isolated candidate process. +- [ ] Complete the arithmetic text gate and the documented image fixtures. +- [ ] Restore the normal Qwen service after candidate teardown. +- [ ] Confirm the normal service returns `status: ok` and a real completion + with `finish_reason: stop`. + +## Performance evidence boundaries + +- [ ] Use the machine-readable manifest in + `gmktec-evo-x2-cross-model-manifest-20260905.json`. +- [ ] Preserve prompt length, completion cap, warmup policy, concurrency, and + cache state for every comparison. +- [ ] Treat client-observed prefill, decode, TTFT, and token-gap metrics as + separate measurements. +- [ ] Do not compare cold-start TTFT with a loaded-runtime TTFT. +- [ ] For the Qwen Q4_K_M same-format control, use the exact checkpoint and + tokenizer recorded in the manifest. +- [ ] Report the warmed requests 2 through 5 separately from the first + request. +- [ ] Treat the current Qwen result as decode near-parity, not a material + universal performance lead. +- [ ] Report Gemma single-client and concurrent results separately because the + runtime ranking changes with concurrency. + +## Reliability evidence + +- [ ] Review the completed Qwen 1,440-session endurance artifact. +- [ ] Confirm candidate and host swap telemetry remained zero for the accepted + endurance run. +- [ ] Confirm malformed JSON, failed quality markers, and incomplete sessions + are absent from the accepted artifact. +- [ ] Review the bounded Gemma endurance, long-context, and concurrency + artifacts. +- [ ] Do not reinterpret the endurance suite as a prefill-TPS measurement. + +## Claim boundaries + +- [ ] Keep strict NVIDIA paper comparison marked unresolved because the exact + fixtures, policies, and reference hardware are unavailable. +- [ ] Keep 284B interactive serving marked unresolved because the exact model + payload is absent and the available memory configuration is materially + different from the paper system. +- [ ] Keep archived model routing names separate from admitted model payloads. +- [ ] Do not add llama-swap integration to the ROCm MVP unless separately + reviewed and requested. +- [ ] Keep CUDA graph capture disabled on the HIP MVP unless a new quality and + stability qualification is completed. + +## Evidence index + +- `gmktec-evo-x2-final-campaign-report.md` +- `gmktec-evo-x2-campaign-completion-audit.md` +- `gmktec-evo-x2-cross-model-matrix-20260904.md` +- `gmktec-evo-x2-cross-model-manifest-20260905.json` +- `gmktec-evo-x2-amd-run-log.md` +- `gmktec-evo-x2-284b-capacity-manifest-20260904.md` +- `gmktec-evo-x2-paper-model-capacity-gate.md` +- `gmktec-evo-x2-deepseek-capacity-gate-result-20260905.json` +- `gmktec-evo-x2-deepseek-expert-slice-metadata-20260905.json` +- `scripts/gmk-evo-x2/deepseek_expert_slice_benchmark.py` +- `gmktec-evo-x2-deepseek-expert-slice-result-20260905.json` +- `gmktec-evo-x2-deepseek-expert-slice-16-result-20260905.json` +- `gmktec-evo-x2-deepseek-expert-slice-2layer-result-20260905.json` +- `expert-route-group-0.json` +- `expert-route-group-16.json` +- `expert-route-group-32.json` +- `expert-route-group-64.json` +- `gmktec-evo-x2-deepseek-route-transfer-projection-20260905.json` +- `scripts/gmk-evo-x2/deepseek_route_transfer_projection.py` + +The checklist is a review aid. The raw benchmark artifacts remain the +authoritative evidence for measured claims. + +## Local handoff audit + +The final local audit passed on 2026-09-05: + +- The branch worktree is clean and synchronized with its remote branch. +- `git diff --check` passed. +- The machine-readable manifest parses as valid JSON. +- Every relative evidence link in this checklist and the final campaign report + resolves to a tracked local file. diff --git a/docs/models.md b/docs/models.md index e4850a1241..c9499163f9 100644 --- a/docs/models.md +++ b/docs/models.md @@ -7,10 +7,12 @@ for them; other checkpoints of the same architectures work too. | Model | HF checkpoints | |---|---| | DeepSeek-V4 | [deepseek-ai/DeepSeek-V4-Flash-0731](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731) | +| GLM-5.3-Flash | [RedHatAI/GLM-5.3-Flash-NVFP4](https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4) | | GLM-5.2 | [nvidia/GLM-5.2-NVFP4](https://huggingface.co/nvidia/GLM-5.2-NVFP4) | | GLM-4.7 | [nvidia/GLM-4.7-NVFP4](https://huggingface.co/nvidia/GLM-4.7-NVFP4) | +| Qwen3.8-Flash-Next | [Qwen/Qwen3.8-Flash-Next-FP8](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8), [RadixArk/Qwen3.8-Flash-Next-NVFP4](https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4) | | Qwen3.6 / Qwen3.5 MoE | [Qwen/Qwen3.6-35B-A3B](https://huggingface.co/Qwen/Qwen3.6-35B-A3B) ([-FP8](https://huggingface.co/Qwen/Qwen3.6-35B-A3B-FP8)), [nvidia/Qwen3.6-35B-A3B-NVFP4](https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4), [Qwen/Qwen3.5-35B-A3B](https://huggingface.co/Qwen/Qwen3.5-35B-A3B) ([-FP8](https://huggingface.co/Qwen/Qwen3.5-35B-A3B-FP8)) | -| Qwen3.6 dense | [Qwen/Qwen3.6-27B](https://huggingface.co/Qwen/Qwen3.6-27B) ([-FP8](https://huggingface.co/Qwen/Qwen3.6-27B-FP8)), [nvidia/Qwen3.6-27B-NVFP4](https://huggingface.co/nvidia/Qwen3.6-27B-NVFP4) | +| Qwen3.8 / Qwen3.6 dense | [Qwen/Qwen3.8-27B](https://huggingface.co/Qwen/Qwen3.8-27B) ([-FP8](https://huggingface.co/Qwen/Qwen3.8-27B-FP8)), [RadixArk/Qwen3.8-27B-NVFP4](https://huggingface.co/RadixArk/Qwen3.8-27B-NVFP4), [Qwen/Qwen3.6-27B](https://huggingface.co/Qwen/Qwen3.6-27B) ([-FP8](https://huggingface.co/Qwen/Qwen3.6-27B-FP8)), [nvidia/Qwen3.6-27B-NVFP4](https://huggingface.co/nvidia/Qwen3.6-27B-NVFP4) | | Qwen3-MoE | [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) | | gpt-oss | [openai/gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b), [openai/gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b) | | Gemma-4 | [google/gemma-4-26B-A4B-it](https://huggingface.co/google/gemma-4-26B-A4B-it), [nvidia/Gemma-4-26B-A4B-NVFP4](https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4), [google/gemma-4-12B-it](https://huggingface.co/google/gemma-4-12B-it), [nvidia/Gemma-4-31B-IT-NVFP4](https://huggingface.co/nvidia/Gemma-4-31B-IT-NVFP4) .. | @@ -37,4 +39,5 @@ for them; other checkpoints of the same architectures work too. FreeToken's fast-load format, and `ft serve --model` auto-detects the result. - DeepSeek-V4 checkpoints must keep the `inference/config.json` subdir — the authoritative model args are read from there. +- Qwen3.8-Flash-Next keeps a 47.7 GiB PLE n-gram table pinned in host RAM. - Multimodal checkpoints are served text-only. diff --git a/docs/qwen-swap-validation.md b/docs/qwen-swap-validation.md new file mode 100644 index 0000000000..0b0067c432 --- /dev/null +++ b/docs/qwen-swap-validation.md @@ -0,0 +1,63 @@ +# Dense Qwen GGUF swap validation + +This candidate targets the FreeToken AMD branch. It supports the resident Qwen3.6 27B and Qwen3.8 27B Q4_K_M layouts by retaining independent packed projection types, restoring GDN value-head order, and mapping dense `qwen35` tokenizer metadata. It is not a general qualification of every Qwen checkpoint or quantization recipe. + +## Repair + +The old loader concatenated packed QKV and gate tensors even when their row-byte formats differed. The dense path now takes each attention/GDN projection type from the GGUF tensor descriptors and constructs separate native operators. This is required for both Q6_K/Q4_K and Q8_0 gate combinations. Full-attention Q, K, and V also retain their separate formats. + +GDN output weights remain byte-exact in their quantized blocks. The activation is regrouped before the output projection rather than attempting to move part of a quantization block. The legacy MoE fused-QKV path and its expert-cache ownership remain separate. A dense expert-only load phase is an intentional no-op. + +The change preserves unrelated current model configuration fields, including other model families' configuration payloads. Reusing an older candidate's complete configuration file would have removed those fields, so only the new GGUF descriptor field is added to the current base. + +## Verified results + +Tests were run on GMKtek EVO-X2 using an isolated source checkout, not the protected inference service's files. + +The model SHA-256 checksums were independently verified after testing: + +- Qwen3.6-27B-Q4_K_M.gguf: `33625d8dc3a5dd8d88c324d47db58561b11f7072816287078bfe58b4c55782f9`. +- Qwen3.8-27B-Q4_K_M.gguf: `31629f53165ab6a7dad8c9847dcfd1fdf55829dac1e6e748f4a68581b0033d34`. + +| Gate | Result | +| --- | --- | +| Model metadata, GDN packing combinations, head-order tests | 21 passed | +| Qwen3.6 exact tensor-name/shape/dtype contract | Passed on CPU/meta | +| Qwen3.8 exact tensor-name/shape/dtype contract | Passed on CPU/meta | +| Qwen3.6 tokenizer text round-trip | Passed | +| Qwen3.8 tokenizer text round-trip | Passed | +| Public-document privacy and benchmark regression tests | 27 passed | +| Qwen3.6 through llama-swap, ordinary response | `4`, 39.00 seconds including load | +| Switch to Qwen3.8, SSE response | `4` and `[DONE]`, 41.10 seconds including switch | +| Switch back to Qwen3.6, SSE response | `4` and `[DONE]`, 38.25 seconds including switch | +| Protected service restoration | Health and deterministic completion passed | + +A second, extended pass repeated A-to-B-to-A successfully in 35.99, 44.17, and 33.17 seconds. It also passed same-model concurrent requests, different-model concurrent requests, streamed usage-block checks, and five-second idle eviction. The private artifact set is `freetoken-swap-live-20260910-e`. The protected service was restored and verified, and the candidate listeners were closed. These timings include load/switch overhead and should not be used as decode throughput. + +The three live requests used temperature 0 and a 32-token output limit. Each asked for the single-digit answer to 2 + 2. These are deterministic smoke tests, not a broad reasoning benchmark. The request durations include model startup or switching and are not decode throughput or isolated time-to-first-token measurements. + +The runtime used a 4096-token sequence limit, 4096-token cache allocation, 512-token prefill bound, one concurrent backend request, graph batch size 1, Triton attention, fused dense execution, and disabled PyNCCL. The Qwen3.6 cache allocation was 0.25 GiB. This does not establish 64K operation, multi-GPU support, or MoE checkpoint quality. + +## Swap integration requirements + +The successful live run used an unmodified llama-swap binary built from `41ec321b6216d838488b2a7d936274ed227c0c5e`, plus FreeToken's `/ready` endpoint from the separate swap control-plane PR. The latter returns HTTP 503 while loading and 200 when accepting requests. FreeToken's diagnostic `/health` alone is not a compatible llama-swap readiness signal. + +The real Python CLI module is `python -m freetoken.cli serve`. The legacy `python -m freetoken` entrypoint starts a server directly and does not accept the `serve` subcommand. A configuration that mixes those forms exits during argument parsing. + +Each qualification run used its own `TORCH_EXTENSIONS_DIR`. A stale lock in the shared extension cache had caused a graph-preparation stall; the shared cache was not deleted or modified. GGUF kernels were then compiled and imported in the private cache before the protected service was stopped. The host extensions were built from the isolated source, rather than copied from an unverified checkout. + +For streamed token metrics, request `stream_options: {"include_usage": true}`. A valid stream without a usage block can still generate llama-swap's misleading metrics warning about missing valid JSON. The generated content and `[DONE]` framing passed in the initial run; usage reporting is a separate integration check. + +## Evidence and remaining limits + +The private artifact set `freetoken-swap-live-20260910-d` contains the configuration, native kernel build log, proxy/backend log, three raw responses, baseline response, recovery response, and structured results. These raw artifacts are intentionally not committed because they include operational paths and process details. + +An additional approved window passed real Qwen3.6 stream cancellation: the same backend reached zero active requests without increasing its normal-completion count, with observed terminal abort 0.254 seconds after disconnect. Post-cancellation Qwen3.6-to-Qwen3.8-to-Qwen3.6 streaming, concurrent routing, and idle eviction passed. A separate native daemon test rejected a private invalid GGUF fixture, automatically restored Qwen3.6, reached readiness, and generated the expected streamed answer. The prior engine's sealed accounting receipt was complete; the failed loader's crash receipt was explicitly degraded. Both phases restored and verified generation from the protected llama.cpp service. Logical private artifacts are `freetoken-swap-live-20260910-f` and `freetoken-native-recovery-20260910-a`. + +Broader quality testing and longer reliability runs remain necessary before broad production promotion. The bounded cancellation and loader-failure recovery results are not guarantees for every failure mode or model. Existing semaphore-cleanup warnings remain a follow-up; final checks found no test listeners or FreeToken multiprocessing workers. No production configuration was changed or permanently activated, and no change was submitted to llama.cpp or llama-swap. + +## Privacy + +Current public AMD reports use GMKtek EVO-X2, placeholder operator paths, and documentation-only IP addresses. Benchmark launchers derive the invoking user's home directory instead of embedding a personal username; most root-directory defaults also accept `FREETOKEN_ROOT_DIR`. When invoking under a different account, explicitly set the intended root directory. Historical commits and previously generated binary publications are not erased by these working-tree changes. + +Benchmark API clients now require `--expected-host`; host-specific shell wrappers require an explicitly configured `FREETOKEN_EXPECTED_HOST` where they previously embedded the machine hostname. This preserves the host safety check and fails closed if no target is selected. Manuscript contact metadata and the tracked review PDF are anonymized in this branch. Repository-owner URLs, licenses, and third-party attribution remain intact. The separate checkout's in-progress manuscript and PDF edits are preserved, not overwritten by this review copy. diff --git a/docs/reproducibility.md b/docs/reproducibility.md new file mode 100644 index 0000000000..ccec87cb8e --- /dev/null +++ b/docs/reproducibility.md @@ -0,0 +1,134 @@ +# Reproducibility and independent extension + +This guide is the public counterpart to the recorded evidence workflow. It is +for contributors who want to build, validate, benchmark, or extend the native +ROCm/HIP FreeToken port on their own AMD system. It does not require the evaluated +host, an internal address, or a production service. + +## Scope + +The public artifact proves only the experiment it records. It must not be used +to infer strict paper replication, general AMD performance, or quality parity +outside the stated model, tokenizer, prompt, runtime, and measurement contract. + +The exact source revision, model provenance, runtime versions, raw streaming +timestamps, and output-quality evidence are required for any reported result. +Never report a profiler trace as an unprofiled throughput result. + +## Prerequisites + +- Linux system with a native ROCm-capable AMD GPU. +- Git checkout of this repository. +- Python environment containing HIP-enabled PyTorch, Triton, and FreeToken. +- ROCm tools including `rocminfo`; `rocm-smi` is optional but recommended. +- A model obtained directly from its original publisher under its license. + +Confirm that `torch.version.hip` is non-empty and that `torch.cuda.is_available()` +returns true before building FreeToken. HIP maintains the `torch.cuda` namespace +for compatibility, so a successful import alone is not sufficient evidence of +native ROCm execution. + +## Capture a host manifest + +From the repository root, use a new artifact directory for every run: + +```bash +bash scripts/reproduce/collect_host_manifest.sh \ + --source-dir "$PWD" \ + --artifact-dir "$PWD/artifacts/host-$(date -u +%Y%m%dT%H%M%SZ)" \ + --python /path/to/venv/bin/python \ + --expected-gfx gfx1151 +``` + +Omit `--expected-gfx` only when the work is intentionally cross-architecture. +The script requires a native HIP PyTorch device before it creates an artifact. +It never starts or stops a server, changes a clock policy, clears a cache, +changes swap, or records a shell environment. It redacts the host name by +default and deliberately omits process lists, serial numbers, and network +addresses. Each bundle includes a `SHA256SUMS` file covering the raw report +files and manifest. + +The collector was live-validated on the evaluated system on 30 August 2026 using native +ROCm PyTorch and `--expected-gfx gfx1151`. It emitted a redacted host field, +identified `gfx1151`, and produced only the documented non-sensitive artifact +files. This validates the collector itself, not a broader performance claim. + +## Build and functional validation + +Install only into an isolated environment. Do not install CUDA-only packages +such as CUDA-indexed PyTorch wheels, FlashInfer, or NVIDIA kernel wheels into a +ROCm environment. With a verified ROCm Python runtime, build from the checkout: + +```bash +python -m pip install -e . --no-build-isolation --no-deps +python -m unittest tests.utils.test_rocm_runtime +``` + +Then run model-specific functional controls before any throughput run. Record +the model publisher, revision, file byte count, SHA-256, tokenizer revision, +chat template, request JSON, output, and scorer result in the artifact. Do not +redistribute model weights unless the model license explicitly allows it. + +## Benchmark an already-running local server + +`benchmarks/reproduce/run_local_api_benchmark.py` is the public client for a +server that the operator has already started on a loopback endpoint. It accepts +only `localhost`, `127.0.0.1`, or `::1`; it cannot send benchmark traffic to a +LAN or public address. It neither starts nor stops a server. Quality mode +requires an explicit expected visible response, while throughput mode requires +a fixed generation length of at least two tokens. + +```bash +python benchmarks/reproduce/run_local_api_benchmark.py \ + --base-url http://127.0.0.1:8000/v1 \ + --model your-served-model-name \ + --tokenizer /path/to/original/checkpoint \ + --prompt-file /path/to/quality-prompt.txt \ + --expected-text EXPECTED_VISIBLE_ANSWER \ + --samples 5 --warmup \ + --artifact-dir artifacts/quality-$(date -u +%Y%m%dT%H%M%SZ) +``` + +Use a separate throughput artifact after the quality gate passes. Retain the +same prompt and model representation, opt into `--mode throughput`, set a fixed +`--max-tokens` value, and explain any difference between server-side generation +tokens and tokenizer-counted visible text. The client writes a manifest, an +immutable JSON artifact for each request, and a summary. It does not claim that +the server is correct merely because it streamed successfully. + +## Benchmark contract + +For each row, retain raw artifacts before generating a summary table: + +1. A cold-start result and a warm-server result, labelled separately. +2. At least five independently started scored samples for a performance claim. +3. Client-observed TTFT, prompt throughput, decode throughput, and p50, p95, + and p99 content-token gaps. +4. GPU clock, temperature, power policy, memory use, host memory pressure, + swap state, and competing I/O or compute activity. +5. The exact model representation. Do not compare NVFP4 and GGUF performance + as if they were a format-neutral engine comparison. +6. A quality gate that is separate from fixed-length throughput mode. + +Reject or rerun samples affected by unexpected compilation, swapping, thermal +transition, stale service processes, active model copies, or unexplained I/O +contention. Preserve rejected samples and state why they were rejected. + +## Extension policy + +Contributions should target one measured bottleneck at a time. Suitable work +includes a unified-memory-aware cache policy, a profile-ranked HIP kernel, +additional GPU-host manifests, or an expanded quality suite. Every change must +retain its baseline, raw evidence, quality comparison, full API result, and an +accept-or-reject decision. A faster isolated kernel is not an accepted runtime +optimization until it preserves model output and improves the full serving +workload. + +## Publication package + +For an external release, publish a pinned source tag, this guide, `CITATION.cff`, +`.zenodo.json`, the safe host manifests, workload and scorer code, sanitized raw +results, and scripts that regenerate paper tables. Archive the tagged release +with Zenodo and cite its version DOI. Keep private model files, credentials, +internal addresses, serial numbers, and unrelated service logs out of the +release. diff --git a/docs/upstream-qwen-paper-protocol.md b/docs/upstream-qwen-paper-protocol.md new file mode 100644 index 0000000000..3a152f13bd --- /dev/null +++ b/docs/upstream-qwen-paper-protocol.md @@ -0,0 +1,54 @@ +# Upstream Qwen 8 GB benchmark protocol evidence + +## Confirmed source facts + +The FreeToken paper states that its main experiments use Qwen3.6-35B-A3B, +DeepSeek-V4-Flash, and GLM-5.2 on six machines spanning an 8 GB RTX 4060 laptop +through an RTX PRO 6000 workstation. Its RTX 4060 laptop row is a Core +i9-13900H with 20 threads, 32 GiB LPDDR5, 8 GiB RTX 4060 Laptop VRAM, PCIe 4.0 +x8, measured 11.8 GB/s expert-transfer bandwidth, and measured 47.5 GB/s +CPU-side MoE bandwidth. The paper states that this laptop serves a 35B model at +39.3 tokens per second. + +The 8 GB laptop uses Qwen3.6-35B-A3B's official NVFP4 release. The other Qwen +cross-engine comparisons use BF16 for exact weight-format parity. The paper's +metrics are per-request mean decode throughput and per-request mean TTFT. Its +four workloads are AIME math reasoning, an OpenCode SWE-bench coding agent, +the same issue via Claude Code with concurrent subagents, and a 13-turn +OpenClaw email/calendar agent. The broader comparison includes llama.cpp, +Ollama, KTransformers, and MoE-Infinity. + +The source repository identifies `Qwen/Qwen3.6-35B-A3B` and +`nvidia/Qwen3.6-35B-A3B-NVFP4` as known-good Qwen MoE checkpoints. Its backend +documentation defines `offload`, `cpu`, `hybrid`, and `auto`, where the latter +selects offload for MoE and may select hybrid following `ft bench bw`. + +Primary sources: + +- +- + +## Fields the paper summary does not establish + +The published HTML establishes the hardware, model format, metric type, and +workload classes. It does not identify the following fields for the 39.3 TPS +row. They must be resolved from released artifacts, the authors, or marked +unavailable before calling the GMKtek EVO-X2 result a strict replication: + +| Field | State | Required action | +| --- | --- | --- | +| Checkpoint revision and exact quantization | Unknown | Inspect paper appendix, released benchmark assets, and upstream history. | +| Laptop CPU and RAM | Resolved | Core i9-13900H, 20 threads, 32 GiB LPDDR5. Record OS, driver, CUDA, and FreeToken commit if recovered. | +| Prompt corpus and token count | Workload class resolved | Locate the exact AIME questions, SWE issue, tool harness versions, and rendered token counts. | +| Output length and stop policy | Unknown | Locate benchmark runner defaults and raw results. | +| Warmup procedure and cache state | Partially resolved | Paper says the first request warms the cache normally. Recover the scored-run sequence. | +| TPS definition and reported statistic | Resolved at paper level | Per-request mean decode TPS and per-request mean TTFT. Retain the client-side formula and raw timestamps. | +| Expert cache, KV allocation, CPU thread count, and selected backend | Unknown | Recover the launch configuration or state that parity is approximate. | + +## Current GMKtek EVO-X2 comparison status + +Existing evidence proves native HIP functional serving for +`nvidia/Qwen3.6-35B-A3B-NVFP4` and a prior controlled warm output rate around +28.9 client TPS. It does not prove paper parity because the model revision, +workload, and policy contract above are incomplete. The new harness records +those differences rather than hiding them. diff --git a/examples/freetoken-swap.toml b/examples/freetoken-swap.toml new file mode 100644 index 0000000000..e0bdb5130b --- /dev/null +++ b/examples/freetoken-swap.toml @@ -0,0 +1,103 @@ +# Native freetoken-swap catalog. This is not a llama-swap YAML file. +# +# Migration: map each llama-swap models..cmd to one native model path +# plus argv tokens. Map checkEndpoint /ready to the built-in readiness gate. +# Never paste shell fragments, ${PORT}, or command substitutions here. Set +# `port = 0` for a kernel-selected loopback port per native activation. + +# What: open the router TOML table; why: following settings belong to the router configuration namespace. +[router] +# Inference routes accept Bearer, a Basic-auth password, or `X-Api-Key` when +# this is nonempty. Router credentials are never forwarded to the engine. +# What: show an illustrative router API-key list; why: operators must replace the placeholder before clients authenticate to control and inference routes. +api_keys = ["replace-with-a-secret"] +# What: set an illustrative 300-second default idle lifetime; why: operators balance model residency against the latency and memory cost of later reloads. +default_ttl_s = 300 +# What: set an illustrative 30-second unload deadline; why: the router needs a finite bound for stopping an engine before declaring cleanup failure. +unload_timeout_s = 30 +# What: set an illustrative 900-second upstream request deadline; why: long generations may need time while the finite limit prevents requests from hanging forever. +upstream_timeout_s = 900 +# Safe static suffixes that return 409 rather than cold-loading through +# /upstream/{model-id}/...; set [] to disable. No regex is accepted. +# What: list static suffixes that must not cold-load a model; why: asset-like requests fail with 409 instead of spending memory and startup time on accidental activation. +upstream_no_activation_suffixes = [".js", ".json", ".css", ".png", ".gif", ".jpg", ".jpeg", ".ico", ".txt"] +# Body-free activity rows are always bounded. Captures are sensitive, in-memory, +# credential-redacted, and opt-in; zero disables request/response retention. +# What: retain at most 1000 body-free activity rows; why: the bounded history supports diagnosis without unbounded memory growth. +activity_max_entries = 1000 +# What: disable sensitive body capture by setting its budget to zero; why: operators must opt in explicitly before request or response bodies are retained in memory. +capture_buffer_mb = 0 +# Session grouping stores only a stable SHA-256 label, never the raw value. +# What: name headers used to derive a hashed session label; why: activity grouping works without storing the raw potentially sensitive header values. +activity_session_headers = ["X-Session-ID", "X-Litellm-Session-Id"] +# Retain at most one hour of owned-process RAM/VRAM samples in memory. +# What: leave owned-process performance sampling enabled; why: operators can observe RAM and VRAM while retaining the option to disable sampler overhead. +performance_disabled = false +# What: sample performance every five seconds; why: the interval balances trend visibility against monitoring overhead. +performance_every_s = 5 +# What: select first-in-first-out request scheduling; why: the example favors predictable arrival order over alternate prioritization policies. +scheduler = "fifo" +# Zero disables the global cap. Every profile still has a default cap of 10. +# What: cap concurrent routed requests at 32; why: the finite limit protects host capacity while operators tune it for their workload. +global_concurrency_limit = 32 +# Include alternate IDs in /v1/models. They remain routable when this is false. +# What: include aliases in the public model listing; why: clients can discover alternate IDs, with the tradeoff of a larger advertised catalog. +include_aliases_in_list = true + +# What: open the router.groups.interactive TOML table; why: following settings belong to the router.groups.interactive configuration namespace. +[router.groups.interactive] +# What: assign coding and chat profiles to the interactive group; why: the group policy coordinates residency and exclusivity across both profiles. +members = ["coding", "chat"] +# What: enable swaps within the interactive group; why: activating one member may replace another instead of requiring simultaneous residency. +swap = true +# What: make the interactive group mutually exclusive; why: only one member occupies the constrained group at a time. +exclusive = true +# What: disable persistent residency for this group; why: idle members may unload so memory can be reclaimed. +persistent = false + +# What: open the models.coding TOML table; why: following settings belong to the models.coding configuration namespace. +[models.coding] +# What: show an illustrative local GGUF model path; why: operators replace the placeholder with the model file available on their host. +model = "/models/coding.gguf" +# Slash-namespaced IDs are valid; every segment uses letters, digits, `.`, `_`, or `-`. +# What: show alternate routed identifiers for the coding profile; why: compatible clients can select the same model through either approved alias. +aliases = ["coding-compatible", "local/coding-compatible"] +# What: show an illustrative engine port; why: operators must avoid collisions or use the supported dynamic-port policy for their deployment. +port = 1919 +# What: show illustrative ft serve arguments; why: the served name and context limit must match client identity and available memory. +args = ["--served-model-name", "coding", "--max-seq-len-override", "32768"] +# What: allow up to 300 seconds for model readiness; why: large models may need startup time while the finite limit keeps activation bounded. +ready_timeout_s = 300 +# What: set this profile idle lifetime to zero; why: the example unloads immediately after leases end rather than retaining model memory. +ttl_s = 0 +# What: set the profile scheduling priority; why: operators tune this value to control which queued activation wins contention. +priority = 10 +# What: set the profile request concurrency cap; why: the cap protects that model from more simultaneous work than the host can sustain. +concurrency_limit = 2 +# What: assign the profile to the interactive routing group; why: members share the group swap and exclusivity policy. +group = "interactive" +# Optional narrow compatibility filter. It removes only named top-level JSON +# fields from requests for this profile. `model` can never be removed. +# What: drop the optional metadata request field; why: the narrow compatibility filter removes only the named unsupported field. +drop_fields = ["metadata"] + +# What: open the models.chat TOML table; why: following settings belong to the models.chat configuration namespace. +[models.chat] +# What: show an illustrative local GGUF model path; why: operators replace the placeholder with the model file available on their host. +model = "/models/chat.gguf" +# Hidden profiles remain routable and manageable but are omitted from /v1/models, +# along with all of their aliases. +# What: hide the chat profile from model listings; why: the profile remains directly routable while discovery omits it and its aliases. +unlisted = true +# What: show an illustrative engine port; why: operators must avoid collisions or use the supported dynamic-port policy for their deployment. +port = 1919 +# What: show illustrative ft serve arguments; why: the served name and context limit must match client identity and available memory. +args = ["--served-model-name", "chat", "--max-seq-len-override", "16384"] +# What: allow up to 300 seconds for model readiness; why: large models may need startup time while the finite limit keeps activation bounded. +ready_timeout_s = 300 +# What: set the profile scheduling priority; why: operators tune this value to control which queued activation wins contention. +priority = 0 +# What: set the profile request concurrency cap; why: the cap protects that model from more simultaneous work than the host can sustain. +concurrency_limit = 4 +# What: assign the profile to the interactive routing group; why: members share the group swap and exclusivity policy. +group = "interactive" diff --git a/examples/freetoken-swap.yaml b/examples/freetoken-swap.yaml new file mode 100644 index 0000000000..db47ee5e05 --- /dev/null +++ b/examples/freetoken-swap.yaml @@ -0,0 +1,43 @@ +# Integration template, not a claim that these placeholder models are qualified. +# Use a pinned llama-swap build and a FreeToken build containing GET /ready. +# Do not also manage these processes with ft daemon. +# What: set health check timeout to 300; why: the example permits slow model startup while still bounding a failed readiness probe. +healthCheckTimeout: 300 +# What: set global ttl to 0; why: the example disables an implicit global eviction policy so per-model TTL choices remain visible. +globalTTL: 0 +# What: set unload timeout to 30; why: the example allows bounded graceful shutdown before an operator chooses a stricter limit. +unloadTimeout: 30 +# What: show the illustrative models value; why: operators adapt this models choice to their model, memory budget, port policy, and startup latency rather than treating it as universal. +models: + # What: show the illustrative model a value; why: operators adapt this model a choice to their model, memory budget, port policy, and startup latency rather than treating it as universal. + model-a: + # What: show the illustrative cmd value >-; why: operators adapt this cmd choice to their model, memory budget, port policy, and startup latency rather than treating it as universal. + # What: provide the illustrative ft serve model and loopback launch fragment; why: llama-swap starts this model path on the allocated local port; operators replace the placeholder path. + # What: forward llama-swap's model identifier to ft serve; why: the launched engine advertises the same routed model identity selected by llama-swap. + # What: show illustrative context and token-cache limits; why: operators tune these memory-throughput tradeoffs for their hardware instead of treating 4096 as universal. + cmd: >- + ft serve --model /models/model-a --host 127.0.0.1 --port ${PORT} + --served-model-name ${MODEL_ID} + --max-seq-len-override 4096 --num-tokens 4096 + # What: set check endpoint to /ready; why: llama-swap probes the daemon readiness contract rather than treating a listening socket as ready. + checkEndpoint: /ready + # What: set proxy to http://127.0.0.1:${PORT}; why: llama-swap forwards model traffic to the loopback port allocated for this model entry. + proxy: http://127.0.0.1:${PORT} + # What: set ttl to 0; why: the example demonstrates the unload-latency versus residency tradeoff for this model. + ttl: 0 + # What: show the illustrative model b value; why: operators adapt this model b choice to their model, memory budget, port policy, and startup latency rather than treating it as universal. + model-b: + # What: show the illustrative cmd value >-; why: operators adapt this cmd choice to their model, memory budget, port policy, and startup latency rather than treating it as universal. + # What: provide the illustrative ft serve model and loopback launch fragment; why: llama-swap starts this model path on the allocated local port; operators replace the placeholder path. + # What: forward llama-swap's model identifier to ft serve; why: the launched engine advertises the same routed model identity selected by llama-swap. + # What: show illustrative context and token-cache limits; why: operators tune these memory-throughput tradeoffs for their hardware instead of treating 4096 as universal. + cmd: >- + ft serve --model /models/model-b --host 127.0.0.1 --port ${PORT} + --served-model-name ${MODEL_ID} + --max-seq-len-override 4096 --num-tokens 4096 + # What: set check endpoint to /ready; why: llama-swap probes the daemon readiness contract rather than treating a listening socket as ready. + checkEndpoint: /ready + # What: set proxy to http://127.0.0.1:${PORT}; why: llama-swap forwards model traffic to the loopback port allocated for this model entry. + proxy: http://127.0.0.1:${PORT} + # What: set ttl to 300; why: the example demonstrates the unload-latency versus residency tradeoff for this model. + ttl: 300 diff --git a/output/pdf/freetoken-amd-strix-halo-white-paper-v0.1.0-rc1.pdf b/output/pdf/freetoken-amd-strix-halo-white-paper-v0.1.0-rc1.pdf new file mode 100644 index 0000000000..c6ebe579db Binary files /dev/null and b/output/pdf/freetoken-amd-strix-halo-white-paper-v0.1.0-rc1.pdf differ diff --git a/paper-draft/PUBLICATION_CHECKLIST.md b/paper-draft/PUBLICATION_CHECKLIST.md new file mode 100644 index 0000000000..620704f6b3 --- /dev/null +++ b/paper-draft/PUBLICATION_CHECKLIST.md @@ -0,0 +1,46 @@ +# Final publication checklist + +Use this checklist immediately before publishing the public white paper. A +checked box requires fresh evidence, not an assumption based on this release +candidate. + +## Repository and provenance + +- [ ] Review all staged changes and confirm no unrelated local work is included. +- [ ] Commit the white-paper package, release notes, citation metadata, and + public reproduction tools together. +- [ ] Record the resulting full commit SHA in the white paper and release notes. +- [ ] Create and push an immutable tag, for example `amd-strix-halo-white-paper-v0.1.0`. +- [ ] Regenerate the PDF from the tagged source and record its SHA-256. + +## Privacy and reproducibility + +- [ ] Run the manifest collector against a clean native HIP system. +- [ ] Verify all public artifacts omit credentials, LAN addresses, hostnames, + serial numbers, private model paths, and unrelated logs. +- [ ] Verify model provenance lists publisher, revision, byte count, SHA-256, + and license without redistributing model weights. +- [ ] Run the public reproduction tests and retain the output in the release + preparation record. +- [ ] Recheck every quantitative claim against its cited raw artifact. + +## Zenodo and public record + +- [ ] Connect GitHub to Zenodo and enable the repository. +- [ ] Confirm `.zenodo.json` title, creator, version, license, keywords, and + description are correct. Zenodo uses this file in preference to `CITATION.cff` + when both are present. +- [ ] Create the GitHub release from the immutable tag and attach the PDF. +- [ ] Wait for Zenodo processing, verify the record and version DOI, then check + the archival status. +- [ ] Replace the release-candidate version and DOI-pending language in the + paper and `CITATION.cff` with the final tag and version DOI. + +## Distribution + +- [ ] Publish the Zenodo DOI as the canonical citation route. +- [ ] Publish the technical-report PDF in the GitHub release. +- [ ] Optionally submit the same final PDF to arXiv after verifying the current + subject-category and endorsement rules. +- [ ] Enable a public repository contact route, preferably Issues for + reproducible reports and Discussions for general questions. diff --git a/paper-draft/README.md b/paper-draft/README.md new file mode 100644 index 0000000000..eaa06a93cc --- /dev/null +++ b/paper-draft/README.md @@ -0,0 +1,17 @@ +# Technical white paper release candidate package + +`amd_strix_halo_freetoken_port_draft.md` is a research-paper draft based only on the repository's recorded evaluated-system evidence, a live hardware and software manifest captured on 30 August 2026, and the supplied FreeToken paper. + +`paper.tex` and `references.bib` are the venue-neutral LaTeX source. Build them with a standard TeX distribution using `pdflatex paper`, `bibtex paper`, then `pdflatex paper` twice. Select the final venue template only after the submission path is fixed; the existing LaTeX source deliberately avoids venue-specific formatting. + +For reviewer circulation, `output/pdf/freetoken-amd-strix-halo-white-paper-v0.1.0-rc1.pdf` is a polished release-candidate copy generated directly from the Markdown manuscript. Rebuild it with the bundled workspace Python runtime and `scripts/build_paper_pdf.py`. The renderer is intentionally venue-neutral; the Markdown and LaTeX manuscripts remain the authoritative editable sources. + +It is intentionally written as a systems-port and controlled-evaluation paper, not as a claimed replication of FreeToken's published NVIDIA results. The manuscript now includes the full non-sensitive evaluated-system platform table and an artifact-availability section. Before submission, convert the plain references to the target venue's BibTeX style, attach the raw artifact bundle listed in the paper's reproducibility section, and create a tagged archival release. + +The immediate evidence gaps are: the upstream Qwen benchmark contract, five-sample repetitions for the Q4 control, an expanded quality suite, long-context and agentic workloads, parameterized public model-launch recipes, and a second clean-host matrix. + +Use [PUBLICATION_CHECKLIST.md](PUBLICATION_CHECKLIST.md) for the final tag, Zenodo archive, and DOI substitution steps. The release notes in [RELEASE_NOTES_v0.1.0-rc1.md](RELEASE_NOTES_v0.1.0-rc1.md) are the proposed GitHub release body. + +## Contact + +For manuscript correspondence, replication questions, or technical collaboration, use the project repository at . Personal contact details are omitted from this anonymized review copy. Enable and link the public issue tracker before release so reproducible software defects and proposed changes have a searchable public route. diff --git a/paper-draft/RELEASE_NOTES_v0.1.0-rc1.md b/paper-draft/RELEASE_NOTES_v0.1.0-rc1.md new file mode 100644 index 0000000000..532cefa7bb --- /dev/null +++ b/paper-draft/RELEASE_NOTES_v0.1.0-rc1.md @@ -0,0 +1,44 @@ +# FreeToken AMD ROCm/HIP Port for Strix Halo v0.1.0-rc1 + +## Technical white paper release candidate + +This release candidate packages the evidence-backed technical white paper, +portable host-manifest collector, local-only benchmark client, citation metadata, +and reproducibility guidance for the FreeToken ROCm/HIP port on AMD Strix Halo. + +The package is based on the public `amd-rocm-gfx1151` branch tip +`a937862f171900bd5d1d207c8ff59b40a15ce742`, verified on 30 August 2026. +The release-candidate files are not yet included in that public branch. Do not +cite this release candidate as an immutable publication until the final release +checklist is complete and a tag plus Zenodo version DOI exist. + +## Included white-paper claims + +- Native ROCm/HIP execution is established on the evaluated AMD Strix Halo + `gfx1151` system. +- Qwen NVFP4 serving passed the documented deterministic canary with the + reference router at 27.880 mean client-visible decode tokens/s across three + warm quality-matched runs. +- The same-file Qwen Q4_K_M control measured 50.63 tokens/s with the native + HIP router and 50.29 tokens/s with the llama.cpp ROCm 10 control. +- The 0.7 percent Q4 margin is bounded to that stated control and is not a + general engine ranking. +- A faster NVFP4 Triton router is retained as rejected evidence because it + changed deterministic model output. + +## Publication route + +Publish the final package as a GitHub release from an immutable tag, then let +Zenodo archive that release and assign the version DOI. Attach the generated +white-paper PDF to the GitHub release. Use the Zenodo version DOI in the final +paper, release page, and any arXiv technical-report submission. + +## Not included + +Model weights, private model paths, hostnames, LAN addresses, credentials, +serial numbers, unrelated service logs, and raw environment dumps are excluded +from the public release. + +## Correspondence + +FreeToken AMD contributors. Personal contact details are omitted from this anonymized review copy. diff --git a/paper-draft/RELEASE_PAYLOAD.md b/paper-draft/RELEASE_PAYLOAD.md new file mode 100644 index 0000000000..f501be5dda --- /dev/null +++ b/paper-draft/RELEASE_PAYLOAD.md @@ -0,0 +1,35 @@ +# White paper release payload + +Stage the following files for the white-paper release. The tag must include +the code revision and these files together, but it must not include the local +`tmp/` review images or `.reference-llama-cpp/` reference checkout. + +## White paper and metadata + +- `paper-draft/amd_strix_halo_freetoken_port_draft.md` +- `paper-draft/paper.tex` +- `paper-draft/references.bib` +- `paper-draft/README.md` +- `paper-draft/RELEASE_NOTES_v0.1.0-rc1.md` +- `paper-draft/PUBLICATION_CHECKLIST.md` +- `paper-draft/RELEASE_PAYLOAD.md` +- `output/pdf/freetoken-amd-strix-halo-white-paper-v0.1.0-rc1.pdf` +- `CITATION.cff` +- `.zenodo.json` + +## Public reproduction material + +- `docs/amd-rocm-gfx1151.md` +- `docs/reproducibility.md` +- `scripts/build_paper_pdf.py` +- `scripts/reproduce/collect_host_manifest.sh` +- `benchmarks/reproduce/run_local_api_benchmark.py` +- `tests/reproduce/test_collect_host_manifest.py` +- `tests/reproduce/test_run_local_api_benchmark.py` + +## Required exclusions + +- Model weights and model directories +- Raw service logs and environment dumps +- LAN addresses, hostnames, serial numbers, credentials, and private paths +- `tmp/` renderer output and `.reference-llama-cpp/` diff --git a/paper-draft/amd_strix_halo_freetoken_port_draft.md b/paper-draft/amd_strix_halo_freetoken_port_draft.md new file mode 100644 index 0000000000..110f54a593 --- /dev/null +++ b/paper-draft/amd_strix_halo_freetoken_port_draft.md @@ -0,0 +1,172 @@ +# Native FreeToken Serving on AMD Strix Halo: A ROCm/HIP Port and Controlled Unified-Memory Evaluation + +**FreeToken AMD contributors** + +*Anonymized review copy; correspondence through the project repository.* + +*Technical white paper, release candidate v0.1.0-rc1, 30 August 2026* + +## Abstract + +Large mixture-of-experts (MoE) models make capable local inference possible, but most edge-serving systems are designed and evaluated on NVIDIA discrete GPUs. We present a native ROCm/HIP port of FreeToken for AMD Strix Halo, a unified-memory APU platform represented by the Ryzen AI Max+ 395 with Radeon 8060S graphics (`gfx1151`). The port retains FreeToken's CUDA behavior while adding HIP extension builds, ROCm-safe architecture detection, portable Triton paths, and native model-loading and serving validation. It executes without a CUDA compatibility layer, Vulkan substitute, or CPU-only fallback. + +We evaluate the port on a GMKtek EVO-X2, a Strix Halo system with 64 GiB installed LPDDR5 memory and a 4 GiB firmware GPU reservation. Linux exposes 59.46 GiB host memory and ROCm exposes a 56.0 GiB coarse-grained GPU pool. We use Qwen3.6-35B-A3B and Gemma 4 26B A4B controls. The port serves Qwen's NVIDIA NVFP4 checkpoint through an OpenAI-compatible streaming API and reproduces a deterministic AIME canary with the reference router at 27.88 mean client-visible decode tokens/s. A faster NVFP4 Triton-router path was rejected because it changed deterministic model output. For a matched raw-prompt Q4_K_M Qwen control, both FreeToken and llama.cpp used the same 54-token prompt and produced the correct mathematical result; FreeToken reached 50.63 tokens/s after enabling a quality-checked native HIP router, compared with 50.29 tokens/s for the ROCm 10 llama.cpp control. A Gemma 4 Q4 text control reached 57.05 tokens/s and returned the expected deterministic answer. + +These results establish functionality and a bounded same-file Q4 control, not a strict reproduction of FreeToken's published 39.3 tokens/s RTX 4060 result. The upstream prompt corpus, cache state, stop policy, exact revision, and configuration remain incomplete. Profiling instead identifies dense mixed-FP8 decode as the dominant NVFP4 Qwen kernel consumer and shows that a worst-case unified-memory expert-cache fill is materially smaller than end-to-end token time. We release the porting boundary, validation contract, and rejected-candidate evidence to make AMD edge-serving claims reproducible and falsifiable. + +## 1. Introduction + +Open-weight MoE models are increasingly capable, yet practical local serving remains concentrated on systems with CUDA-capable discrete GPUs. FreeToken demonstrated that MoE-aware placement, caching, and execution policies can turn consumer hardware into a viable local serving platform [1]. Its design assumes the practical realities of edge inference: model state frequently exceeds device memory, execution alternates between prefill and decode, and agentic workloads repeatedly edit and extend context. + +AMD Strix Halo changes an important part of that deployment model. Its Radeon 8060S GPU and CPU share a large LPDDR5X memory pool rather than communicating through a discrete-GPU PCIe path. This makes large local models feasible on an APU, but it does not make a CUDA-oriented serving runtime automatically portable or performant. The runtime must compile native extensions with HIP, avoid treating HIP's `torch.cuda` compatibility namespace as evidence of NVIDIA hardware, preserve model semantics across alternate kernels, and measure CPU-GPU contention rather than assuming PCIe transfer is the principal cost. + +This work asks a narrower question than the original FreeToken paper: can FreeToken's serving stack be ported to a Strix Halo `gfx1151` system as a native ROCm/HIP runtime, and what do controlled model-serving experiments show after the port? We make four contributions: + +1. We implement a narrowly gated ROCm/HIP port that preserves CUDA behavior and compiles native extensions for `gfx1151`. +2. We define a validation contract that separates native execution, API correctness, deterministic output, matched controls, and paper-parity claims. +3. We report controlled Qwen and Gemma results with explicit prompt, model-format, and metric boundaries. +4. We use native ROCm profiling and cache-copy experiments to identify the current optimization frontier and report rejected candidates instead of presenting microbenchmark wins as system improvements. + +## 2. Background and Porting Challenges + +FreeToken targets local MoE serving by jointly managing model layout, expert residency, CPU-GPU execution, and cache state [1]. Its published Qwen3.6-35B-A3B result reports 39.3 decode tokens/s on an 8 GiB RTX 4060 laptop. That result uses the official NVIDIA NVFP4 release, and the paper reports per-request mean decode throughput and mean time to first token (TTFT) across agentic workloads [1]. + +The target here differs in both hardware and software. The evaluated system is an AMD Ryzen AI Max+ 395 system with Radeon 8060S graphics, `gfx1151`, 64 GiB installed LPDDR5 memory, and a firmware-reserved integrated-GPU allocation. Its ROCm 10 runtime and HIP compiler enable native execution, but FreeToken contains CUDA-specific extension, JIT, architecture-detection, and Triton assumptions. Further, unified memory eliminates a discrete PCIe transfer boundary but introduces shared-memory contention between CPU fallback work, GPU execution, KV cache, and expert-cache activity. + +We therefore treat the original paper as design motivation and protocol reference, not as an automatically comparable baseline. A strict replication requires the same checkpoint revision, workload corpus, prompt tokenization, generated-token and stop rules, warmup state, policy configuration, and reported statistic. Those fields have not yet all been recovered for the upstream RTX 4060 row. + +## 3. Native ROCm/HIP Port + +The port retains CUDA as a separate runtime path. On ROCm builds, setup detects HIP PyTorch and links the small native extension surface against `libamdhip64` rather than CUDA runtime libraries. A compatibility header maps only the CUDA Runtime API subset already used by FreeToken's pinned-memory and CPU-MoE extensions to HIP. JIT compilation removes NVCC-only flags and uses HIP-compatible launch behavior. + +The Python and Triton surfaces require separate treatment. PyTorch exposes ROCm devices through the `torch.cuda` namespace for compatibility, so FreeToken now rejects ROCm before NVIDIA SM capability checks. This prevents `gfx1151` from being misclassified as a hypothetical NVIDIA architecture. CUDA-only optional dependencies and NVIDIA PTX inline assembly are avoided on HIP, with portable Triton implementations used where validated. The GGUF JIT build supplies system ROCm include and library directories only when the PyTorch ROCm wheel omits the necessary developer surface. This supports a native `gfx1151` object without modifying the system ROCm installation. + +The resulting server preserves FreeToken's OpenAI-compatible model discovery, streaming, non-streaming, cache, and MoE interfaces. All reported experiments use the ROCm/HIP path. We did not use Vulkan or a CPU-only runner as an implementation substitute. + +## 4. Experimental Methodology + +### 4.1 Platform and runtime + +Experiments ran on a GMKtek EVO-X2. Table 1 records the environment observed on 30 August 2026. The port uses ROCm 10, HIP-compiled extensions, and AMD Triton. The Qwen NVFP4 experiment uses the upstream-supported `nvidia/Qwen3.6-35B-A3B-NVFP4` model through native HIP Triton. The same-file Q4 control uses `Qwen3.6-35B-A3B-UD-Q4_K_M.gguf`; Gemma uses `gemma-4-26B_q4_0-it.gguf`. + +**Table 1. Evaluated-system hardware and software environment.** The table reports static platform information. Dynamic measurements such as free memory, temperature, clocks, and active processes are retained per benchmark run in the artifact manifest rather than presented as fixed machine specifications. + +| Component | Specification | +| --- | --- | +| System | GMKtek EVO-X2, SKU `EVO-X2-001`, hardware version 1.0 | +| Firmware | EVO-X2 1.09, 13 September 2025 | +| Processor | AMD Ryzen AI Max+ 395 with Radeon 8060S | +| CPU topology | 16 cores, 32 hardware threads, one NUMA node; boost enabled | +| CPU frequency | 625 MHz minimum and 5.1875 GHz maximum reported by `lscpu` | +| CPU cache | 768 KiB L1d, 512 KiB L1i, 16 MiB L2, and 64 MiB L3 | +| Installed memory | 64 GiB LPDDR5, eight 8 GiB Micron devices; 8,532 MT/s rated and 8,000 MT/s configured | +| Firmware UMA reservation | 4 GiB integrated-GPU reservation | +| Linux-visible host memory | 59.46 GiB (`MemTotal`) | +| ROCm GPU memory pool | 56.0 GiB coarse-grained pool reported for `gfx1151` | +| GPU | AMD Radeon 8060S Graphics, PCI ID `1002:1586`, `gfx1151` | +| GPU execution resources | 40 compute units, wavefront size 32, maximum 32 waves per compute unit | +| HSA configuration | XNACK disabled; coherent host access reported false | +| Operating system | Ubuntu 26.04.1 LTS, Linux 7.0.0-30-generic | +| HIP and compiler | HIP 7.15.26333; AMD Clang 23 from ROCm 10.0 | +| Python runtime | PyTorch `2.13.0+rocm10.0.0`, Triton `3.8.0`, Python 3.12 environment | +| Storage | Lexar ARES 2 TB NVMe SSD | + +The 4 GiB firmware reservation is not the FreeToken model-memory budget. It is a preallocated UMA region. The capacity available to a request changes with host activity, runtime overhead, model weights, expert residency, and KV-cache growth. Each scored run therefore records memory pressure and the serving process state separately. + +### 4.2 Metrics and correctness gates + +Decode throughput is client-visible streaming throughput: generated completion tokens, excluding the first generated token, divided by the interval from the first to final streamed content token. We keep client-observed TTFT separate from runtime-internal timing. Fixed-length throughput and quality are distinct modes so a system cannot obtain an apparently better rate merely by ending early, emitting hidden reasoning tokens, or silently changing the request. + +Every accepted candidate must satisfy all applicable gates: native HIP compilation and execution, successful OpenAI-compatible response, correct tokenizer accounting, deterministic-output or task-quality evidence, and preserved raw artifacts. A microbenchmark gain cannot be accepted if the full model changes the deterministic answer or fails to improve the end-to-end API workload. + +### 4.3 Controls + +The Qwen NVFP4 canary uses greedy sampling with a thinking-enabled template and a forced 128-token decode. Its required SHA-1 is `0acef4eab6f4`. The Q4 raw-prompt control sends the same UTF-8 string to both engines' `/v1/completions` endpoint with `temperature=0`, `top_p=1`, `top_k=-1`, streaming enabled, and a 1024-token cap. The prompt SHA-256 is `224f02631165a176e660363fefeb8eb58e5a150271fed72bdc1f90fa39448523`, and both engines report 54 prompt tokens. The expected mathematical result is 70. + +## 5. Results + +### 5.1 Native Qwen NVFP4 serving is functional but does not establish paper parity + +The native ROCm/HIP Qwen server passed the deterministic AIME canary with the reference PyTorch router. Three warm quality-matched repeats produced 26.786, 28.422, and 28.431 client-visible tokens/s, for a mean of 27.880 tokens/s. Mean warm TTFT was 409.0 ms for the 54-prompt-token and 127-completion-token request. Every run emitted the required SHA-1. + +A ROCm Triton top-k router improved isolated router latency by 1.62 to 1.63x for Qwen's 256-expert top-8 shape, and achieved 29.186 tokens/s in a performance-only NVFP4 workload. However, an end-to-end greedy AIME request changed output hash, so this configuration is rejected for NVFP4 serving. This distinction matters: router-only speed and a transport canary are not a quality-preserving system result. + +The 27.880 tokens/s Qwen NVFP4 value is not a direct comparison with the paper's 39.3 tokens/s RTX 4060 result. The underlying model representation is related, but the exact upstream workload and configuration contract is incomplete, and the platforms have materially different memory architecture. + +### 5.2 Matched Q4 Qwen raw-prompt control + +Table 2 compares FreeToken and llama.cpp on the same Q4_K_M file, raw prompt, tokenizer count, deterministic sampling, and steady decode rule. Before enabling the in-tree HIP Triton router, FreeToken reached 47.12 tokens/s, 6.3% below the llama.cpp control. With the HIP router enabled, FreeToken reached 50.63 tokens/s while preserving the correct derivation for the expected answer. This is 0.7% above llama.cpp's 50.29 tokens/s. + +| Engine | Model representation | Prompt tokens | Generated tokens | Steady decode tokens/s | Quality evidence | +| ------------------------------------ | -------------------- | -------------:| ----------------:| ----------------------:| -------------------------------- | +| FreeToken AMD | Qwen Q4_K_M GGUF | 54 | 1023 | 47.12 | Correct derivation for answer 70 | +| FreeToken AMD with native HIP router | Qwen Q4_K_M GGUF | 54 | 1023 | 50.63 | Same correct derivation | +| llama.cpp ROCm 10 | Qwen Q4_K_M GGUF | 54 | 1024 | 50.29 | Same correct derivation | + +**Figure 1. Controlled result overview.** The PDF review copy renders this figure with separate visual groups for the Qwen NVFP4 canary, the same-file Qwen Q4 control, and the Gemma text control. The groups must not be read as a single model-format or paper-parity ranking. + +This is intentionally a bounded result. Both outputs remained within Qwen's reasoning trace at the 1024-token ceiling, so neither exposed the requested boxed final line. The reasoning nevertheless explicitly derived the two valid bases, whose sum is 70. Future quality experiments should use a larger generation cap or a concise-answer task, plus repeated samples and a task suite. + +### 5.3 Gemma 4 Q4 text control + +The native Gemma GGUF path returned `323` for the fixed multiplication prompt, with a prompt hash of `0f65acd07a4f57b2644f7720b725d7795999406b90a9f91486da5effa39bb95d`. The client and local tokenizer agreed on 30 prompt tokens; the response used four completion tokens and reached 57.05 steady decode tokens/s. This validates the text-only loader, canonical template, OpenAI-compatible completion API, and token accounting for this fixed control. It does not alone qualify multimodal handling or long-context behavior. + +### 5.4 Native profiling changes the optimization priority + +Profiling the Qwen NVFP4 server under a wheel-compatible ROCm profiler recorded 353,457 dispatches. The trace was intrusive and measured only 15.61 tokens/s, so it is not used for throughput scoring. In its final active window, the largest GPU-time consumer was dense mixed-FP8 `_gemv_splitk_kernel`, not the routed NVFP4 expert kernel. The corresponding measured GPU times were 5,631.844 ms for `_gemv_splitk_kernel`, 1,676.018 ms for `_gemm_kernel`, 1,566.004 ms for `_decode_nvfp4_marlin_kernel`, and 593.192 ms for `fast_index_copy`. + +The port also measured its actual Qwen cache-copy path. With one active token, eight routed experts missing, and a 513-slot cache, native HIP copied 13.5 MiB in 0.097 ms, or 146.8 GB/s. The all-hit case took 0.023 ms. Extrapolated across 40 MoE layers, the all-miss copy component is 3.87 ms per decode token, below the approximately 35 ms end-to-end token interval of the accepted NVFP4 configuration. This does not prove copies are irrelevant, but it does rule out treating cache-copy bypass as the first unvalidated optimization. + +## 6. Discussion + +The port demonstrates that an edge-native MoE serving design can operate natively on an AMD unified-memory APU. It also shows why portability cannot be reduced to translating CUDA symbols. Correctness is coupled to router selection, tokenizer special-token handling, model representation, cache lifecycle, and the distinction between a microbenchmark and a client-visible request. + +The Q4 control is the cleanest current cross-runtime result because it holds the model file and raw prompt constant. It is not a full paper-style agentic comparison, and its 0.7% margin is too small to generalize beyond the stated workload. The NVFP4 path is the closest to FreeToken's original Qwen deployment but has a lower accepted throughput and an unrecovered upstream protocol. It should be described as a native port result, never as an RTX 4060 reproduction or a general AMD performance claim. + +Strix Halo also changes FreeToken's systems hypothesis. On a discrete GPU, expert movement crosses PCIe and the CPU and GPU have distinct primary memory systems. On this APU, CPU fallback, GPU kernels, expert residency, and KV growth compete for a shared memory pool. The next policy should therefore be based on measured contention curves over cache size, KV allocation, and CPU contribution. It should not assume that a PCIe-oriented hybrid rule or pinned-buffer strategy transfers unchanged to UMA. + +## 7. Artifact Availability and Independent Extension + +The AMD ROCm/HIP port is developed under Apache-2.0 at `https://github.com/dbourdea/FreeToken`, branch `amd-rocm-gfx1151`. This release candidate is based on public branch tip `a937862f171900bd5d1d207c8ff59b40a15ce742`, verified on 30 August 2026. The white-paper package and portable reproduction tools are not yet committed to that branch, and no immutable tag or DOI exists. Before publication, commit the complete package, archive an immutable tag, and replace this statement with the tag and version DOI. The release candidate includes HIP portability tests, Qwen and Gemma controls, and the read-only collector `scripts/reproduce/collect_host_manifest.sh`, which redacts the hostname by default and does not change service or host configuration. + +To make the work independently reproducible, the archival release must contain four separable components. First, the source archive must include the pinned commit, an environment lockfile, and a machine-readable schema for the run manifest. Second, the workload archive must contain the exact prompt text, request settings, tokenizer expectations, scoring code, and result schema. Third, the result archive must contain raw streaming timestamps, response text or output hashes as appropriate, telemetry, logs sanitized for credentials and host identifiers, and the script that generates each manuscript table. Fourth, model provenance must specify publisher, revision, byte count, SHA-256, and license, while directing users to obtain weights from the original publisher rather than redistributing weights without permission. + +The original host-specific helpers preserve a protected local service and use host-specific model paths. They remain appropriate for evidence capture on the test system, but they are not the public entry point. The public artifact now provides a parameterized host collector and a loopback-only API client at `benchmarks/reproduce/run_local_api_benchmark.py`. The client accepts explicit model, tokenizer, prompt, visible-text quality gate, sample count, and artifact path; it cannot send traffic to a LAN or public address and never starts or stops a server. Future release work must add parameterized model-launch recipes, but public runners must continue to avoid user-specific home paths, LAN addresses, or an assumed production service. + +Independent contributors can extend this artifact in four well-defined directions: add a host manifest and clean benchmark matrix for another AMD target; implement a unified-memory-aware expert-cache policy; contribute a profile-ranked HIP kernel candidate; or expand the deterministic and task-level quality suite. Every extension should retain raw evidence, preserve the stated output gate, and report rejected as well as accepted candidates. + + +## 8. Limitations and Reproducibility + +This study reports a single `gfx1151` host, a small number of controlled workloads, and no 24-hour endurance result. It does not establish broad AMD support, cross-device generalization, agentic quality equivalence, or strict parity with the upstream paper. Some currently useful comparisons still involve different representations, such as NVFP4 versus Q4_K_M, and must not be interpreted as architecture-independent engine rankings. + +Future work should recover the upstream Qwen benchmark contract, complete a five-sample paper-matched matrix, add long-context and multi-turn quality controls, measure UMA contention directly, and repeat any accepted optimization on a second clean-host matrix. The public artifact protocol in Section 7 is the required path for reproducing and extending those experiments. + +## 9. Conclusion + +We ported FreeToken to native ROCm/HIP execution on AMD Strix Halo and evaluated it with a claim discipline suited to an evolving edge-serving system. The port compiles and serves through HIP, preserves CUDA as a separate path, and passes deterministic Qwen and Gemma controls. In a same-file Q4 Qwen control, the native HIP router produced 50.63 tokens/s versus 50.29 tokens/s for llama.cpp ROCm 10, while a faster but output-changing NVFP4 router was rejected. Profiling identifies dense FP8 decode as the leading current target and shows that unified-memory expert-cache copies are not, by themselves, the dominant observed decode cost. The result is a reproducible foundation for further UMA-aware optimization, not a claim of paper replication or universal AMD superiority. + +## References + +[1] Shuo Yang et al. *FreeToken: Efficient Edge-Native MoE Serving with Bandwidth-Adaptive Execution.* arXiv:2608.16157, 2026. + +[2] Georgi Gerganov et al. *llama.cpp.* https://github.com/ggml-org/llama.cpp. + +[3] AMD. *ROCm Documentation.* https://rocm.docs.amd.com/. + +[4] FreeToken AMD contributors. *FreeToken AMD ROCm/HIP Port for Strix Halo: Technical White Paper and Artifact Release Candidate v0.1.0-rc1.* Branch `amd-rocm-gfx1151`, commit `a937862f171900bd5d1d207c8ff59b40a15ce742`; tag and DOI pending, 2026. + +[5] Apache Software Foundation. *Apache License, Version 2.0.* https://www.apache.org/licenses/LICENSE-2.0. + +## Appendix A. Claim ledger for reviewers + +| Claim | Evidence status | Boundary | +| -------------------------------------------------- | ---------------------- | ------------------------------------------------------------------------------ | +| Native AMD execution | Established | HIP-compiled extension and native ROCm/HIP server, no Vulkan or CPU substitute | +| Qwen NVFP4 deterministic serving | Established for canary | Reference-router AIME hash, not a complete task-quality suite | +| Qwen NVFP4 27.88 tokens/s | Measured | Three warm quality-matched runs, 54-prompt-token/127-completion-token canary | +| Qwen Q4 50.63 tokens/s | Measured | One same-file raw-prompt control with native HIP router | +| Faster than llama.cpp for Qwen Q4 control | Bounded | 0.7% on one stated steady-decode control, not a general ranking | +| Gemma 4 Q4 57.05 tokens/s | Measured | Fixed text arithmetic control | +| Reproduces FreeToken 39.3 tokens/s RTX 4060 result | Not established | Upstream workload and configuration contract incomplete | +| General Strix Halo or AMD advantage | Not established | One host and limited workload matrix | diff --git a/paper-draft/paper.tex b/paper-draft/paper.tex new file mode 100644 index 0000000000..53d58195f6 --- /dev/null +++ b/paper-draft/paper.tex @@ -0,0 +1,127 @@ +\documentclass[10pt,letterpaper,twocolumn]{article} +\usepackage[margin=0.75in]{geometry} +\usepackage[T1]{fontenc} +\usepackage{lmodern} +\usepackage{microtype} +\usepackage{booktabs} +\usepackage{tabularx} +\usepackage{array} +\usepackage{hyperref} +\usepackage{xurl} +\hypersetup{colorlinks=true,linkcolor=black,citecolor=black,urlcolor=blue} + +\title{Native FreeToken Serving on AMD Strix Halo:\\A ROCm/HIP Port and Controlled Unified-Memory Evaluation} +\author{FreeToken AMD contributors} +\date{Technical white paper, release candidate v0.1.0-rc1, 30 August 2026} + +\begin{document} +\maketitle + +\begin{abstract} +Large mixture-of-experts (MoE) models make capable local inference possible, but most edge-serving systems are designed and evaluated on NVIDIA discrete GPUs. We present a native ROCm/HIP port of FreeToken for AMD Strix Halo, represented by the Ryzen AI Max+ 395 with Radeon 8060S graphics (\texttt{gfx1151}). The port retains FreeToken's CUDA behavior while adding HIP extension builds, ROCm-safe architecture detection, portable Triton paths, and native model-serving validation. It executes without a CUDA compatibility layer, Vulkan substitute, or CPU-only fallback. + +The evaluated GMKtek EVO-X2 has 64 GiB installed LPDDR5 memory and a 4 GiB firmware GPU reservation. Linux exposes 59.46 GiB host memory and ROCm exposes a 56.0 GiB coarse-grained GPU pool. The native server produces a deterministic Qwen NVFP4 AIME canary at 27.88 mean client-visible decode tokens/s. A faster NVFP4 Triton-router path was rejected because it changed deterministic model output. In a matched raw-prompt Q4\_K\_M Qwen control, FreeToken reaches 50.63 tokens/s with a quality-checked HIP router versus 50.29 tokens/s for the ROCm 10 llama.cpp control. A Gemma 4 Q4 text control reaches 57.05 tokens/s and returns the expected deterministic answer. These are native-port and bounded same-file control results, not a strict reproduction of FreeToken's published RTX 4060 result. +\end{abstract} + +\section{Introduction} + +FreeToken shows that MoE-aware placement, caching, and execution policies can turn consumer hardware into a viable local serving platform~\cite{freetoken}. Its design addresses an important edge-inference reality: model state often exceeds device memory, execution alternates between prefill and decode, and agentic workloads repeatedly edit and extend context. The published system, however, is principally designed and evaluated around NVIDIA discrete GPUs. + +AMD Strix Halo changes the deployment model. Its Radeon 8060S GPU and CPU share a large LPDDR5X memory pool rather than communicating through a discrete-GPU PCIe path. This makes large local models feasible on an APU, but it does not make a CUDA-oriented runtime automatically portable or performant. A runtime must compile native extensions with HIP, avoid treating HIP's \texttt{torch.cuda} compatibility namespace as evidence of NVIDIA hardware, preserve semantics across alternate kernels, and measure CPU-GPU contention rather than assuming PCIe transfer is the principal cost. + +We ask a deliberately narrow question: can FreeToken's serving stack be ported to a Strix Halo \texttt{gfx1151} system as a native ROCm/HIP runtime, and what do controlled serving experiments demonstrate after the port? This work contributes a narrowly gated HIP port that preserves CUDA behavior, a validation contract that separates native execution from performance and quality claims, controlled Qwen and Gemma results, and evidence from profiling and rejected candidates that identifies the current optimization frontier. + +\section{Scope and Native Port} + +FreeToken jointly manages model layout, expert residency, CPU-GPU execution, and cache state for local MoE serving~\cite{freetoken}. Its published Qwen result reports 39.3 decode tokens/s on an 8 GiB RTX 4060 laptop. We use that work as design motivation and a protocol reference, not as an automatically comparable baseline. A strict replication would require the same checkpoint revision, workload corpus, rendered prompt, tokenization, generation and stop rules, warmup state, policy configuration, and reported statistic. Those fields have not all been recovered for the upstream RTX 4060 row. No result reported here is therefore described as a replication of that result. + +The port retains CUDA as a separate runtime path. On ROCm builds, setup detects HIP PyTorch and links the native extension surface against \texttt{libamdhip64} rather than CUDA runtime libraries. A compatibility header maps only the CUDA Runtime API subset used by FreeToken's pinned-memory and CPU-MoE extensions to HIP. JIT compilation removes NVCC-only flags and uses HIP-compatible launch behavior. CUDA-only optional dependencies and NVIDIA PTX inline assembly are avoided on HIP, with portable Triton paths used where validated. The port also rejects ROCm before NVIDIA SM capability checks, preventing \texttt{gfx1151} from being misclassified as an NVIDIA architecture. + +The resulting server preserves FreeToken's OpenAI-compatible model discovery, streaming, non-streaming, cache, and MoE interfaces. All experiments use the ROCm/HIP path. We did not use Vulkan or a CPU-only runner as an implementation substitute. + +\section{Experimental Methodology} + +Experiments ran on a GMKtek EVO-X2. Table~\ref{tab:platform} reports the static environment observed on 30 August 2026. Dynamic measurements such as free memory, temperature, clocks, and active processes are retained per run in the artifact manifest rather than presented as fixed specifications. + +\begin{table*}[t] +\caption{Evaluated-system hardware and software environment.} +\label{tab:platform} +\centering +\small +\begin{tabularx}{\textwidth}{>{\raggedright\arraybackslash}p{0.28\textwidth}X} +\toprule +Component & Specification \\ +\midrule +System and firmware & GMKtek EVO-X2, SKU \texttt{EVO-X2-001}, hardware version 1.0, firmware EVO-X2 1.09 dated 13 September 2025 \\ +Processor & AMD Ryzen AI Max+ 395 with Radeon 8060S; 16 cores, 32 hardware threads, one NUMA node, boost enabled \\ +CPU frequency and cache & 625 MHz minimum and 5.1875 GHz maximum; 768 KiB L1d, 512 KiB L1i, 16 MiB L2, and 64 MiB L3 \\ +Installed memory & 64 GiB LPDDR5, eight 8 GiB Micron devices; 8,532 MT/s rated and 8,000 MT/s configured \\ +Memory exposure & 4 GiB firmware UMA reservation; 59.46 GiB Linux host memory; 56.0 GiB ROCm coarse-grained GPU pool \\ +GPU and HSA & Radeon 8060S, PCI ID \texttt{1002:1586}, \texttt{gfx1151}, 40 compute units, wavefront size 32, XNACK disabled, coherent host access false \\ +Operating system & Ubuntu 26.04.1 LTS, Linux 7.0.0-30-generic \\ +Toolchain & ROCm 10.0, HIP 7.15.26333, AMD Clang 23, PyTorch 2.13.0+rocm10.0.0, Triton 3.8.0 \\ +Storage & Lexar ARES 2 TB NVMe SSD \\ +\bottomrule +\end{tabularx} +\end{table*} + +The firmware reservation is not the FreeToken model-memory budget. It is a preallocated UMA region. Capacity available to a request changes with host activity, runtime overhead, model weights, expert residency, and KV-cache growth. + +Decode throughput is client-visible streaming throughput: generated completion tokens, excluding the first generated token, divided by the interval from the first to final streamed content token. Client-observed time to first token is reported separately from runtime-internal timing. Fixed-length throughput and quality are separate modes, so a system cannot appear faster merely by ending early, emitting hidden reasoning tokens, or silently changing the request. + +Every accepted candidate must satisfy the applicable gates: native HIP compilation and execution, an OpenAI-compatible response, correct tokenizer accounting, deterministic-output or task-quality evidence, and preserved raw artifacts. A microbenchmark gain is not accepted if the full model changes the deterministic answer or fails to improve the end-to-end API workload. + +\section{Results} + +\subsection{Native Qwen NVFP4 serving} + +The native ROCm/HIP Qwen server passed a deterministic AIME canary with the reference PyTorch router. Three warm quality-matched repeats produced 26.786, 28.422, and 28.431 client-visible tokens/s, for a mean of 27.880 tokens/s. Mean warm time to first token was 409.0 ms for the 54-prompt-token and 127-completion-token request. Every run emitted the required SHA-1, \texttt{0acef4eab6f4}. + +A ROCm Triton top-k router improved isolated router latency by 1.62 to 1.63x for Qwen's 256-expert top-8 shape and achieved 29.186 tokens/s in a performance-only NVFP4 workload. An end-to-end greedy AIME request changed output hash, however, so this configuration is rejected for NVFP4 serving. Router-only speed and a transport canary are not quality-preserving system results. + +\subsection{Matched Q4 Qwen raw-prompt control} + +Table~\ref{tab:q4} compares FreeToken and llama.cpp~\cite{llamacpp} on the same Q4\_K\_M file, raw prompt, tokenizer count, deterministic sampling, and steady-decode rule. With the native HIP router, FreeToken reaches 50.63 tokens/s while preserving the correct derivation for the expected answer. This is 0.7\% above the llama.cpp control's 50.29 tokens/s. + +\begin{table}[t] +\caption{Matched raw-prompt Qwen Q4\_K\_M control.} +\label{tab:q4} +\centering +\small +\begin{tabular}{p{0.35\columnwidth}rrr} +\toprule +Engine & Prompt & Generated & Tokens/s \\ +\midrule +FreeToken AMD & 54 & 1023 & 47.12 \\ +FreeToken AMD plus HIP router & 54 & 1023 & 50.63 \\ +llama.cpp ROCm 10 & 54 & 1024 & 50.29 \\ +\bottomrule +\end{tabular} +\end{table} + +This is intentionally a bounded result. Both outputs remained within Qwen's reasoning trace at the 1024-token ceiling, so neither exposed the requested boxed final line. Future quality experiments should use a larger generation cap or a concise-answer task, repeated samples, and a task suite. + +\subsection{Gemma 4 and profiling evidence} + +The native Gemma GGUF path returned \texttt{323} for a fixed multiplication prompt. The client and local tokenizer agreed on 30 prompt tokens; the response used four completion tokens and reached 57.05 steady decode tokens/s. This validates the text-only loader, canonical template, OpenAI-compatible completion API, and token accounting for this fixed control. It does not alone qualify multimodal handling or long-context behavior. + +The Qwen NVFP4 ROCm trace recorded 353,457 dispatches. It was intrusive and measured only 15.61 tokens/s, so it is not used for throughput scoring. In the final active window, the largest GPU-time consumer was dense mixed-FP8 \texttt{\_gemv\_splitk\_kernel}, not the routed NVFP4 expert kernel. Measured times were 5,631.844 ms for \texttt{\_gemv\_splitk\_kernel}, 1,676.018 ms for \texttt{\_gemm\_kernel}, 1,566.004 ms for \texttt{\_decode\_nvfp4\_marlin\_kernel}, and 593.192 ms for \texttt{fast\_index\_copy}. + +The port also measured the Qwen cache-copy path. With one active token, eight routed experts missing, and a 513-slot cache, native HIP copied 13.5 MiB in 0.097 ms, or 146.8 GB/s. The all-hit case took 0.023 ms. Across 40 MoE layers, the documented all-miss extrapolation is 3.87 ms per decode token, below the approximately 35 ms end-to-end token interval of the accepted NVFP4 configuration. This does not prove copies are irrelevant, but it rules out treating cache-copy bypass as the first unvalidated optimization. + +\section{Limitations and Artifact Availability} + +This study reports a single \texttt{gfx1151} host, a small set of controlled workloads, and no 24-hour endurance result. It does not establish broad AMD support, cross-device generalization, agentic quality equivalence, or strict parity with the upstream paper. The Q4 control is the cleanest current cross-runtime result because it holds the model file and raw prompt constant, but its 0.7\% margin is too small to generalize beyond the stated workload. + +The AMD ROCm/HIP port is developed under Apache-2.0 at \url{https://github.com/dbourdea/FreeToken}, branch \texttt{amd-rocm-gfx1151}~\cite{freetokenamd,apache}. This release candidate is based on public branch tip \texttt{a937862f171900bd5d1d207c8ff59b40a15ce742}, verified on 30 August 2026. The white-paper package and portable reproduction tools are not yet committed to that branch, and no immutable tag or DOI exists. Before publication, commit the complete package, archive an immutable tag, and replace this statement with the tag and version DOI. The release candidate includes HIP portability tests, Qwen and Gemma controls, and the read-only collector \texttt{scripts/reproduce/collect\_host\_manifest.sh}, which redacts the hostname by default and does not change service or host configuration. + +The archival release must contain source and an environment lockfile, workload and scoring code, sanitized raw results and table-generation scripts, and model provenance consisting of publisher, revision, byte count, SHA-256, and license. Model weights must be obtained from their original publishers rather than redistributed without permission. The current host-specific helpers preserve a protected local service and use host-specific paths, but the public artifact now provides a parameterized host collector and the loopback-only API client \texttt{benchmarks/reproduce/run\_local\_api\_benchmark.py}. The client accepts an explicit model, tokenizer, prompt, visible-text quality gate, sample count, and artifact path; it cannot send traffic to a LAN or public address and never starts or stops a server. Future release work must add parameterized model-launch recipes while avoiding user-specific home paths, LAN addresses, or an assumed production service. + + +\section{Conclusion} + +We ported FreeToken to native ROCm/HIP execution on AMD Strix Halo and evaluated it with claim discipline suited to an evolving edge-serving system. The port compiles and serves through HIP, preserves CUDA as a separate path, and passes deterministic Qwen and Gemma controls. In a same-file Q4 Qwen control, the native HIP router produced 50.63 tokens/s versus 50.29 tokens/s for llama.cpp ROCm 10, while a faster but output-changing NVFP4 router was rejected. The result is a reproducible foundation for UMA-aware optimization, not a claim of paper replication or universal AMD superiority. + +\bibliographystyle{plain} +\bibliography{references} +\end{document} diff --git a/paper-draft/references.bib b/paper-draft/references.bib new file mode 100644 index 0000000000..957715eae0 --- /dev/null +++ b/paper-draft/references.bib @@ -0,0 +1,29 @@ +@article{freetoken, + title = {FreeToken: Efficient Edge-Native {MoE} Serving with Bandwidth-Adaptive Execution}, + author = {Yang, Shuo and Fan, Xiaoze and Pan, Melissa and Xi, Haocheng and Wang, Zhe and Sun, Shanlin and Keutzer, Kurt and Han, Song and Zaharia, Matei and Xu, Chenfeng and Stoica, Ion}, + journal = {arXiv preprint arXiv:2608.16157}, + year = {2026}, + url = {https://arxiv.org/abs/2608.16157} +} + +@misc{llamacpp, + title = {llama.cpp}, + author = {Gerganov, Georgi and contributors}, + year = {2026}, + howpublished = {\url{https://github.com/ggml-org/llama.cpp}} +} + +@misc{freetokenamd, + title = {FreeToken AMD ROCm/HIP Port for Strix Halo}, + author = {{FreeToken AMD contributors}}, + year = {2026}, + note = {Release candidate v0.1.0-rc1 based on branch \texttt{amd-rocm-gfx1151}, commit \texttt{a937862f171900bd5d1d207c8ff59b40a15ce742}; immutable tag and DOI pending}, + howpublished = {\url{https://github.com/dbourdea/FreeToken}} +} + +@misc{apache, + title = {Apache License, Version 2.0}, + author = {{Apache Software Foundation}}, + year = {2004}, + howpublished = {\url{https://www.apache.org/licenses/LICENSE-2.0}} +} diff --git a/pyproject.toml b/pyproject.toml index 8bd653f87d..4d2e991d80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ # own, and a mismatch links the C++ extensions against the wrong libtorch. # setuptools floor: 77 is the first release that understands the PEP 639 `license` # SPDX string and `license-files` below. -requires = ["setuptools>=77", "torch>=2.11,<2.12", "wheel"] +requires = ["setuptools>=77", "torch>=2.11,<2.14", "wheel"] build-backend = "setuptools.build_meta" [project] @@ -22,6 +22,7 @@ classifiers = [ "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Operating System :: POSIX :: Linux", + "Environment :: GPU :: AMD ROCm", "Environment :: GPU :: NVIDIA CUDA", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", @@ -47,6 +48,9 @@ dependencies = [ "numpy>=2.0,<2.5", "openai>=2.0,<3", "partial-json-parser>=0.2,<1", + # OpenAI-compatible image inputs decode through Pillow before being packed + # into Gemma4's patch-major CPU tensors for the tokenizer-to-engine wire. + "pillow>=10,<12", "prompt_toolkit>=3.0,<4", "pydantic>=2.9,<3", "pyzmq>=27,<28", @@ -54,10 +58,15 @@ dependencies = [ # floor+ceiling: sglang-kernel 0.4.5 links libtorch symbols only 2.11 has. # PyPI's torch 2.11.0 wheel is itself the cu130 build, so plain pip resolves # correctly from PyPI alone; uv additionally pins the index below. - "torch>=2.11,<2.12", + # ROCm note: on AMD (rocm.nightlies.amd.com builds) this range is intentionally + # loosened -- those wheels report their own local version segment + # (2.13.0a0+rocm...) which the sglang-kernel/cu130 constraint above doesn't apply to. + "torch>=2.11,<2.14", "tqdm>=4.66,<5", + # What: add tomli for Python versions earlier than 3.11; why: those interpreters lack tomllib, so the daemon needs this fallback to parse TOML model catalogs. + "tomli>=2.0,<3; python_version < '3.11'", "transformers>=5.5,<6", - "triton==3.6.0; platform_system == 'Linux'", + "triton>=3.6,<3.8; platform_system == 'Linux'", "uvicorn>=0.30,<1", ] diff --git a/python/freetoken/attention/__init__.py b/python/freetoken/attention/__init__.py index 746c04c4bc..8a410ffe4a 100644 --- a/python/freetoken/attention/__init__.py +++ b/python/freetoken/attention/__init__.py @@ -33,10 +33,6 @@ class BackendInfo: # Whether forward() honors a per-call AttentionSpec (window/sm_scale/sinks). # Non-consumers raise on a non-None spec instead of silently dropping it. consumes_attn_spec: bool = False - # Whether this backend coexists with hybrid-linear (GDN/mamba) models. The - # linear layers bypass the backend entirely, but a backend whose metadata or - # graph machinery assumes layer 0 is an attention layer can opt out here. - hybrid_linear_ok: bool = True SUPPORTED_ATTENTION_BACKENDS = Registry[BackendCreator]("Attention Backend") @@ -111,6 +107,11 @@ def create_dsv4_sparse_backend(config: ModelConfig): BackendInfo(supported_types=frozenset({AttnType.MLA, AttnType.DSA})), ) def create_dsa_backend(config: ModelConfig): + # MLA with a grouped index (index_ratio > 1) is the kpool indexer layout. + if any(s.mla and s.index_ratio > 1 for s in config.kv_cache_group_specs()): + from .dsa_indexer_kpool import Glm5NextDSABackend + + return Glm5NextDSABackend(config) from .dsa import DSAAttnBackend return DSAAttnBackend(config) @@ -132,6 +133,21 @@ def create_m3_sparse_backend(config: ModelConfig): return M3SparseAttnBackend(config) +@SUPPORTED_ATTENTION_BACKENDS.register( + "qsa_sparse", + BackendInfo( + supported_types=frozenset({AttnType.QSA}), + # 64-token pages: a 4-token compress group never straddles a page, so the + # compressed row of a group is page_base // 4 + block-in-page. + page_sizes=(64,), + ), +) +def create_qsa_sparse_backend(config: ModelConfig): + from .qsa_sparse import QSASparseAttnBackend + + return QSASparseAttnBackend(config) + + def attention_backend_info(name: str) -> BackendInfo: return SUPPORTED_ATTENTION_BACKENDS.info(name) diff --git a/python/freetoken/attention/base.py b/python/freetoken/attention/base.py index eb39d4721c..d28050bd86 100644 --- a/python/freetoken/attention/base.py +++ b/python/freetoken/attention/base.py @@ -24,6 +24,10 @@ class AttnType(str, Enum): # GQA block-sparse (MiniMax-M3): paged GQA K/V + a per-sparse-layer index-key # slab; the indexer picks top-k 128-token blocks per query -> BSAKVCache BSA = "bsa" + # QSA compressed-block sparse (Qwen3.8-Flash-Next): paged GQA K/V + a compressed + # index-key slab (one row per index_ratio tokens, row = slot // index_ratio) + + # a per-request pending ring for unclosed groups -> QSAKVCache + QSA = "qsa" @property def backend_driven(self) -> bool: @@ -37,6 +41,9 @@ class AttentionSpec: sliding_window: int | None = None sm_scale: float | None = None sinks: torch.Tensor | None = None + # Gemma 4's sliding-attention layers make image soft-token groups + # bidirectional during prefill. Full-attention layers remain causal. + multimodal_bidirectional: bool = False @dataclass diff --git a/python/freetoken/attention/dsa.py b/python/freetoken/attention/dsa.py index ff4fa7ca6e..398d170022 100644 --- a/python/freetoken/attention/dsa.py +++ b/python/freetoken/attention/dsa.py @@ -31,7 +31,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Dict, List, Tuple +from typing import TYPE_CHECKING, Dict, List, Tuple, NamedTuple import torch from freetoken.core import Batch, get_global_ctx @@ -50,6 +50,32 @@ _PREFILL_SCORE_CHUNK = 512 +class KpoolPlan(NamedTuple): + """glm5_next kpool: one forward's slab/ring routing (layer-invariant; the + first DSA layer plans, the rest reuse -- Glm5NextDSABackend._plan_kpool_writes).""" + + cmp_rows: torch.Tensor # [T] shadow row (closing) or scratch row + ring_rows: torch.Tensor # [T] tail-ring row, -1 = masked off + ring_slots: torch.Tensor # [n_req] Req.table_idx + token_to_req: torch.Tensor # [T] + cu_seqlens: torch.Tensor # [n_req + 1] + + +@dataclass(frozen=True) +class DSAIndexerInputs: + """Per-forward indexer tensors, built by the MODEL's indexer module and passed + per call (QSAIndexerInputs precedent): the backend holds no model parameters. + ``gate``/``ape`` are the glm5_next kpool extras (None for GLM-5.2): the raw + per-channel compression gate scores and the [kpool, Di] APE weight.""" + # fmt: off + q: torch.Tensor # [T, Hi, Di] + k: torch.Tensor # [T, Di] + w: torch.Tensor # [T, Hi] fp32 + gate: torch.Tensor | None = None + ape: torch.Tensor | None = None + # fmt: on + + @dataclass class DSAMetadata(BaseAttnMetadata): # fmt: off @@ -63,6 +89,11 @@ class DSAMetadata(BaseAttnMetadata): kvlen: torch.Tensor | None = None # group leader layer -> (sel_rows, counts); only the LIVE leader is retained sel: dict = field(default_factory=dict) + # glm5_next kpool decode: per-request tail-ring slots (Req.table_idx). Under CUDA + # graphs this is a view of the backend's STATIC buffer (restaged per replay by + # _stage_decode, QSA ring_slots precedent); eager decode reads active_table_idx. + ring_slots: torch.Tensor | None = None + kpool_plan: "KpoolPlan | None" = None # fmt: on def get_last_indices(self, bs: int) -> torch.Tensor: @@ -73,8 +104,7 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend): def __init__(self, config: ModelConfig) -> None: from freetoken.kvcache.dsa_pool import DSAKVCache, MLAKVCache - args = config.glm_dsa_args - assert args is not None, "dsa backend needs ModelConfig.glm_dsa_args (MLA dims)" + args = self._model_args(config) self.config = config self.num_heads = config.num_qo_heads self.kv_lora_rank = args.kv_lora_rank @@ -99,15 +129,7 @@ def __init__(self, config: ModelConfig) -> None: self._leader: Dict[int, int] = {} self._idx_slot: Dict[int, int] = {} if self.dsa_enabled: - lead = None - # Capped to the SERVED layer count (dev num_layers overrides must not - # index slots past the pool the factory sized from the same cap). - for lid, kind in enumerate(args.indexer_types[: config.num_layers]): - if kind == "full": - lead = lid - self._idx_slot[lid] = len(self._idx_slot) - assert lead is not None, "indexer_types must start with a 'full' layer" - self._leader[lid] = lead + self._build_index_slots(args, config) # decode staging (static buffers under CUDA graphs; eager decode builds # per-forward tensors in prepare_metadata instead) self._rows_buf: torch.Tensor | None = None @@ -115,6 +137,32 @@ def __init__(self, config: ModelConfig) -> None: self.max_seq_len = 0 self.capture_bs: List[int] = [] + # ----- model-family hooks (overridden by the glm5_next kpool backend) --------------- + def _model_args(self, config: ModelConfig): + """The MLA/indexer dims payload. GLM-5.2 reads glm_dsa_args; glm5_next's + subclass reads glm5_args (same duck-typed fields).""" + args = config.glm_dsa_args + assert args is not None, "dsa backend needs ModelConfig.glm_dsa_args (MLA dims)" + return args + + def _build_index_slots(self, args, config: ModelConfig) -> None: + """IndexShare (GLM-5.2): "full" layers own an indexer slot; followers reuse + the most recent leader's selection.""" + lead = None + # Capped to the SERVED layer count (dev num_layers overrides must not + # index slots past the pool the factory sized from the same cap). + for lid, kind in enumerate(args.indexer_types[: config.num_layers]): + if kind == "full": + lead = lid + self._idx_slot[lid] = len(self._idx_slot) + assert lead is not None, "indexer_types must start with a 'full' layer" + self._leader[lid] = lead + + def _store_index(self, inputs: "DSAIndexerInputs", batch: Batch, layer_id: int) -> None: + """Scatter this forward's index keys (kpool subclass adds gate scores and + pool-completion compression).""" + self.kvcache.store_index_k(inputs.k, batch.out_loc, self._idx_slot[layer_id]) + def forward(self, q, k, v, layer_id, batch, attn_spec: AttentionSpec | None = None): raise NotImplementedError("MLA models use mla_forward(), not forward().") @@ -156,14 +204,14 @@ def _attend( ) def mla_forward( - self, q_nope, q_pe, c_kv, k_rope, layer_id, batch, indexer_qkw=None + self, q_nope, q_pe, c_kv, k_rope, layer_id, batch, indexer_inputs=None ) -> torch.Tensor: """Store this forward's latent rows and attend over the paged latent history. ``q_nope`` [T, H, kv_lora_rank] (kv_b-absorbed), ``q_pe`` [T, H, rope_dim], ``c_kv`` [T, kv_lora_rank] / ``k_rope`` [T, rope_dim] (the pool scatters the - two latent halves). ``indexer_qkw`` = (q [T, Hi, Di], k [T, Di], w [T, Hi]) - on full-indexer layers, None on shared layers. Returns [T, H, kv_lora_rank]. + two latent halves). ``indexer_inputs`` is a :class:`DSAIndexerInputs` on + full-indexer layers, None on shared layers. Returns [T, H, kv_lora_rank]. """ md = batch.attn_metadata assert isinstance(md, DSAMetadata) @@ -174,17 +222,17 @@ def mla_forward( md.rows = self._decode_rows(batch).to(torch.int32) md.kvlen = md.kv_len_cpu.to(self.device, non_blocking=True) self.kvcache.store_kv(c_kv, k_rope, batch.out_loc, layer_id) - if self.dsa_enabled and indexer_qkw is not None: + if self.dsa_enabled and indexer_inputs is not None: # Scatter index keys unconditionally: short prefills serve through the # identity path TODAY, but their keys must exist once decode passes topk. - self.kvcache.store_index_k(indexer_qkw[1], batch.out_loc, self._idx_slot[layer_id]) + self._store_index(indexer_inputs, batch, layer_id) if md.is_decode: - return self._decode(md, layer_id, q_nope, q_pe, indexer_qkw) - return self._prefill(md, layer_id, q_nope, q_pe, batch, indexer_qkw) + return self._decode(md, layer_id, q_nope, q_pe, indexer_inputs) + return self._prefill(md, layer_id, q_nope, q_pe, batch, indexer_inputs) # ----- decode (CUDA-graph capturable, single code path) ----------------------------- - def _decode(self, md, layer_id, q_nope, q_pe, indexer_qkw) -> torch.Tensor: + def _decode(self, md, layer_id, q_nope, q_pe, inputs) -> torch.Tensor: bs = q_nope.shape[0] rows, kvlen = md.rows, md.kvlen if not self.dsa_enabled: @@ -192,8 +240,8 @@ def _decode(self, md, layer_id, q_nope, q_pe, indexer_qkw) -> torch.Tensor: # whole row list, bounded by the device-side live length. sel, cnt = rows.view(bs, 1, -1), kvlen.view(bs, 1) else: - if indexer_qkw is not None: - q_idx, _, w = indexer_qkw + if inputs is not None: + q_idx, w = inputs.q, inputs.w s = self.dsa_decode_scores(q_idx, w, self._idx_slot[layer_id], rows, kvlen) k_sel = min(self.index_topk, s.shape[-1]) picks = self.indexer_select_decode( @@ -212,15 +260,16 @@ def _decode(self, md, layer_id, q_nope, q_pe, indexer_qkw) -> torch.Tensor: # ----- prefill / extend (eager) ------------------------------------------------------ def _select_prefill( self, slot: int, q_idx: torch.Tensor, w: torch.Tensor, - rows: torch.Tensor, positions: torch.Tensor, + rows: torch.Tensor, positions: torch.Tensor, start_pos: int, ) -> Tuple[torch.Tensor, torch.Tensor]: - """Per-request causal top-k: ([1, m, K] physical rows, [1, m] counts).""" + """Per-request causal top-k: ([1, m, K] physical rows, [1, m] counts). + ``start_pos`` is the request's first query position -- a host int + (cached_len), so no device->host sync on the prefill path.""" kv_len = rows.numel() k_all = self.kvcache.index_k_cache(slot).index_select(0, rows.long()) k_sel = min(self.index_topk, kv_len) m = q_idx.shape[0] sel = torch.empty(m, k_sel, dtype=torch.int32, device=self.device) - start_pos = int(positions[0]) # Bound the fp32 [chunk, kv_len] logits transient (worst case is capped by the # model's max_position: floor 16 x 1M x 4 B = 64 MB, see _PREFILL_SCORE_BYTES). chunk = max(16, min(_PREFILL_SCORE_CHUNK, _PREFILL_SCORE_BYTES // max(kv_len * 4, 1))) @@ -236,15 +285,16 @@ def _select_prefill( cnt = torch.clamp(positions + 1, max=k_sel).to(torch.int32) return sel.view(1, m, k_sel), cnt.view(1, m) - def _prefill(self, md, layer_id, q_nope, q_pe, batch, indexer_qkw) -> torch.Tensor: + def _prefill(self, md, layer_id, q_nope, q_pe, batch, inputs) -> torch.Tensor: t = q_nope.shape[0] q_cat = torch.cat([q_nope, q_pe], dim=-1) # [T, H, 576] reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs page_table = get_global_ctx().page_table qo = md.qo_indptr_cpu.tolist() - sparse = self.dsa_enabled and int(md.kv_len_cpu.max()) > self.index_topk - if sparse and indexer_qkw is not None: - q_idx, _, w = indexer_qkw + kv_lens = md.kv_len_cpu.tolist() # one D2H for both the gate and start_pos + sparse = self.dsa_enabled and max(kv_lens) > self.index_topk + if sparse and inputs is not None: + q_idx, w = inputs.q, inputs.w md.sel.clear() # one live group leader at a time md.sel[layer_id] = [ self._select_prefill( @@ -252,6 +302,7 @@ def _prefill(self, md, layer_id, q_nope, q_pe, batch, indexer_qkw) -> torch.Tens q_idx[qo[i] : qo[i + 1]], w[qo[i] : qo[i + 1]], page_table[r.table_idx, : r.device_len], batch.positions[qo[i] : qo[i + 1]], + kv_lens[i] - (qo[i + 1] - qo[i]), # cached_len == first position ) for i, r in enumerate(reqs) ] diff --git a/python/freetoken/attention/dsa_indexer_kpool.py b/python/freetoken/attention/dsa_indexer_kpool.py new file mode 100644 index 0000000000..6743a1438b --- /dev/null +++ b/python/freetoken/attention/dsa_indexer_kpool.py @@ -0,0 +1,288 @@ +"""glm5_next (GLM-5.3-Flash) kpool DSA backend: pooled-indexer addressing. + +Extends the GLM-5.2 DSA backend with the kpool compression scheme: the indexer K +cache is scored at POOL granularity -- every ``index_kpool`` (4) consecutive +tokens fold into one entry, a per-channel ``softmax(gate + APE)``-weighted sum of +their raw keys -- so indexer compute and top-k shrink by 4x. Selection picks +``index_topk // kpool`` pools, each pool expands back to its constituent token +rows, and the request's trailing incomplete pool ("tail", up to kpool-1 tokens) +is force-included (``index_kpool_always_select_tail``). Selection widths are +therefore ``(kpool - 1) + select_k * kpool`` with ``-1`` gather-only sentinels; +the sparse-MLA kernel masks them. + +Slab convention (KpoolDSAKVCache): the index slab is a 1/kpool SHADOW of the KV +pages -- a pool's entry lives at row ``token_slot // kpool`` (well-defined because +the engine pins ``page_size % kpool == 0``, so a pool never straddles a page). +Raw keys + gate scores of the in-progress pool live in per-request tail rings +written at ``pos % kpool``; a pool that closes reads its older members from the +ring (so a chunk may start mid-pool), and a decode step whose pool does NOT close +scatters its (garbage) pooled candidate into the request's scratch row -- an +unconditional, CUDA-graph-safe write that scoring never reads (QSA precedent). + +Faithfulness: pooled entries are stored bf16 (the reference Hadamard-rotates and +fp8-quantizes them -- a memory device whose rotation cancels in the score); the +pooling softmax runs fp32, matching the reference kernel. Selection is a plain +top-k over pool scores plus the tail: the reference does NOT force-include the +query's own (complete) pool -- the model is trained with that scheme, so neither +do we. Sharing note: IndexShare does not exist here; every DSA layer owns its +indexer slot and is its own leader. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Tuple + +import torch + +from .dsa import DSAAttnBackend, DSAMetadata, KpoolPlan + +if TYPE_CHECKING: + from freetoken.core import Batch + from freetoken.models import ModelConfig + + +class Glm5NextDSABackend(DSAAttnBackend): + def __init__(self, config: "ModelConfig") -> None: + super().__init__(config) + self._slots_buf: torch.Tensor | None = None # static graph buffer for ring_slots + if self.dsa_enabled: + from freetoken.kvcache.dsa_pool import KpoolDSAKVCache + + assert isinstance(self.kvcache, KpoolDSAKVCache), ( + "glm5_next kpool indexer needs the KpoolDSAKVCache (tail rings); " + f"the pool factory built {type(self.kvcache).__name__}" + ) + + # ----- CUDA-graph decode staging ------------------------------------------------------ + # The tail rings are keyed by Req.table_idx; a captured decode step must read it + # from a static buffer restaged per replay (rows/kvlen precedent in the parent). + def init_capture_graph(self, max_seq_len: int, bs_list) -> None: + super().init_capture_graph(max_seq_len, bs_list) + self._slots_buf = torch.zeros(max(bs_list), dtype=torch.int64, device=self.device) + + def _stage_decode(self, batch: "Batch", bs: int, table_idx: torch.Tensor) -> None: + super()._stage_decode(batch, bs, table_idx) + self._slots_buf[:bs].copy_(table_idx) + batch.attn_metadata.ring_slots = self._slots_buf[:bs] + + def reset_capture(self) -> None: + super().reset_capture() + self._slots_buf = None + + # ----- model-family hooks ----------------------------------------------------------- + def _model_args(self, config: "ModelConfig"): + args = config.glm5_args + assert args is not None, "kpool dsa backend needs ModelConfig.glm5_args" + self.kpool = args.index_kpool + assert self.kpool > 1 and args.index_kpool_compress, ( + "Glm5NextDSABackend serves the kpool-compressed indexer; a kpool=1 " + "checkpoint should run the plain DSA backend" + ) + assert args.index_topk % self.kpool == 0 + self.select_k = args.index_topk // self.kpool + assert args.index_kpool_always_select_tail, ( + "tail force-inclusion is baked into the selection layout" + ) + return args + + def _build_index_slots(self, args, config: "ModelConfig") -> None: + # No IndexShare: every DSA layer owns its indexer and is its own leader. + for lid in args.dsa_layer_ids: + if lid >= config.num_layers: + continue + self._idx_slot[lid] = len(self._idx_slot) + self._leader[lid] = lid + + # ----- store: fused single path (prefill AND decode, CUDA-graph capturable) ---------- + def _plan_kpool_writes(self, md, batch: "Batch", slot: int): + """Per-token slab/ring routing for this forward; layer-invariant, so the + first DSA layer computes it and the rest reuse it (QSA _plan_index_writes + shape). Pure device arithmetic: no host sync, graph-capturable. + + Rebuilt at the FIRST indexer slot of every forward, never trusted across + forwards: a capture batch runs its warmup and its capture through ONE + metadata object, and a cached plan would bake the warmup's (non-graph-pool) + tensor addresses into the graph (QSA precedent).""" + if slot != 0 and md.kpool_plan is not None: + return md.kpool_plan + kp = self.kpool + out_loc = batch.out_loc.to(torch.int64) + positions = batch.positions.to(torch.int64) + t = out_loc.numel() + if md.is_decode: + # One token per request. ring_slots is the backend's STATIC buffer under + # graphs (restaged per replay in _stage_decode); eager reads the + # scheduler-staged active_table_idx. arange shapes are fixed per capture. + ring_slots = ( + md.ring_slots if md.ring_slots is not None else batch.active_table_idx + ).to(torch.int64) + token_to_req = torch.arange(t, device=self.device, dtype=torch.int32) + cu_seqlens = torch.arange(t + 1, device=self.device, dtype=torch.int32) + else: + reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs + ring_slots = torch.tensor( + [r.table_idx for r in reqs], dtype=torch.int64, pin_memory=True + ).to(self.device, non_blocking=True) + cu_cpu = md.qo_indptr_cpu + cu_seqlens = cu_cpu.to(self.device, non_blocking=True) + token_to_req = torch.repeat_interleave( + torch.arange(len(reqs), dtype=torch.int32), + (cu_cpu[1:] - cu_cpu[:-1]).to(torch.int64), + ).pin_memory().to(self.device, non_blocking=True) + slots = ring_slots.index_select(0, token_to_req.to(torch.int64)) + # page_size % kpool == 0, so out_loc % kp == position % kp: a group closes + # exactly on position % kp == kp - 1. Non-closing rows land in the request's + # scratch row (never scored). + closing = positions % kp == kp - 1 + cmp_rows = torch.where( + closing, out_loc // kp, self.kvcache.cmp_scratch_base + slots + ).to(torch.int32) + # Ring refresh: only each request's last kp rows survive to the next forward + # (one keeper per pos%kp residue -- deterministic, no write races). + rows = torch.arange(t, device=self.device, dtype=torch.int64) + ends = cu_seqlens.to(torch.int64).index_select(0, token_to_req.to(torch.int64) + 1) + keep = rows >= ends - kp + ring_row = slots * kp + positions % kp + ring_rows = torch.where(keep, ring_row, torch.full_like(ring_row, -1)).to( + torch.int32 + ) + md.kpool_plan = KpoolPlan(cmp_rows, ring_rows, ring_slots, token_to_req, cu_seqlens) + return md.kpool_plan + + def _store_index(self, inputs, batch: "Batch", layer_id: int) -> None: + """One fused kernel serves prefill and decode (layout and member + resolution: module docstring); the ring refresh follows the read.""" + from freetoken.kernel.triton.kpool_compress import kpool_compress_store + from freetoken.kernel.triton.qsa import qsa_store_rows + + md = batch.attn_metadata + assert isinstance(md, DSAMetadata) + assert inputs.gate is not None and inputs.ape is not None, ( + "kpool store needs DSAIndexerInputs.gate/ape from the model's indexer" + ) + k, gate = inputs.k, inputs.gate.to(inputs.k.dtype) + slot = self._idx_slot[layer_id] + tail_k, tail_g = self.kvcache.tail_k(slot), self.kvcache.tail_gate(slot) + plan = self._plan_kpool_writes(md, batch, slot) + kpool_compress_store( + k, gate, + tail_k.view(-1, k.shape[-1]), tail_g.view(-1, k.shape[-1]), + inputs.ape, + plan.ring_slots, plan.token_to_req, plan.cu_seqlens, batch.positions, + self.kvcache.index_k_cache(slot), plan.cmp_rows, + self.kpool, + ) + # After the compression read: the ring rows this forward overwrites are + # exactly the ones a straddling group just consumed. + qsa_store_rows(tail_k, plan.ring_rows, k) + qsa_store_rows(tail_g, plan.ring_rows, gate) + + # ----- selection: pools -> token rows + tail ------------------------------------------ + def _expand_and_tail( + self, + picks: torch.Tensor, # [B, m, select_k] pool ids, -1 sentinel + rows: torch.Tensor, # [B, W] or [B, m, W] position-ordered physical rows + q_pos: torch.Tensor, # [B, m] query positions (token-granular) + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Selected pools -> token rows, tail tokens appended FIRST (fixed kpool-1 + slots, -1 padded). Returns (sel [B, m, (kpool-1) + select_k*kpool] int32, + cnt [B, m] int32).""" + kp = self.kpool + b, m, k_sel = picks.shape + device = picks.device + offs = torch.arange(kp, device=device) + + # History: pool pick p -> token positions p*kp + [0, kp). + hist_pos = picks.unsqueeze(-1) * kp + offs # [B, m, k_sel, kp] + hist_pos = torch.where( + picks.unsqueeze(-1) < 0, hist_pos.new_full((), -1), hist_pos + ).view(b, m, k_sel * kp) + + # Tail: positions [n_pools*kp, q_pos] of each query's own request/step. + n_pools = (q_pos + 1) // kp # complete pools at this query + tail_start = n_pools * kp + toffs = torch.arange(kp - 1, device=device) + tail_pos = tail_start.unsqueeze(-1) + toffs # [B, m, kp-1] + tail_pos = torch.where(tail_pos <= q_pos.unsqueeze(-1), tail_pos, tail_pos.new_full((), -1)) + + pos = torch.cat([tail_pos, hist_pos], dim=-1) # [B, m, width] + rows_b = rows if rows.dim() == 3 else rows.unsqueeze(1).expand(b, m, -1) + sel = rows_b.gather(-1, pos.clamp_min(0).long()).to(torch.int32) + sel = torch.where(pos < 0, sel.new_full((), -1), sel) + # Valid entries: the fixed tail slots + all expanded picked pools. -1 + # sentinels inside the bound are masked by the sparse kernel. + # clamp_max (not minimum(new_tensor)): no H2D copy, CUDA-graph safe. + cnt = ((kp - 1) + n_pools.clamp_max(k_sel) * kp).to(torch.int32) + return sel, cnt + + def _decode(self, md, layer_id, q_nope, q_pe, inputs) -> torch.Tensor: + bs = q_nope.shape[0] + rows, kvlen = md.rows, md.kvlen + if not self.dsa_enabled: + return super()._decode(md, layer_id, q_nope, q_pe, inputs) + if inputs is not None: + q_idx, w = inputs.q, inputs.w + kp = self.kpool + n_pools = (kvlen // kp).to(torch.int32) + # Pool p's shadow row = any member's token slot // kp (pools never + # straddle pages): stride the position-ordered row snapshot down to + # pool granularity, then divide into the shadow slab. + pool_rows = (rows[:, kp - 1 :: kp] // kp).contiguous() + s = self.dsa_decode_scores(q_idx, w, self._idx_slot[layer_id], pool_rows, n_pools) + k_sel = min(self.select_k, s.shape[-1]) + picks = self.indexer_select_decode( + s.view(bs, 1, -1), valid=n_pools, topk=k_sel, offset=0 + ) # [bs, 1, k_sel] pool ids + sel, cnt = self._expand_and_tail( + picks.transpose(0, 1), # [1, bs, k_sel] + rows.unsqueeze(0), # [1, bs, W] + (kvlen - 1).view(1, bs), + ) + sel = sel.transpose(0, 1) # [bs, 1, width] + cnt = cnt.view(bs, 1) + md.sel.clear() + md.sel[layer_id] = (sel, cnt) + sel, cnt = md.sel[self._leader[layer_id]] + q_cat = torch.cat([q_nope, q_pe], dim=-1).view(bs, 1, self.num_heads, self.latent_dim) + o = self._attend(q_cat, layer_id, sel, cnt) + return o.view(bs, self.num_heads, self.kv_lora_rank) + + def _select_prefill( + self, slot: int, q_idx: torch.Tensor, w: torch.Tensor, + rows: torch.Tensor, positions: torch.Tensor, start_pos: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Per-request causal top-k at pool granularity, expanded to token rows. + ``start_pos`` is the request's cached_len (host int, no device sync).""" + kp = self.kpool + kv_len = rows.numel() + n_pools_total = kv_len // kp + pool_rows = rows[kp - 1 :: kp] // kp # shadow rows (token slot // kpool) + k_pool = self.kvcache.index_k_cache(slot).index_select(0, pool_rows.long()) + k_sel = min(self.select_k, max(n_pools_total, 1)) + m = q_idx.shape[0] + width = (kp - 1) + k_sel * kp + if n_pools_total == 0: + # No complete pool to score (a sub-kpool request riding a sparse batch): + # the selection is the tail alone (select_prefill would return zero pick + # columns and break the fixed width below). + picks = torch.full((1, m, k_sel), -1, dtype=torch.int32, device=self.device) + return self._expand_and_tail(picks, rows.view(1, -1), positions.view(1, -1)) + sel = torch.empty(m, width, dtype=torch.int32, device=self.device) + cnt = torch.empty(m, dtype=torch.int32, device=self.device) + chunk = 512 + for s0 in range(0, m, chunk): + s1 = min(s0 + chunk, m) + scores = self.dsa_prefill_logits(q_idx[s0:s1], k_pool, w[s0:s1]) + picks = self.indexer_select_prefill( + scores.unsqueeze(0), start_pos=start_pos + s0, seqlen=s1 - s0, + ratio=kp, topk=k_sel, offset=0, + ) # [1, s1-s0, k_sel] pool ids + sel_c, cnt_c = self._expand_and_tail( + picks, rows.view(1, -1), positions[s0:s1].view(1, -1) + ) + sel[s0:s1] = sel_c[0] + cnt[s0:s1] = cnt_c[0] + return sel.view(1, m, width), cnt.view(1, m) + + +__all__ = ["Glm5NextDSABackend"] diff --git a/python/freetoken/attention/linear.py b/python/freetoken/attention/linear.py index 1283ba025f..f3717e58e7 100644 --- a/python/freetoken/attention/linear.py +++ b/python/freetoken/attention/linear.py @@ -41,6 +41,7 @@ class FLAMetadata: track_dst: torch.Tensor | None = None # [nt] int64 dst pool slot per tracked req track_h_row: torch.Tensor | None = None # [nt] int64 row into h (boh_i + aligned//CHUNK) track_conv_src: torch.Tensor | None = None # [nt, kernel-1] int64 conv-input token positions + track_boundary_row: torch.Tensor | None = None # [nt] int64 forward-local row of the track boundary; states with their own left context (qwen4_exp PLE) derive their windows from it def build_fla_metadata(batch: "Batch", device: torch.device) -> FLAMetadata: @@ -55,7 +56,7 @@ def build_fla_metadata(batch: "Batch", device: torch.device) -> FLAMetadata: builder serves the eager scheduler path and direct-op test callers. """ reqs = batch.padded_reqs - pin = {"device": "cpu", "pin_memory": True} + pin = {"device": "cpu", "pin_memory": torch.cuda.is_available()} # GDN state slot per request: the hybrid-radix live slot (decoupled from table_idx) when # allocated, else table_idx (naive / force-naive GDN models keep the old keying). @@ -77,7 +78,7 @@ def gdn_slot(r): fresh = [gdn_slot(r) for r in reqs if r.cached_len == 0] fresh_host = torch.tensor(fresh, dtype=torch.int64, **pin) if fresh else None - track_dst, track_h_row, track_conv_src = _build_track_metadata(reqs, cu_host, device, pin) + track = _build_track_metadata(reqs, cu_host, device, pin) return FLAMetadata( cu_seqlens=cu_host.to(device, non_blocking=True), @@ -86,24 +87,29 @@ def gdn_slot(r): fresh_state_indices=( fresh_host.to(device, non_blocking=True) if fresh_host is not None else None ), - track_dst=track_dst, track_h_row=track_h_row, track_conv_src=track_conv_src, + **track, ) def _build_track_metadata(reqs, cu_host, device, pin): """Hybrid-radix (extra_buffer): for each request that crosses a ×CHUNK boundary this prefill forward, snapshot its GDN state at the deepest mid-chunk boundary into its current - ping-pong slot. Returns (track_dst, track_h_row, track_conv_src) device int64 tensors, or - (None, None, None) when no request tracks (non-hybrid, or all extends < CHUNK+1).""" + ping-pong slot. Returns the ``FLAMetadata`` track kwargs, all None when no request + tracks (non-hybrid, or all extends < CHUNK+1).""" + empty = dict(track_dst=None, track_h_row=None, track_conv_src=None, track_boundary_row=None) if not any(r.mamba_ping_pong is not None for r in reqs): - return None, None, None + return empty from freetoken.core import get_global_ctx from freetoken.kernel.fla.chunk import CHUNK_SIZE from freetoken.kernel.fla.index import prepare_chunk_offsets km1 = get_global_ctx().linear_state_pool.conv_states.shape[-1] # conv_kernel_dim - 1 + assert km1 <= CHUNK_SIZE, ( + f"conv history {km1} exceeds CHUNK_SIZE {CHUNK_SIZE}: the snapshot window " + "would reach before this forward's first token" + ) boh = prepare_chunk_offsets(cu_host, CHUNK_SIZE).tolist() - dst, h_row, conv_src = [], [], [] + dst, h_row, conv_src, boundary_rows = [], [], [], [] for i, r in enumerate(reqs): if r.mamba_ping_pong is None: continue @@ -117,13 +123,18 @@ def _build_track_metadata(reqs, cu_host, device, pin): dst.append(r.mamba_ping_pong[r.mamba_next_track_idx]) h_row.append(boh[i] + c) conv_src.append([off + c * CHUNK_SIZE - km1 + j for j in range(km1)]) + boundary_rows.append(off + c * CHUNK_SIZE) r.mamba_last_track_seqlen = boundary r.mamba_next_track_idx = 1 - r.mamba_next_track_idx if not dst: - return None, None, None + return empty to = lambda xs, **kw: torch.tensor(xs, **pin, **kw).to(device, non_blocking=True) - return (to(dst, dtype=torch.int64), to(h_row, dtype=torch.int64), - to(conv_src, dtype=torch.int64)) + return dict( + track_dst=to(dst, dtype=torch.int64), + track_h_row=to(h_row, dtype=torch.int64), + track_conv_src=to(conv_src, dtype=torch.int64), + track_boundary_row=to(boundary_rows, dtype=torch.int64), + ) __all__ = ["FLAMetadata", "build_fla_metadata"] diff --git a/python/freetoken/attention/qsa_sparse.py b/python/freetoken/attention/qsa_sparse.py new file mode 100644 index 0000000000..4a28dc8529 --- /dev/null +++ b/python/freetoken/attention/qsa_sparse.py @@ -0,0 +1,507 @@ +"""Qwen3.8-Flash-Next QSA compressed-block sparse attention backend. + +Serves ``AttnType.QSA`` over ``kvcache/qsa_pool.py``: paged GQA K/V for the 12 full-attention +layers, a compressed index-key slab holding one key per ``index_ratio`` tokens, and a +per-request pending ring for the group a forward leaves open. The 36 GDN layers never reach +this backend, and the model has no dense attention layer, so :meth:`forward` is not served -- +the only entry point is :meth:`qsa_forward` (``models/qwen4_exp/attention.py``). + +One QSA layer's forward, all ragged over ``[T, ...]`` metadata: + +1. store K/V at ``batch.out_loc``; +2. pool each row's closing group (members at positions >= ``cached_len`` come from this + forward's raw index keys, the older ones from the pending ring), zero-centered rmsnorm it + and rope it at the group's first position, then scatter it into the slab row + ``out_loc // index_ratio`` (rows whose group does not close land on the request's scratch + row and are never read); +3. store this forward's last ``ring_capacity`` raw index keys per request into the ring; +4. norm+rope the indexer queries at their own positions; +5. score every COMPLETE visible block (``sum_h relu() / sqrt(index_head_dim)``, + clamped to ``kvlen // index_ratio`` -- slab rows are never cleared, so stale rows must stay + unreachable), take the top ``index_budget // index_ratio`` blocks, expand them to token + indices plus the causal tail of the open group; +6. attend to exactly those tokens. + +Addressing: the engine pins ``page_size == 64`` (this backend's ``page_sizes``), so a group of +``index_ratio`` tokens never straddles a page and ``block_table[req, p] = page_table[req, p * +64] // 64`` names both the K/V page and, viewed as ``page_size // index_ratio`` compressed +rows, the block's slab page. Decode stages that table plus the live lengths and table_idx into +static buffers (``prepare_for_replay``) so the whole path is CUDA-graph capturable. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import TYPE_CHECKING, Callable, List + +import torch +from freetoken.core import Batch, get_global_ctx +from freetoken.utils import init_logger + +from .base import AttentionSpec, BaseAttnBackend, BaseAttnMetadata + +logger = init_logger(__name__) + +if TYPE_CHECKING: + from freetoken.models import ModelConfig + +_CPU_PINNED = {"device": "cpu", "dtype": torch.int32, "pin_memory": True} +# Block-score transient budget (vLLM's number): the fp32 [rows, n_blocks] logits tile is +# 256 KB per row at a 1M-token context, so a long prefill must be scored in row chunks. +_LOGITS_WORKSPACE_BYTES = 128 << 20 + + +TORCH_TOPK_ENV = "FREETOKEN_QSA_TORCH_TOPK" + + +def _resolve_block_topk() -> Callable | None: + """The in-repo Triton block top-k, or None to fall back on torch.topk.""" + if os.getenv(TORCH_TOPK_ENV, "0") == "1": + logger.info(f"qsa_sparse block top-k: torch.topk ({TORCH_TOPK_ENV}=1)") + return None + try: + from freetoken.kernel.triton.qsa import qsa_block_topk + except Exception as exc: + logger.info(f"qsa_sparse block top-k: torch.topk (triton unavailable: {exc})") + return None + logger.info("qsa_sparse block top-k: triton qsa_block_topk") + return qsa_block_topk + + +@dataclass +class QSASparseMetadata(BaseAttnMetadata): + # fmt: off + is_decode: bool + last_indices: torch.Tensor # gpu + qo_indptr_cpu: torch.Tensor # cpu pinned int32 [bs+1] + kv_len_cpu: torch.Tensor # cpu pinned int32 [bs] + # Ragged per-token / per-request addressing. Decode defers these to the static graph + # buffers (prepare_for_replay) or to a lazy eager snapshot at the first QSA layer. + token_to_req: torch.Tensor | None = None # [T] int32 + cu_seqlens: torch.Tensor | None = None # [bs+1] int32 + seq_lens: torch.Tensor | None = None # [bs] int32, device_len + ring_slots: torch.Tensor | None = None # [bs] int32, Req.table_idx + block_table: torch.Tensor | None = None # [bs, W//page_size] int32, physical page ids + # Per-forward scatter plans, built once by the first QSA layer and reused by the rest. + # positions is bound here (not in prepare_metadata) because a capture batch has none yet. + cmp_rows: torch.Tensor | None = None # [T] int32, compressed slab destination + ring_rows: torch.Tensor | None = None # [T] int32, flat ring row or -1 + positions: torch.Tensor | None = None # [T] int32, logical query positions + # fmt: on + + def get_last_indices(self, bs: int) -> torch.Tensor: + return self.last_indices[:bs] + + +class QSASparseAttnBackend(BaseAttnBackend): + def __init__(self, config: ModelConfig) -> None: + from freetoken.kvcache.qsa_pool import QSAKVCache + + args = config.qwen4_args + assert args is not None, "qsa_sparse backend needs ModelConfig.qwen4_args" + self.head_dim = config.head_dim + self.index_heads = args.index_n_heads + self.token_topk = args.index_budget + self.kvcache = get_global_ctx().kv_cache + assert isinstance(self.kvcache, QSAKVCache), ( + f"qsa_sparse backend needs a QSA pool, got {type(self.kvcache).__name__}" + ) + self.device = self.kvcache.device + self.dtype = self.kvcache.dtype + self.index_head_dim = self.kvcache.index_head_dim + self.ratio = self.kvcache.index_ratio + self.ring_capacity = self.kvcache.ring_capacity + self.page_size = get_global_ctx().page_size + assert self.page_size % self.ratio == 0, ( + f"QSA needs page_size ({self.page_size}) divisible by index_ratio ({self.ratio})" + ) + self.cmp_page_size = self.page_size // self.ratio + self.block_topk = self.token_topk // self.ratio + self.select_width = self.token_topk + self.ratio - 1 + assert self.token_topk % self.ratio == 0, "QSA budget must be a whole number of blocks" + # The sparse attend kernel bakes 1/sqrt(head_dim) into its exp2 scale. + assert config.attn_sm_scale in (None, self.head_dim**-0.5), ( + "qsa_sparse serves the default 1/sqrt(head_dim) attention scale only" + ) + # QSA layer -> index slab slot, in sparse-layer order (the pool's own convention). + group = self._qsa_group(config) + self._idx_slot = {lid: i for i, lid in enumerate(group.layer_ids)} + self.rotary_config = group.rotary_config + self._index_cos_sin: torch.Tensor | None = None + + self._block_topk_kernel = _resolve_block_topk() + # decode staging (static buffers under CUDA graphs; eager decode snapshots per step) + self._graph: dict[str, torch.Tensor] = {} + self.capture_bs: List[int] = [] + + @staticmethod + def _qsa_group(config: ModelConfig): + from freetoken.models.config import FullAttentionGroupConfig + + groups = [ + g + for g in config.attention_groups + if isinstance(g, FullAttentionGroupConfig) and g.index_ratio > 1 + ] + assert len(groups) == 1, f"expected one QSA attention group, got {len(groups)}" + return groups[0] + + # ----- slab views --------------------------------------------------------------------- + def _cmp_pages(self, slot: int) -> torch.Tensor: + """The compressed slab as ``[pages, page_size // ratio, 1, dim]``, the score kernel's + paged layout. The scratch rows past ``cmp_scratch_base`` stay out of the view.""" + rows = self.kvcache.cmp_k_cache(slot)[: self.kvcache.cmp_scratch_base] + return rows.view(-1, self.cmp_page_size, 1, self.index_head_dim) + + def _index_rope_cache(self) -> torch.Tensor: + """cos/sin table of the indexer rope: same rotary_dim and frequencies as the main + attention, ``head_size`` 128 instead of 256, so it is a separate get_rope instance. + + The table itself (not RotaryEmbedding.forward) because the indexer's norm+rope is one + fused kernel and the compressed keys rope at their group's position, not the query's.""" + if self._index_cos_sin is None: + from freetoken.layers.rotary import get_rope + + rotary = self.rotary_config + with torch.device(self.device): + rope = get_rope( + head_dim=self.index_head_dim, + rotary_dim=rotary.rotary_dim, + max_position=rotary.max_position, + base=rotary.base, + rope_scaling=tuple(rotary.scaling.items()) if rotary.scaling else None, + ) + self._index_cos_sin = rope._cos_sin_cache.to(self.device) + return self._index_cos_sin + + # ----- metadata ----------------------------------------------------------------------- + def prepare_metadata(self, batch: Batch) -> None: + reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs + seqlens_q = [r.extend_len for r in reqs] + seqlens_k = [r.device_len for r in reqs] + is_decode = getattr(batch, "phase", None) == "decode" + qo_indptr = torch.tensor([0] + seqlens_q, **_CPU_PINNED).cumsum_(0).to(torch.int32) + kv_len = torch.tensor(seqlens_k, **_CPU_PINNED) + last = (qo_indptr[1:].to(torch.int32) - 1).to(self.device, non_blocking=True) + md = QSASparseMetadata( + is_decode=is_decode, + last_indices=last, + qo_indptr_cpu=qo_indptr, + kv_len_cpu=kv_len, + ) + batch.attn_metadata = md + if not is_decode: + table_idx = torch.tensor([r.table_idx for r in reqs], **_CPU_PINNED) + token_to_req = torch.repeat_interleave( + torch.arange(len(reqs), dtype=torch.int32), + torch.tensor(seqlens_q, dtype=torch.int32), + ).pin_memory() + md.cu_seqlens = qo_indptr.to(self.device, non_blocking=True) + md.token_to_req = token_to_req.to(self.device, non_blocking=True) + md.seq_lens = kv_len.to(self.device, non_blocking=True) + md.ring_slots = table_idx.to(self.device, non_blocking=True) + md.block_table = self._block_table(md.ring_slots.to(torch.int64)) + # Decode addressing is DEFERRED: a graph-bound step stages it into the static + # buffers (prepare_for_replay), an eager step snapshots at the first QSA layer. + + def _block_base_view(self) -> torch.Tensor: + """Every-``page_size``-th column of the page table: the per-page base slots. A strided + VIEW, so gathering rows through it materializes only [bs, W/page_size].""" + return get_global_ctx().page_table[:, :: self.page_size] + + def _block_table(self, table_idx: torch.Tensor) -> torch.Tensor: + return (self._block_base_view().index_select(0, table_idx) // self.page_size).to( + torch.int32 + ) + + def _stage_decode(self, md: QSASparseMetadata, bs: int, table_idx: torch.Tensor) -> None: + """Copy this step's addressing into the static graph buffers and point the metadata + at them (restage-per-replay, m3/dsa precedent).""" + self._graph["block_table"][:bs].copy_( + self._block_base_view().index_select(0, table_idx) // self.page_size + ) + self._graph["kvlen"][:bs].copy_(md.kv_len_cpu.to(self.device, non_blocking=True)) + self._graph["table_idx"][:bs].copy_(table_idx) + md.block_table = self._graph["block_table"][:bs] + md.seq_lens = self._graph["kvlen"][:bs] + md.ring_slots = self._graph["table_idx"][:bs] + md.token_to_req = self._graph["token_to_req"][:bs] + md.cu_seqlens = self._graph["cu_seqlens"][: bs + 1] + + def _snapshot_decode(self, md: QSASparseMetadata, batch: Batch) -> None: + """Eager decode (not graph-staged): this step's rows, once per forward. The live + page-table row may mutate for the next batch while this one runs, so gather now.""" + reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs + bs = len(reqs) + table_idx = torch.tensor([r.table_idx for r in reqs], **_CPU_PINNED) + md.ring_slots = table_idx.to(self.device, non_blocking=True) + md.block_table = self._block_table(md.ring_slots.to(torch.int64)) + md.seq_lens = md.kv_len_cpu.to(self.device, non_blocking=True) + md.token_to_req = torch.arange(bs, dtype=torch.int32, device=self.device) + md.cu_seqlens = torch.arange(bs + 1, dtype=torch.int32, device=self.device) + + # ----- dense layers ------------------------------------------------------------------- + def forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + layer_id: int, + batch: Batch, + attn_spec: AttentionSpec | None = None, + ) -> torch.Tensor: + raise NotImplementedError( + "qsa_sparse serves QSA layers only (Qwen3.8-Flash-Next has no dense attention " + "layer); the QSA layer calls qsa_forward" + ) + + # ----- QSA layers --------------------------------------------------------------------- + def qsa_forward( + self, + q: torch.Tensor, # [T, HQ, D] + k: torch.Tensor, # [T, KVH * D] + v: torch.Tensor, # [T, KVH * D] + index, # models.qwen4_exp.attention.QSAIndexerInputs + layer_id: int, + batch: Batch, + ) -> torch.Tensor: + from freetoken.kernel.triton.qsa import qsa_sparse_paged_attention + + md = batch.attn_metadata + assert isinstance(md, QSASparseMetadata) + slot = self._idx_slot[layer_id] + self.kvcache.store_kv(k, v, batch.out_loc, layer_id) + if md.block_table is None: + self._snapshot_decode(md, batch) + if slot == 0 or md.cmp_rows is None: + # Rebuilt at the first QSA layer of every forward, not cached on the metadata: a + # capture batch runs its warmup and its capture through ONE metadata object, and a + # cached plan would bake the warmup's addresses into the graph. + self._plan_index_writes(md, batch) + + self._update_index_cache(index, md, slot) + indices = self._select(index, md, slot) + return qsa_sparse_paged_attention( + q, + self.kvcache.k_cache(layer_id), + self.kvcache.v_cache(layer_id), + indices, + md.block_table, + md.token_to_req, + torch.empty_like(q), + ) + + def _plan_index_writes(self, md: QSASparseMetadata, batch: Batch) -> None: + """Per-token slab row and ring row for this forward; the other QSA layers reuse it + (it is layer-invariant). Pure device arithmetic: no host sync, graph-capturable.""" + md.positions = batch.positions + out_loc = batch.out_loc.to(torch.int64) + positions = batch.positions.to(torch.int64) + rows = torch.arange(out_loc.numel(), device=self.device) + req = md.token_to_req.to(torch.int64) + slots = md.ring_slots.to(torch.int64).index_select(0, req) + # out_loc % page_size == position % page_size and index_ratio divides page_size, so a + # group closes exactly on out_loc % index_ratio == index_ratio - 1. + closing = out_loc % self.ratio == self.ratio - 1 + scratch = self.kvcache.cmp_scratch_base + slots + md.cmp_rows = torch.where(closing, out_loc // self.ratio, scratch).to(torch.int32) + # Only the last ring_capacity rows of a request survive to the next forward; the rest + # are masked off instead of dumped somewhere (vLLM rule). + ends = md.cu_seqlens.to(torch.int64).index_select(0, req + 1) + keep = rows >= ends - self.ring_capacity + ring_row = slots * self.ring_capacity + positions % self.ring_capacity + md.ring_rows = torch.where(keep, ring_row, torch.full_like(ring_row, -1)).to( + torch.int32 + ) + + def _update_index_cache(self, index, md: QSASparseMetadata, slot: int) -> None: + """Compress each closing group into the slab, then refresh the pending ring.""" + from freetoken.kernel.triton.qsa import ( + qsa_compress_groups, + qsa_index_norm_rope, + qsa_store_rows, + ) + + rows = index.k.shape[0] + ring = self.kvcache.pending_ring(slot) + pooled = self._scratch("pooled", rows, self.index_head_dim, dtype=self.dtype) + first = self._scratch("first_pos", rows, dtype=torch.int32) + qsa_compress_groups( + index.k, + ring, + md.ring_slots, + md.token_to_req, + md.cu_seqlens, + md.positions, + self.ratio, + pooled, + first, + ) + qsa_index_norm_rope( + pooled, + first, + self._index_rope_cache(), + index.k_norm_weight, + index.eps, + self.kvcache.cmp_k_cache(slot), + dest_rows=md.cmp_rows, + ) + # After the compression read: the ring rows this forward overwrites are exactly the + # ones a straddling group just consumed. + qsa_store_rows(ring, md.ring_rows, index.k) + + def _select(self, index, md: QSASparseMetadata, slot: int) -> torch.Tensor: + """Score complete visible blocks, take the top-k, expand them to token indices.""" + from freetoken.kernel.triton.qsa import ( + expand_qsa_block_indices, + qsa_index_norm_rope, + qsa_mqa_paged, + ) + + rows = index.q.shape[0] + positions = md.positions + q_index = self._scratch( + "q_index", rows, self.index_heads, self.index_head_dim, dtype=self.dtype + ) + qsa_index_norm_rope( + index.q.view(-1, self.index_head_dim), + positions, + self._index_rope_cache(), + index.q_norm_weight, + index.eps, + q_index.view(-1, self.index_head_dim), + heads=self.index_heads, + ) + cmp_pages = self._cmp_pages(slot) + columns = md.block_table.shape[1] * self.cmp_page_size + indices = self._scratch("indices", rows, self.select_width, dtype=torch.int32) + rows_per_chunk = max(1, _LOGITS_WORKSPACE_BYTES // max(columns * 4, 1)) + for start in range(0, rows, rows_per_chunk): + end = min(start + rows_per_chunk, rows) + chunk = slice(start, end) + logits = self._scratch("logits", end - start, columns, dtype=torch.float32) + visible = self._scratch("visible", end - start, dtype=torch.int32) + qsa_mqa_paged( + q_index[chunk], + cmp_pages, + md.block_table, + md.token_to_req[chunk], + positions[chunk], + md.seq_lens, + self.ratio, + logits, + visible, + ) + blocks = self._scratch("blocks", end - start, self.block_topk, dtype=torch.int32) + self._top_blocks(logits, visible, blocks) + expand_qsa_block_indices( + blocks, + positions[chunk], + md.seq_lens, + md.token_to_req[chunk], + self.ratio, + self.token_topk, + indices[chunk], + ) + return indices + + def _top_blocks( + self, + logits: torch.Tensor, + visible: torch.Tensor, + blocks: torch.Tensor, + ) -> None: + """Top ``block_topk`` complete blocks per row, row-relative, -1 padded.""" + assert blocks.shape == (logits.shape[0], self.block_topk), ( + f"qsa block top-k output must be [rows, {self.block_topk}], got {tuple(blocks.shape)}" + ) + if self._block_topk_kernel is not None: + scratch_width = self._topk_scratch_width(logits.shape[1]) + scratch = ( + self._scratch("topk_scratch", logits.shape[0], scratch_width, dtype=torch.int32) + if scratch_width + else None + ) + self._block_topk_kernel(logits, visible, blocks, scratch) + return + # The score kernel only writes columns below visible_blocks; mask the rest so a + # stale row cannot win a slot. Real block scores are relu sums, never -inf. + columns = logits.shape[1] + column = torch.arange(columns, dtype=torch.int32, device=logits.device) + logits.masked_fill_(column.unsqueeze(0) >= visible.unsqueeze(1), -float("inf")) + width = min(self.block_topk, columns) + values, chosen = torch.topk(logits, width, dim=-1) + blocks[:, :width] = torch.where(values > -float("inf"), chosen.to(torch.int32), -1) + if width < self.block_topk: + blocks[:, width:] = -1 + + def _topk_scratch_width(self, columns: int) -> int: + """int32 columns per row the block top-k wants as scratch, 0 when it wants none.""" + if self._block_topk_kernel is None: + return 0 + from freetoken.kernel.triton.qsa import qsa_block_topk_scratch_width + + return qsa_block_topk_scratch_width(columns, self.block_topk) + + # ----- scratch ------------------------------------------------------------------------ + def _scratch(self, name: str, rows: int, *shape: int, dtype: torch.dtype) -> torch.Tensor: + """A per-forward transient: the static decode buffer when it is wide enough (so a + captured graph keeps one address), otherwise a fresh allocation.""" + buffer = self._graph.get(name) + if buffer is not None and rows <= buffer.shape[0] and buffer.shape[1:] == shape: + return buffer[:rows] + return torch.empty((rows, *shape), dtype=dtype, device=self.device) + + # ----- CUDA graph (decode) -------------------------------------------------------------- + def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None: + self.capture_bs = sorted(bs_list) + max_bs = max(bs_list) + width = get_global_ctx().page_table.shape[1] + pages = -(-width // self.page_size) + columns = pages * self.cmp_page_size + chunk = max(1, min(max_bs, _LOGITS_WORKSPACE_BYTES // max(columns * 4, 1))) + topk_scratch = self._topk_scratch_width(columns) + + def empty(*shape: int, dtype: torch.dtype) -> torch.Tensor: + return torch.empty(shape, dtype=dtype, device=self.device) + + self._graph = { + "block_table": torch.zeros((max_bs, pages), dtype=torch.int32, device=self.device), + "kvlen": torch.zeros(max_bs, dtype=torch.int32, device=self.device), + "table_idx": torch.zeros(max_bs, dtype=torch.int32, device=self.device), + "token_to_req": torch.arange(max_bs, dtype=torch.int32, device=self.device), + "cu_seqlens": torch.arange(max_bs + 1, dtype=torch.int32, device=self.device), + "logits": empty(chunk, columns, dtype=torch.float32), + "visible": empty(max_bs, dtype=torch.int32), + "blocks": empty(max_bs, self.block_topk, dtype=torch.int32), + "indices": empty(max_bs, self.select_width, dtype=torch.int32), + "pooled": empty(max_bs, self.index_head_dim, dtype=self.dtype), + "first_pos": empty(max_bs, dtype=torch.int32), + "q_index": empty(max_bs, self.index_heads, self.index_head_dim, dtype=self.dtype), + } + if topk_scratch: + self._graph["topk_scratch"] = empty(chunk, topk_scratch, dtype=torch.int32) + + def prepare_for_capture(self, batch: Batch) -> None: + self.prepare_metadata(batch) + md = batch.attn_metadata + assert isinstance(md, QSASparseMetadata) + bs = batch.size + dummy = torch.full( + (bs,), batch.padded_reqs[0].table_idx, dtype=torch.int64, device=self.device + ) + self._stage_decode(md, bs, dummy) + + def prepare_for_replay(self, batch: Batch) -> None: + md = batch.attn_metadata + assert isinstance(md, QSASparseMetadata) + assert batch.active_table_idx is not None, "decode batch is missing its page-table rows" + self._stage_decode(md, batch.padded_size, batch.active_table_idx.to(torch.int64)) + + def reset_capture(self) -> None: + super().reset_capture() + self._graph = {} + + +__all__ = ["QSASparseAttnBackend", "QSASparseMetadata"] diff --git a/python/freetoken/attention/triton.py b/python/freetoken/attention/triton.py index 9eed1e1d21..64a0911c4f 100644 --- a/python/freetoken/attention/triton.py +++ b/python/freetoken/attention/triton.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, Iterable, List import torch from freetoken.core import Batch, get_global_ctx @@ -70,6 +70,10 @@ class TritonMetadata(BaseAttnMetadata): is_decode: bool prefix_lens: torch.Tensor max_q_len: int + # Per-query-token image-group ids during prefill. ``-1`` denotes normal + # causal text. Equal non-negative ids may attend to one another in either + # direction, as required by Gemma 4 image soft-token blocks. + image_group_ids: torch.Tensor | None = None attn_logits: torch.Tensor | None = None attn_lse: torch.Tensor | None = None num_kv_splits: torch.Tensor | None = None @@ -79,6 +83,42 @@ def get_last_indices(self, bs: int) -> torch.Tensor: return self.cu_seqlens_q_gpu[1 : 1 + bs] - 1 +def _image_group_ids_for_prefill( + reqs: Iterable[object], image_token_id: int | None +) -> torch.Tensor | None: + """Return packed prefill image-group ids, or ``None`` for causal-only batches. + + The scheduler packs each request's uncached suffix contiguously. Gemma 4 + requires bidirectional attention only among the repeated soft-image tokens + belonging to the same image, not for surrounding text, delimiters, or a + second image in the same prompt. Group ids are deliberately CPU tensors + here because request token ids are CPU-resident until the scheduler stages + the forward batch. + """ + if image_token_id is None: + return None + pieces: list[torch.Tensor] = [] + next_group = 0 + found_image = False + for req in reqs: + input_ids = req.input_ids[req.cached_len : req.device_len] + groups = torch.full_like(input_ids, -1, dtype=torch.int32) + image_mask = input_ids == image_token_id + if bool(image_mask.any()): + found_image = True + starts = image_mask & torch.cat( + (torch.ones(1, dtype=torch.bool, device=input_ids.device), ~image_mask[:-1]) + ) + for start in starts.nonzero(as_tuple=False).flatten().tolist(): + end = start + while end < input_ids.numel() and bool(image_mask[end]): + end += 1 + groups[start:end] = next_group + next_group += 1 + pieces.append(groups) + return torch.cat(pieces) if found_image else None + + class TritonAttentionBackend(BaseAttnBackend): def __init__(self, config: ModelConfig): self.config = config @@ -157,6 +197,9 @@ def forward( v_cache = v_raw.view(-1, kv_heads, head_dim) spec = attn_spec or AttentionSpec() + image_group_ids = ( + metadata.image_group_ids if spec.multimodal_bidirectional else None + ) indices = metadata.indices if spec.sliding_window is not None and metadata.swa_indices is not None: indices = metadata.swa_indices @@ -185,7 +228,11 @@ def forward( if ( (not metadata.is_decode) and q.dtype in (torch.float16, torch.bfloat16) - and (q.shape[-1] <= 256 or metadata.max_q_len >= self.prefill_tile_min_q) + and ( + q.shape[-1] <= 256 + or metadata.max_q_len >= self.prefill_tile_min_q + or image_group_ids is not None + ) ): return extend_paged_attention( q=q, @@ -201,6 +248,7 @@ def forward( sinks=spec.sinks, k_extend=k.view(q.shape[0], kv_heads, head_dim), v_extend=v.view(q.shape[0], kv_heads, head_dim), + image_group_ids=image_group_ids, ) return paged_attention( q=q, @@ -255,6 +303,11 @@ def prepare_metadata(self, batch: Batch) -> None: q_positions = getattr(batch, "positions", None) if q_positions is None: q_positions = torch.zeros(num_query_tokens, dtype=torch.int64, device=device) + image_group_ids_cpu = ( + _image_group_ids_for_prefill(reqs, getattr(self.config, "image_token_id", None)) + if not is_decode + else None + ) batch.attn_metadata = TritonMetadata( cu_seqlens_q_gpu=cu_seqlens_q_gpu, @@ -265,6 +318,11 @@ def prepare_metadata(self, batch: Batch) -> None: is_decode=is_decode, prefix_lens=prefix_lens, max_q_len=max(seqlens_q), + image_group_ids=( + image_group_ids_cpu.to(device, non_blocking=True) + if image_group_ids_cpu is not None + else None + ), swa_indices=swa_indices, ) diff --git a/python/freetoken/daemon/README.md b/python/freetoken/daemon/README.md index ea3b016eaf..e775b94091 100644 --- a/python/freetoken/daemon/README.md +++ b/python/freetoken/daemon/README.md @@ -45,9 +45,15 @@ ft daemon start MODEL --port 1919 -- --moe-cache-auto # args after -- go to ft ft daemon status ft daemon logs # stream engine logs (SSE) ft daemon health # proxied serve /health (camelCased) -ft daemon metrics # engine-only RAM(PSS)+VRAM footprint +ft daemon metrics # engine-only RAM(PSS)+process GPU-memory footprint ft daemon switch OTHER_MODEL # stop old + start new +ft daemon models # list freetoken-swap named profiles +ft daemon routing-profiles # list runtime model-ID pin profiles +ft daemon activate-routing-profile coding # atomically activate a pin map +ft daemon clear-routing-profile # return to direct model IDs +ft daemon switch-profile coding # atomic switch via the local TOML catalog ft daemon stop +ft daemon shutdown # stop the serve and then the control plane # Recovery only: permit a degraded receipt if the failed engine cannot seal final totals. ft daemon stop --force ``` @@ -55,17 +61,31 @@ ft daemon stop --force Target a non-default daemon with `--url http://host:1900` (or `$FREETOKEN_DAEMON_URL`) and `--token`/`$FREETOKEN_DAEMON_TOKEN`. +For named model catalogs and the `start-profile` / `switch-profile` controls, see +[`docs/freetoken-swap.md`](../../../docs/freetoken-swap.md). Catalog profiles are argument +vectors for `ft serve`, never shell commands. Optional readiness paths and proxy +path prefixes remain restricted to the exact manager-owned loopback `${PORT}` target. + ## HTTP API (camelCase JSON, loopback by default) | Method / path | Notes | | --- | --- | | `GET /health` | Daemon self-health; always answers, never gated by `--token`. | +| `GET /v1/models`, `GET /models` | Identical catalog-key-protected public canonical/optional alternate IDs with atomic loaded/unloaded status and no model paths or launch arguments. | | `POST /engine/start` `{model,port,args[]}` | Idempotent on the full `(model,port,args)`; a differing config on the same port → `409`. | | `POST /engine/stop` `{force?:false}` | Close admission, drain/abort, durably enqueue the final-accounting receipt, then `SIGTERM`→grace→`SIGKILL`. A prepare/outbox failure preserves the engine. | | `POST /engine/switch` `{model,port,args[],force?:false}` | One serialized stop-accounting-start transaction. | +| `GET /router/profiles` | Lists local freetoken-swap named lifecycle profiles for authenticated control clients. | +| `PUT /router/profiles/active` `{name:string\|null}` | Atomically activates or clears a runtime model-ID pin profile. | +| `POST /engine/start-profile\|switch-profile` `{name,force?:false}` | Starts or atomically replaces the engine using a validated local profile. | | `GET /engine/status` | `{running,pid,model,port,uptimeS,lastExitCode,…}`; outlives any single serve. | | `GET /engine/logs?since=` | SSE, ANSI-stripped, tqdm-`\r` collapsed, ring replay, `id:`, `Last-Event-ID` resume. | -| `GET /engine/metrics` | `{ramBytes,vramBytes}` — the serve tree's own footprint only. | +| `GET /router/logs?since=` | SSE, bounded native router admission/proxy/cancellation events. It is separate from engine stdout and records route templates only—never concrete paths, request bodies, headers, query strings, model paths, or keys. | +| `GET /router/activity`, `/router/activity/stats` | Authenticated bounded body-free inference history and aggregates. Real daemon runs fsync and compact rows under `--state-dir`; persistence health is explicit. | +| `GET /api/performance`, `/router/performance` | Authenticated, memory-only one-hour history of owned engine process-tree RAM/VRAM; strict RFC3339 `after` filtering. | +| `GET /router/captures/{id}` | Authenticated opt-in, memory-bounded request/response capture. Credential headers are redacted and binary bodies are Base64. | +| `/upstream/{model-id}/...` | Guarded direct passthrough with longest-prefix slash-namespaced ID resolution and escaped suffix preservation. Safe configured static suffixes return 409 instead of cold-loading and proxy normally when the exact model is resident. | +| `GET /engine/metrics` | The serve tree's own `{ramBytes,vramBytes,pids}` footprint only. `ramAvailable`/`vramAvailable` and source fields distinguish a measured zero from an unavailable probe; Linux PSS, NVIDIA NVML/SMI, and AMD SMI process memory are supported. | | `GET /engine/health` | Proxied serve `/health` + daemon reachability. | | `GET /engine/stats` | Proxied serve `/v1/stats`. | | `GET /accounting/pending` | Unacknowledged durable final-accounting receipts, replayable after a Desktop/client crash. | diff --git a/python/freetoken/daemon/activity.py b/python/freetoken/daemon/activity.py new file mode 100644 index 0000000000..2e7ee8307d --- /dev/null +++ b/python/freetoken/daemon/activity.py @@ -0,0 +1,662 @@ +"""Bounded inference activity and opt-in redacted request/response captures.""" +# What: document bounded inference activity and opt in redacted in the activity docstring; why: introspection and maintainers read this exact docstring fragment to understand activity behavior without executing it. + +# What: enable postponed evaluation of annotations; why: type hints in activity can reference runtime types without eager imports or forward-reference failures. +from __future__ import annotations + +# What: import base64 for record using base64; why: record uses base64 b64encode, making that imported dependency available to its named operation. +import base64 +# What: import ordered dict and deque for init using collections and ordered dict and deque; why: __init__ uses ordered dict and deque, making that imported dependency available to its named operation. +from collections import OrderedDict, deque +# What: import dataclass for module initialization using dataclasses and dataclass; why: module initialization uses dataclass, making that imported dependency available to its named operation. +from dataclasses import dataclass +# What: import hashlib for record using hashlib; why: record uses hashlib sha256, making that imported dependency available to its named operation. +import hashlib +# What: import json for load using json; why: _load uses json jsondecode error, making that imported dependency available to its named operation. +import json +# What: import math for from public using math; why: from_public uses math isfinite, making that imported dependency available to its named operation. +import math +# What: import os for compact locked using os; why: _compact_locked uses os replace, making that imported dependency available to its named operation. +import os +# What: import threading for init using threading; why: __init__ uses threading lock, making that imported dependency available to its named operation. +import threading +# What: import time for record using time; why: record uses time time, making that imported dependency available to its named operation. +import time +# What: import mapping for headers using typing and mapping; why: _headers uses the mapping annotation in headers, making that imported dependency available to its named operation. +from typing import Mapping + + +# What: compute sensitive headers from authorization and proxy authorization and cookie and set cookie and x api key; why: normalized in sensitive headers later reads sensitive headers, so activity must retain the computed value under that name. +_SENSITIVE_HEADERS = { + # What: apply the authorization proxy authorization cookie set cookie x api key x ft token portion of sensitive headers; why: activity uses this clause to evaluate sensitive headers as one grouped value. + "authorization", "proxy-authorization", "cookie", "set-cookie", "x-api-key", "x-ft-token", +# What: complete the _SENSITIVE_HEADERS collection with authorization and proxy authorization and cookie and set cookie; why: activity groups the supplied clauses as one _SENSITIVE_HEADERS collection before its value is consumed. +} +# What: compute max persisted row chars from 8192; why: while line source readline max persisted row chars later reads max persisted row chars, so activity must retain the computed value under that name. +_MAX_PERSISTED_ROW_CHARS = 8192 + + +# What: define _sensitive_header around name; why: its direct callers call _sensitive_header for sensitive header and rely on this exact input and result contract. +def _sensitive_header(name: str) -> bool: + # What: compute normalized from replace and lower and name and value and value; why: parts normalized split later reads normalized, so _sensitive_header must retain the computed value under that name. + normalized = name.lower().replace("_", "-") + # What: compute parts from split and normalized and value; why: or token in parts later reads parts, so _sensitive_header must retain the computed value under that name. + parts = normalized.split("-") + # What: return normalized and sensitive headers and parts and token and secret from _sensitive_header; why: _sensitive_header exposes normalized and sensitive headers and parts and token and secret so its caller can continue with the function\'s computed outcome. + return ( + # What: apply the normalized in sensitive headers portion of the enclosing predicate; why: this clause remains in _sensitive_header\'s enclosing expression so its grouping and evaluation order stay intact. + normalized in _SENSITIVE_HEADERS + # What: apply the or token in parts portion of the enclosing predicate; why: this clause remains in _sensitive_header\'s enclosing expression so its grouping and evaluation order stay intact. + or "token" in parts + # What: apply the or secret in parts portion of the enclosing predicate; why: this clause remains in _sensitive_header\'s enclosing expression so its grouping and evaluation order stay intact. + or "secret" in parts + # What: apply the or api in parts and key portion of the enclosing predicate; why: this clause remains in _sensitive_header\'s enclosing expression so its grouping and evaluation order stay intact. + or ("api" in parts and "key" in parts) + # What: complete the _sensitive_header signature with name; why: _sensitive_header groups the supplied clauses as one _sensitive_header signature before its value is consumed. + ) + + +# What: define _headers around values; why: its direct callers call _headers for headers and rely on this exact input and result contract. +def _headers(values: Mapping[str, str]) -> dict[str, str]: + # What: initialize result as an empty runtime accumulator; why: _headers appends or maps entries into it during result key redacted if sensitive header key else before consuming the aggregate. + result: dict[str, str] = {} + # What: iterate across list and items and values to perform result and key and sensitive header and str and value; why: _headers repeats the body only while or for the loop header admits an iteration. + for key, value in list(values.items())[:64]: + # What: compute result entry from sensitive header and key and str and value and redacted; why: return result later reads result entry, so _headers must retain the computed value under that name. + result[key] = "[REDACTED]" if _sensitive_header(key) else str(value)[:1024] + # What: return result from _headers; why: _headers exposes result so its caller can continue with the function\'s computed outcome. + return result + + +# What: generate dataclass initialization and value semantics for ActivityRecord; why: ActivityRecord acts as a typed state record with consistent construction, comparison, and representation. +@dataclass(frozen=True) +# What: define ActivityRecord as the owner of public and from_public; why: daemon callers use this class boundary so those methods share one activity record state invariant. +class ActivityRecord: + # What: compute id from the named fixture input; why: id self id later reads id, so activity must retain the computed value under that name. + id: int + # What: compute timestamp from the named fixture input; why: timestamp self timestamp later reads timestamp, so activity must retain the computed value under that name. + timestamp: float + # What: compute model from the named fixture input; why: model self model later reads model, so activity must retain the computed value under that name. + model: str + # What: compute route from the named fixture input; why: route self route later reads route, so activity must retain the computed value under that name. + route: str + # What: compute method from the named fixture input; why: method self method later reads method, so activity must retain the computed value under that name. + method: str + # What: compute status from the named fixture input; why: status self status later reads status, so activity must retain the computed value under that name. + status: int + # What: compute duration s from the named fixture input; why: duration s self duration s later reads duration s, so activity must retain the computed value under that name. + duration_s: float + # What: compute ttft s from the named fixture input; why: ttft s self ttft s later reads ttft s, so activity must retain the computed value under that name. + ttft_s: float | None + # What: compute response bytes from the named fixture input; why: response bytes self response bytes later reads response bytes, so activity must retain the computed value under that name. + response_bytes: int + # What: compute cancelled from the named fixture input; why: cancelled self cancelled later reads cancelled, so activity must retain the computed value under that name. + cancelled: bool + # What: compute session id from the named fixture input; why: session id self session id later reads session id, so activity must retain the computed value under that name. + session_id: str | None + # What: compute has capture from the named fixture input; why: has capture self has capture later reads has capture, so activity must retain the computed value under that name. + has_capture: bool + + # What: define public around the current object state; why: its direct callers call public for public and rely on this exact input and result contract. + def public(self) -> dict: + # What: return id and timestamp and model and route from public; why: public exposes id and timestamp and model and route so its caller can continue with the function\'s computed outcome. + return { + # What: map the id field as id; why: ActivityRecord.public carries id into "id": self.id. + "id": self.id, + # What: map the timestamp field as timestamp; why: ActivityRecord.public carries timestamp into "timestamp": self.timestamp. + "timestamp": self.timestamp, + # What: map the model field as model; why: ActivityRecord.public sends this field through "model": self.model so the router selects the canonical model or alias for upstream dispatch. + "model": self.model, + # What: map the route field as route; why: ActivityRecord.public carries route into "route": self.route. + "route": self.route, + # What: map the method field as method; why: ActivityRecord.public carries method into "method": self.method. + "method": self.method, + # What: map the status field as status; why: ActivityRecord.public carries status into "status": self.status. + "status": self.status, + # What: map the duration s field as duration s; why: ActivityRecord.public carries duration s into "durationS": self.duration_s. + "durationS": self.duration_s, + # What: map the ttft s field as ttft s; why: ActivityRecord.public carries ttft s into "ttftS": self.ttft_s. + "ttftS": self.ttft_s, + # What: map the response bytes field as response bytes; why: ActivityRecord.public carries response bytes into "responseBytes": self.response_bytes. + "responseBytes": self.response_bytes, + # What: map the cancelled field as cancelled; why: ActivityRecord.public carries cancelled into "cancelled": self.cancelled. + "cancelled": self.cancelled, + # What: map the session id field as session id; why: ActivityRecord.public carries session id into "sessionId": self.session_id. + "sessionId": self.session_id, + # What: map the has capture field as has capture; why: ActivityRecord.public carries has capture into "hasCapture": self.has_capture. + "hasCapture": self.has_capture, + # What: complete the enclosing predicate mapping with id and timestamp and model and route and method; why: ActivityRecord.public groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + # What: bind from_public to the class rather than an instance; why: factory and parser callers construct from_public from class-level state without requiring an existing object. + @classmethod + # What: define from_public around item; why: the registered API client call from_public for from public and rely on this exact input and result contract. + def from_public(cls, item: dict) -> "ActivityRecord": + # What: gate on isinstance and item and dict before value error; why: from_public admits value error only for this predicate and excludes the opposite state. + if not isinstance(item, dict): + # What: raise ValueError for the caller; why: ActivityRecord.from_public stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("activity row must be an object") + # What: compute integer fields from id and status and response bytes; why: if any type item get key is later reads integer fields, so from_public must retain the computed value under that name. + integer_fields = ("id", "status", "responseBytes") + # What: gate on any and int and key and integer fields and type before value error; why: from_public admits value error only for this predicate and excludes the opposite state. + if any(type(item.get(key)) is not int for key in integer_fields): + # What: raise ValueError for the caller; why: ActivityRecord.from_public stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("activity integer field is invalid") + # What: gate on bool and type and get and item before value error; why: from_public admits value error only for this predicate and excludes the opposite state. + if type(item.get("cancelled")) is not bool: + # What: raise ValueError for the caller; why: ActivityRecord.from_public stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("activity cancellation field is invalid") + # What: iterate across the computed value to perform value and get and key and item; why: from_public repeats the body only while or for the loop header admits an iteration. + for key, maximum in (("model", 128), ("route", 256), ("method", 16)): + # What: compute value from get and key and item; why: if not isinstance value str or later reads value, so from_public must retain the computed value under that name. + value = item.get(key) + # What: gate on value and maximum and isinstance and str and len before value error; why: from_public admits value error only for this predicate and excludes the opposite state. + if not isinstance(value, str) or not value or len(value) > maximum: + # What: raise ValueError for the caller; why: ActivityRecord.from_public stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("activity string field is invalid") + # What: compute numeric from get and item and timestamp and duration s; why: if any type value not in later reads numeric, so from_public must retain the computed value under that name. + numeric = (item.get("timestamp"), item.get("durationS")) + # What: gate on any and value and numeric and type and int before value error; why: from_public admits value error only for this predicate and excludes the opposite state. + if any(type(value) not in (int, float) or not math.isfinite(value) for value in numeric): + # What: raise ValueError for the caller; why: ActivityRecord.from_public stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("activity timing field is invalid") + # What: compute ttft from get and item and ttft s; why: if ttft is not and later reads ttft, so from_public must retain the computed value under that name. + ttft = item.get("ttftS") + # What: gate on ttft and type and int and float and isfinite before value error; why: from_public admits value error only for this predicate and excludes the opposite state. + if ttft is not None and ( + # What: call type with ttft; why: from_public consumes the type return value while evaluating type(ttft) not in (int, float) or not math.isfinite(ttft) or ttft < 0. + type(ttft) not in (int, float) or not math.isfinite(ttft) or ttft < 0 + # What: complete the enclosing predicate with ttft is not and type ttft not in int; why: ActivityRecord.from_public groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise ValueError for the caller; why: ActivityRecord.from_public stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("activity TTFT field is invalid") + # What: compute session id from get and item and session id; why: if session id is not and later reads session id, so from_public must retain the computed value under that name. + session_id = item.get("sessionId") + # What: gate on session id and any and isinstance and str and len before value error; why: from_public admits value error only for this predicate and excludes the opposite state. + if session_id is not None and ( + # What: call isinstance with session id and str; why: from_public invokes isinstance while performing or len session id; the call advances that operation through its result or side effect. + not isinstance(session_id, str) + # What: call len with session id; why: from_public invokes len while performing or any char not in abcdef; the call advances that operation through its result or side effect. + or len(session_id) != 16 + # What: call any with char and session id and abcdef; why: from_public consumes the any return value while evaluating or any(char not in "0123456789abcdef" for char in session_id). + or any(char not in "0123456789abcdef" for char in session_id) + # What: complete the enclosing predicate with session id is not and not isinstance session id str or; why: ActivityRecord.from_public groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise ValueError for the caller; why: ActivityRecord.from_public stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("activity session field is invalid") + # What: gate on item before value error; why: from_public admits value error only for this predicate and excludes the opposite state. + if ( + # What: apply the item id or item response bytes portion of the enclosing predicate; why: this clause remains in from_public\'s enclosing expression so its grouping and evaluation order stay intact. + item["id"] < 1 or item["responseBytes"] < 0 + # What: apply the or not item status portion of the enclosing predicate; why: this clause remains in from_public\'s enclosing expression so its grouping and evaluation order stay intact. + or not 100 <= item["status"] <= 599 + # What: apply the or item timestamp or item duration s portion of the enclosing predicate; why: this clause remains in from_public\'s enclosing expression so its grouping and evaluation order stay intact. + or item["timestamp"] < 0 or item["durationS"] < 0 + # What: complete the enclosing predicate with if item id 1 or item response bytes 0 or; why: ActivityRecord.from_public groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise ValueError for the caller; why: ActivityRecord.from_public stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("activity row is out of range") + # What: return session id and int and float and str from from_public; why: from_public exposes session id and int and float and str so its caller can continue with the function\'s computed outcome. + return cls( + # What: supply id to int; why: from_public binds this int and item and id value to int's id input. + id=int(item["id"]), + # What: supply timestamp to float; why: from_public binds this float and item and timestamp value to float's timestamp input. + timestamp=float(item["timestamp"]), + # What: supply model to str; why: from_public binds this str and item and model value to str's model input. + model=str(item["model"]), + # What: supply route to str; why: from_public binds this str and item and route value to str's route input. + route=str(item["route"]), + # What: supply method to str; why: from_public binds this str and item and method value to str's method input. + method=str(item["method"]), + # What: supply status to int; why: from_public binds this int and item and status value to int's status input. + status=int(item["status"]), + # What: supply duration s to float; why: from_public binds this float and item and duration s value to float's duration s input. + duration_s=float(item["durationS"]), + # What: supply ttft s to float; why: from_public binds this float and get and item and ttft s and ttft s value to float's ttft s input. + ttft_s=float(item["ttftS"]) if item.get("ttftS") is not None else None, + # What: supply response bytes to int; why: from_public binds this int and item and response bytes value to int's response bytes input. + response_bytes=int(item["responseBytes"]), + # What: supply cancelled to bool; why: from_public binds this bool and item and cancelled value to bool's cancelled input. + cancelled=bool(item["cancelled"]), + # What: supply session id to cls; why: from_public binds this session id value to cls's session id input. + session_id=session_id, + # What: supply has capture to cls; why: from_public binds this false value to cls's has capture input. + has_capture=False, + # What: complete the cls call with id and timestamp and model and route and method; why: ActivityRecord.from_public groups the supplied clauses as one cls call before its value is consumed. + ) + + +# What: define ActivityStore as the owner of __init__ and reconfigure and capture_item_limit and record and list; why: daemon callers use this class boundary so those methods share one activity store state invariant. +class ActivityStore: + """Thread-safe bounded rows plus a byte-budgeted capture LRU.""" +# What: document thread safe bounded rows plus a byte budgeted in the ActivityStore docstring; why: introspection and maintainers read this exact docstring fragment to understand activity store behavior without executing it. + + # What: define __init__ around max entries and capture budget bytes and persistence path and session headers; why: its direct callers call __init__ for init and rely on this exact input and result contract. + def __init__( + # What: declare the self input for __init__; why: __init__ consumes self during self lock threading lock, so callers must bind it with the other signature inputs. + self, max_entries: int, capture_budget_bytes: int, persistence_path: str | None = None, + # What: declare the session headers input for __init__; why: __init__ consumes session headers during self session headers session headers, so callers must bind it with the other signature inputs. + session_headers: tuple[str, ...] = (), + # What: complete the enclosing predicate with group delimiter; why: ActivityStore.__init__ groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) -> None: + # What: compute lock from lock and threading; why: the enclosing return or state update later reads lock, so __init__ must retain the computed value under that name. + self._lock = threading.Lock() + # What: compute next id from 1; why: the enclosing return or state update later reads next id, so __init__ must retain the computed value under that name. + self._next_id = 1 + # What: compute records from deque; why: the enclosing return or state update later reads records, so __init__ must retain the computed value under that name. + self._records: deque[ActivityRecord] = deque() + # What: compute captures from ordered dict; why: the enclosing return or state update later reads captures, so __init__ must retain the computed value under that name. + self._captures: OrderedDict[int, tuple[int, dict]] = OrderedDict() + # What: compute capture bytes from 0; why: the enclosing return or state update later reads capture bytes, so __init__ must retain the computed value under that name. + self._capture_bytes = 0 + # What: compute max entries from max entries; why: the enclosing return or state update later reads max entries, so __init__ must retain the computed value under that name. + self._max_entries = max_entries + # What: compute capture budget from capture budget bytes; why: the enclosing return or state update later reads capture budget, so __init__ must retain the computed value under that name. + self._capture_budget = capture_budget_bytes + # What: compute persistence path from persistence path; why: the enclosing return or state update later reads persistence path, so __init__ must retain the computed value under that name. + self._persistence_path = persistence_path + # What: compute persisted rows from 0; why: the enclosing return or state update later reads persisted rows, so __init__ must retain the computed value under that name. + self._persisted_rows = 0 + # What: compute persistence error from the named fixture input; why: the enclosing return or state update later reads persistence error, so __init__ must retain the computed value under that name. + self._persistence_error: str | None = None + # What: compute rewrite required from false; why: the enclosing return or state update later reads rewrite required, so __init__ must retain the computed value under that name. + self._rewrite_required = False + # What: compute session headers from session headers; why: the enclosing return or state update later reads session headers, so __init__ must retain the computed value under that name. + self._session_headers = session_headers + # What: call self._load with the declared inputs; why: __init__ invokes self._load while performing the enclosing return; the call advances that operation through its result or side effect. + self._load() + + # What: define reconfigure around max entries and capture budget bytes and session headers; why: its direct callers call reconfigure for reconfigure and rely on this exact input and result contract. + def reconfigure( + # What: declare the self input for reconfigure; why: reconfigure consumes self during with self lock, so callers must bind it with the other signature inputs. + self, max_entries: int, capture_budget_bytes: int, + # What: declare the session headers input for reconfigure; why: reconfigure consumes session headers during if session headers is not, so callers must bind it with the other signature inputs. + session_headers: tuple[str, ...] | None = None, + # What: complete the enclosing predicate with group delimiter; why: ActivityStore.reconfigure groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) -> None: + # What: enter the lock managed context before self max entries max entries; why: reconfigure releases this resource or lock after self max entries max entries on both success and failure paths. + with self._lock: + # What: compute max entries from max entries; why: the enclosing return or state update later reads max entries, so reconfigure must retain the computed value under that name. + self._max_entries = max_entries + # What: compute capture budget from capture budget bytes; why: the enclosing return or state update later reads capture budget, so reconfigure must retain the computed value under that name. + self._capture_budget = capture_budget_bytes + # What: gate on session headers before session headers and session headers; why: reconfigure admits session headers and session headers only for this predicate and excludes the opposite state. + if session_headers is not None: + # What: compute session headers from session headers; why: the enclosing return or state update later reads session headers, so reconfigure must retain the computed value under that name. + self._session_headers = session_headers + # What: iterate across max entries and len and records to perform removed and popleft and records; why: reconfigure repeats the body only while or for the loop header admits an iteration. + while len(self._records) > max_entries: + # What: compute removed from popleft and records; why: self drop capture locked removed id later reads removed, so reconfigure must retain the computed value under that name. + removed = self._records.popleft() + # What: call self._drop_capture_locked with id and removed; why: reconfigure invokes self._drop_capture_locked while performing while self capture bytes capture budget bytes and self captures; the call advances that operation through its result or side effect. + self._drop_capture_locked(removed.id) + # What: iterate across captures and capture bytes and capture budget bytes to perform value and popitem and size and captures; why: reconfigure repeats the body only while or for the loop header admits an iteration. + while self._capture_bytes > capture_budget_bytes and self._captures: + # What: compute and size and from popitem and captures and false; why: the enclosing return or state update later reads and size and, so reconfigure must retain the computed value under that name. + _, (size, _) = self._captures.popitem(last=False) + # What: compute capture bytes from size; why: the enclosing return or state update later reads capture bytes, so reconfigure must retain the computed value under that name. + self._capture_bytes -= size + # What: gate on persistence path before compact locked; why: reconfigure admits compact locked only for this predicate and excludes the opposite state. + if self._persistence_path is not None: + # What: call self._compact_locked with the declared inputs; why: reconfigure invokes self._compact_locked while performing the enclosing return; the call advances that operation through its result or side effect. + self._compact_locked() + + # What: expose capture_item_limit as a read-only computed property; why: callers read capture_item_limit through attribute access while its getter retains control of the derived value. + @property + # What: define capture_item_limit around the current object state; why: the registered API client call capture_item_limit for capture item limit and rely on this exact input and result contract. + def capture_item_limit(self) -> int: + # What: enter the lock managed context before return min self capture budget; why: capture_item_limit releases this resource or lock after return min self capture budget on both success and failure paths. + with self._lock: + # What: return min and capture budget and 1024 and 1024 from capture_item_limit; why: capture_item_limit exposes min and capture budget and 1024 and 1024 so its caller can continue with the function\'s computed outcome. + return min(self._capture_budget, 1024 * 1024) + + # What: define record around model and route and method and status and started and ttft s and response bytes and cancelled and request headers and request body and response headers and response body; why: its direct callers call record for record and rely on this exact input and result contract. + def record( + # What: declare the self input for record; why: record consumes self during with self lock, so callers must bind it with the other signature inputs. + self, *, model: str, route: str, method: str, status: int, + # What: declare the started input for record; why: record consumes started during duration max time monotonic started, so callers must bind it with the other signature inputs. + started: float, ttft_s: float | None, response_bytes: int, cancelled: bool, + # What: declare the request headers input for record; why: record consumes request headers during lowered headers key lower value for key value, so callers must bind it with the other signature inputs. + request_headers: Mapping[str, str], request_body: bytes, + # What: declare the response headers input for record; why: record consumes response headers during response headers headers response headers, so callers must bind it with the other signature inputs. + response_headers: Mapping[str, str], response_body: bytes | None, + # What: complete the enclosing predicate with dict; why: ActivityStore.record groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) -> dict: + # What: compute ended from time; why: id row id timestamp ended model model later reads ended, so record must retain the computed value under that name. + ended = time.time() + # What: compute duration from max and started and monotonic and time and 0 0; why: status status duration s round duration later reads duration, so record must retain the computed value under that name. + duration = max(0.0, time.monotonic() - started) + # What: compute lowered headers from value and lower and key and items; why: hashlib sha256 str lowered headers header encode utf 8 later reads lowered headers, so record must retain the computed value under that name. + lowered_headers = {key.lower(): value for key, value in request_headers.items()} + # What: compute session id from next and header and session headers and hexdigest; why: session id session id has capture has capture later reads session id, so record must retain the computed value under that name. + session_id = next( + # What: complete the next call with header; why: ActivityStore.record groups the supplied clauses as one next call before its value is consumed. + ( + # What: call operation.hexdigest with the declared inputs; why: record invokes operation.hexdigest while performing for header in self session headers if lowered headers get; the call advances that operation through its result or side effect. + hashlib.sha256(str(lowered_headers[header]).encode("utf-8")).hexdigest()[:16] + # What: call lowered_headers.get with header; why: record consumes the lowered_headers.get return value while evaluating for header in self._session_headers if lowered_headers.get(header). + for header in self._session_headers if lowered_headers.get(header) + # What: complete the next call with header; why: ActivityStore.record groups the supplied clauses as one next call before its value is consumed. + ), + # What: apply the grouped expression portion of session id; why: record uses this clause to evaluate session id as one grouped value. + None, + # What: complete the next call with header; why: ActivityStore.record groups the supplied clauses as one next call before its value is consumed. + ) + # What: enter the lock managed context before row id self next id; why: record releases this resource or lock after row id self next id on both success and failure paths. + with self._lock: + # What: compute row id from next id; why: id row id later reads row id, so record must retain the computed value under that name. + row_id = self._next_id + # What: compute next id from 1; why: the enclosing return or state update later reads next id, so record must retain the computed value under that name. + self._next_id += 1 + # What: compute has capture from false; why: has capture later reads has capture, so record must retain the computed value under that name. + has_capture = False + # What: gate on capture budget and cancelled and response body and len and request body before capture and row id and route and method and headers; why: record admits capture and row id and route and method and headers only for this predicate and excludes the opposite state. + if ( + # What: apply the self capture budget portion of the enclosing predicate; why: this clause remains in record\'s enclosing expression so its grouping and evaluation order stay intact. + self._capture_budget > 0 + # What: apply the and not cancelled portion of the enclosing predicate; why: this clause remains in record\'s enclosing expression so its grouping and evaluation order stay intact. + and not cancelled + # What: apply the and response body is not portion of the enclosing predicate; why: this clause remains in record\'s enclosing expression so its grouping and evaluation order stay intact. + and response_body is not None + # What: call len with request body; why: record consumes the len return value while evaluating and len(request_body) + len(response_body) <= self._capture_budget. + and len(request_body) + len(response_body) <= self._capture_budget + # What: complete the enclosing predicate with if self capture budget 0 and not cancelled and response body is; why: ActivityStore.record groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: compute capture from row id and route and method and headers; why: size len json dumps capture separators encode later reads capture, so record must retain the computed value under that name. + capture = { + # What: map the id field as row id; why: ActivityStore.record carries id through capture into size len json dumps capture separators encode utf 8. + "id": row_id, + # What: map the route field as route; why: ActivityStore.record carries route through capture into size len json dumps capture separators encode utf 8. + "route": route, + # What: map the method field as method; why: ActivityStore.record carries method through capture into size len json dumps capture separators encode utf 8. + "method": method, + # What: map the request headers field as headers and request headers; why: ActivityStore.record carries request headers through capture into size len json dumps capture separators encode utf 8. + "requestHeaders": _headers(request_headers), + # What: map the request body base64 field as decode and b64encode and request body and base64 and ascii; why: ActivityStore.record carries request body base64 through capture into size len json dumps capture separators encode utf 8. + "requestBodyBase64": base64.b64encode(request_body).decode("ascii"), + # What: map the response headers field as headers and response headers; why: ActivityStore.record carries response headers through capture into size len json dumps capture separators encode utf 8. + "responseHeaders": _headers(response_headers), + # What: map the response body base64 field as decode and b64encode and response body and base64 and ascii; why: ActivityStore.record carries response body base64 through capture into size len json dumps capture separators encode utf 8. + "responseBodyBase64": base64.b64encode(response_body).decode("ascii"), + # What: complete the capture mapping with id and route and method and request headers and request body base64; why: ActivityStore.record groups the supplied clauses as one capture mapping before its value is consumed. + } + # What: compute size from len and encode and dumps and capture; why: if size self capture budget later reads size, so record must retain the computed value under that name. + size = len(json.dumps(capture, separators=(",", ":")).encode("utf-8")) + # What: gate on size and capture budget before captures and capture bytes and old size and capture budget and value; why: record admits captures and capture bytes and old size and capture budget and value only for this predicate and excludes the opposite state. + if size <= self._capture_budget: + # What: iterate across captures and capture budget and capture bytes and size to perform value and popitem and old size and captures; why: record repeats the body only while or for the loop header admits an iteration. + while self._capture_bytes + size > self._capture_budget and self._captures: + # What: compute and old size and from popitem and captures and false; why: the enclosing return or state update later reads and old size and, so record must retain the computed value under that name. + _, (old_size, _) = self._captures.popitem(last=False) + # What: compute capture bytes from old size; why: self capture bytes size later reads capture bytes, so record must retain the computed value under that name. + self._capture_bytes -= old_size + # What: compute captures entry from size and capture; why: the enclosing return or state update later reads captures entry, so record must retain the computed value under that name. + self._captures[row_id] = (size, capture) + # What: compute capture bytes from size; why: the enclosing return or state update later reads capture bytes, so record must retain the computed value under that name. + self._capture_bytes += size + # What: compute has capture from true; why: session id session id has capture has capture later reads has capture, so record must retain the computed value under that name. + has_capture = True + # What: compute record from activity record and row id and ended and model; why: self records append record later reads record, so record must retain the computed value under that name. + record = ActivityRecord( + # What: supply id to ActivityRecord; why: record binds this row id value to ActivityRecord's id input. + id=row_id, timestamp=ended, model=model, route=route, method=method, + # What: supply status to round; why: record binds this status value to round's status input. + status=status, duration_s=round(duration, 6), + # What: supply ttft s to round; why: record binds this ttft s and round and 6 value to round's ttft s input. + ttft_s=round(ttft_s, 6) if ttft_s is not None else None, + # What: supply response bytes to ActivityRecord; why: record binds this response bytes value to ActivityRecord's response bytes input. + response_bytes=response_bytes, cancelled=cancelled, + # What: supply session id to ActivityRecord; why: record binds this session id value to ActivityRecord's session id input. + session_id=session_id, has_capture=has_capture, + # What: complete the ActivityRecord call with id and timestamp and model and route and method; why: ActivityStore.record groups the supplied clauses as one ActivityRecord call before its value is consumed. + ) + # What: call self._records.append with record; why: record invokes self._records.append while performing while len self records self max entries; the call advances that operation through its result or side effect. + self._records.append(record) + # What: iterate across max entries and len and records to perform removed and popleft and records; why: record repeats the body only while or for the loop header admits an iteration. + while len(self._records) > self._max_entries: + # What: compute removed from popleft and records; why: self drop capture locked removed id later reads removed, so record must retain the computed value under that name. + removed = self._records.popleft() + # What: call self._drop_capture_locked with id and removed; why: record invokes self._drop_capture_locked while performing self append locked record; the call advances that operation through its result or side effect. + self._drop_capture_locked(removed.id) + # What: call self._append_locked with record; why: record invokes self._append_locked while performing return record public; the call advances that operation through its result or side effect. + self._append_locked(record) + # What: return public and record from record; why: record exposes public and record so its caller can continue with the function\'s computed outcome. + return record.public() + + # What: define list around limit and before id and model; why: its direct callers call list for list and rely on this exact input and result contract. + def list(self, *, limit: int = 100, before_id: int | None = None, model: str | None = None) -> dict: + # What: enter the lock managed context before rows row for row in reversed; why: list releases this resource or lock after rows row for row in reversed on both success and failure paths. + with self._lock: + # What: compute rows from row and reversed and records and before id; why: for row in rows limit later reads rows, so list must retain the computed value under that name. + rows = [row for row in reversed(self._records) + # What: apply the if before id is or row id before id portion of rows; why: list uses this clause to evaluate rows as one grouped value. + if (before_id is None or row.id < before_id) and (model is None or row.model == model)] + # What: initialize data as an empty runtime accumulator; why: ActivityStore.list appends or maps entries into it during data append item before consuming the aggregate. + data = [] + # What: iterate across rows and limit to perform item and public and row; why: list repeats the body only while or for the loop header admits an iteration. + for row in rows[:limit]: + # What: compute item from public and row; why: item has capture row id in self captures later reads item, so list must retain the computed value under that name. + item = row.public() + # What: compute item entry from id and captures and row; why: data append item later reads item entry, so list must retain the computed value under that name. + item["hasCapture"] = row.id in self._captures + # What: call data.append with item; why: list invokes data.append while performing return; the call advances that operation through its result or side effect. + data.append(item) + # What: return data and len and persistence locked and limit from list; why: list exposes data and len and persistence locked and limit so its caller can continue with the function\'s computed outcome. + return { + # What: map the data field as data; why: ActivityStore.list carries data into "data": data. + "data": data, + # What: map the count field as len and data; why: ActivityStore.list carries count into "count": len(data). + "count": len(data), + # What: map the next before id field as limit and len and rows and data and id; why: ActivityStore.list carries next before id into "nextBeforeId": data[-1]["id"] if len(rows) > limit else None. + "nextBeforeId": data[-1]["id"] if len(rows) > limit else None, + # What: map the persistence field as persistence locked; why: ActivityStore.list carries persistence into "persistence": self._persistence_locked(). + "persistence": self._persistence_locked(), + # What: complete the enclosing predicate mapping with data and count and next before id and persistence; why: ActivityStore.list groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + # What: define stats around model; why: its direct callers call stats for stats and rely on this exact input and result contract. + def stats(self, *, model: str | None = None) -> dict: + # What: enter the lock managed context before rows row for row in self records; why: stats releases this resource or lock after rows row for row in self records on both success and failure paths. + with self._lock: + # What: compute rows from row and records and model; why: count len rows later reads rows, so stats must retain the computed value under that name. + rows = [row for row in self._records if model is None or row.model == model] + # What: compute count from len and rows; why: count count later reads count, so stats must retain the computed value under that name. + count = len(rows) + # What: return count and sum and persistence locked and cancelled from stats; why: stats exposes count and sum and persistence locked and cancelled so its caller can continue with the function\'s computed outcome. + return { + # What: map the count field as count; why: ActivityStore.stats carries count into "count": count. + "count": count, + # What: map the cancelled field as sum and cancelled and row and rows; why: ActivityStore.stats carries cancelled into "cancelled": sum(row.cancelled for row in rows). + "cancelled": sum(row.cancelled for row in rows), + # What: map the errors field as sum and status and row and rows and 400; why: ActivityStore.stats carries errors into "errors": sum(row.status >= 400 for row in rows). + "errors": sum(row.status >= 400 for row in rows), + # What: map the response bytes field as sum and response bytes and row and rows; why: ActivityStore.stats carries response bytes into "responseBytes": sum(row.response_bytes for row in rows). + "responseBytes": sum(row.response_bytes for row in rows), + # What: map the average duration s field as count and round and sum and duration s; why: ActivityStore.stats carries average duration s into "averageDurationS": round(sum(row.duration_s for row in rows) / count, 6. + "averageDurationS": round(sum(row.duration_s for row in rows) / count, 6) if count else None, + # What: map the persistence field as persistence locked; why: ActivityStore.stats carries persistence into "persistence": self._persistence_locked(). + "persistence": self._persistence_locked(), + # What: complete the enclosing predicate mapping with count and cancelled and errors and response bytes and average duration s; why: ActivityStore.stats groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + # What: define capture around row id; why: its direct callers call capture for capture and rely on this exact input and result contract. + def capture(self, row_id: int) -> dict | None: + # What: enter the lock managed context before item self captures get row id; why: capture releases this resource or lock after item self captures get row id on both success and failure paths. + with self._lock: + # What: compute item from get and row id and captures; why: if item is later reads item, so capture must retain the computed value under that name. + item = self._captures.get(row_id) + # What: gate on item before the computed value; why: capture admits the computed value only for this predicate and excludes the opposite state. + if item is None: + # What: return no value from capture; why: capture returns no value to callers that depend on its completed result. + return None + # What: call self._captures.move_to_end with row id; why: capture invokes self._captures.move_to_end while performing return dict item; the call advances that operation through its result or side effect. + self._captures.move_to_end(row_id) + # What: return dict and item and 1 from capture; why: capture exposes dict and item and 1 so its caller can continue with the function\'s computed outcome. + return dict(item[1]) + + # What: define _drop_capture_locked around row id; why: its direct callers call _drop_capture_locked for drop capture locked and rely on this exact input and result contract. + def _drop_capture_locked(self, row_id: int) -> None: + # What: compute item from pop and row id and captures; why: if item is not later reads item, so _drop_capture_locked must retain the computed value under that name. + item = self._captures.pop(row_id, None) + # What: gate on item before capture bytes and item; why: _drop_capture_locked admits capture bytes and item only for this predicate and excludes the opposite state. + if item is not None: + # What: compute capture bytes from item and 0; why: the enclosing return or state update later reads capture bytes, so _drop_capture_locked must retain the computed value under that name. + self._capture_bytes -= item[0] + + # What: define _persistence_locked around the current object state; why: its direct callers call _persistence_locked for persistence locked and rely on this exact input and result contract. + def _persistence_locked(self) -> dict: + # What: return persistence error and persistence path and enabled and healthy and error from _persistence_locked; why: _persistence_locked exposes persistence error and persistence path and enabled and healthy and error so its caller can continue with the function\'s computed outcome. + return { + # What: map the enabled field as persistence path; why: ActivityStore._persistence_locked carries enabled into "enabled": self._persistence_path is not None. + "enabled": self._persistence_path is not None, + # What: map the healthy field as persistence path and persistence error; why: ActivityStore._persistence_locked carries healthy into "healthy": self._persistence_path is None or self._persistence_error is. + "healthy": self._persistence_path is None or self._persistence_error is None, + # What: map the error field as persistence error; why: ActivityStore._persistence_locked carries error into "error": self._persistence_error. + "error": self._persistence_error, + # What: complete the enclosing predicate mapping with enabled and healthy and error; why: ActivityStore._persistence_locked groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + # What: define _load around the current object state; why: its direct callers call _load for load and rely on this exact input and result contract. + def _load(self) -> None: + # What: gate on persistence path before the computed value; why: _load admits the computed value only for this predicate and excludes the opposite state. + if self._persistence_path is None: + # What: return no value from _load; why: _load returns no value to callers that depend on its completed result. + return + # What: compute invalid from false; why: invalid later reads invalid, so _load must retain the computed value under that name. + invalid = False + # What: compute last id from 0; why: if row id last id later reads last id, so _load must retain the computed value under that name. + last_id = 0 + # What: establish the handler boundary for the protected operation; why: ActivityStore._load routes failures to file not found error and oserror while preserving cleanup and success flow. + try: + # What: enter the open managed context before while line source readline max persisted row chars; why: _load releases this resource or lock after while line source readline max persisted row chars on both success and failure paths. + with open(self._persistence_path, encoding="utf-8") as source: + # What: iterate across line and readline and source and max persisted row chars to perform max persisted row chars and invalid and len and line and readline; why: _load repeats the body only while or for the loop header admits an iteration. + while line := source.readline(_MAX_PERSISTED_ROW_CHARS + 1): + # What: gate on max persisted row chars and len and line before invalid; why: _load admits invalid only for this predicate and excludes the opposite state. + if len(line) > _MAX_PERSISTED_ROW_CHARS: + # What: compute invalid from true; why: invalid later reads invalid, so _load must retain the computed value under that name. + invalid = True + # What: iterate across line and endswith to perform line and readline and source and max persisted row chars; why: _load repeats the body only while or for the loop header admits an iteration. + while line and not line.endswith("\n"): + # What: compute line from readline and source and max persisted row chars and 1; why: row activity record from public json loads line later reads line, so _load must retain the computed value under that name. + line = source.readline(_MAX_PERSISTED_ROW_CHARS + 1) + # What: apply the continue portion of the enclosing predicate; why: this clause remains in _load\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: establish the handler boundary for the protected operation; why: ActivityStore._load routes failures to key error and type error and value error and jsondecode error and json while preserving cleanup and success flow. + try: + # What: compute row from from public and activity record and loads and line; why: if row id last id later reads row, so _load must retain the computed value under that name. + row = ActivityRecord.from_public(json.loads(line)) + # What: gate on id and last id and row before value error; why: _load admits value error only for this predicate and excludes the opposite state. + if row.id <= last_id: + # What: raise ValueError for the caller; why: ActivityStore._load stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("activity IDs must increase") + # What: handle key error and type error and value error and jsondecode error and json by invalid true; why: ActivityStore._load converts that failure into this concrete recovery, response, or cleanup behavior. + except (KeyError, TypeError, ValueError, json.JSONDecodeError): + # What: compute invalid from true; why: if invalid or self persisted rows self max entries later reads invalid, so _load must retain the computed value under that name. + invalid = True + # What: apply the continue portion of the enclosing predicate; why: this clause remains in _load\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: call self._records.append with row; why: _load invokes self._records.append while performing last id row id; the call advances that operation through its result or side effect. + self._records.append(row) + # What: compute last id from id and row; why: the enclosing return or state update later reads last id, so _load must retain the computed value under that name. + last_id = row.id + # What: compute persisted rows from 1; why: if invalid or self persisted rows self max entries later reads persisted rows, so _load must retain the computed value under that name. + self._persisted_rows += 1 + # What: compute next id from max and next id and id and row and 1; why: the enclosing return or state update later reads next id, so _load must retain the computed value under that name. + self._next_id = max(self._next_id, row.id + 1) + # What: iterate across max entries and len and records to perform popleft and records; why: _load repeats the body only while or for the loop header admits an iteration. + while len(self._records) > self._max_entries: + # What: call self._records.popleft with the declared inputs; why: _load invokes self._records.popleft while performing except file not found error; the call advances that operation through its result or side effect. + self._records.popleft() + # What: handle file not found error by return; why: ActivityStore._load converts that failure into this concrete recovery, response, or cleanup behavior. + except FileNotFoundError: + # What: return no value from _load; why: _load returns no value to callers that depend on its completed result. + return + # What: handle oserror by self persistence error load failed; why: ActivityStore._load converts that failure into this concrete recovery, response, or cleanup behavior. + except OSError: + # What: compute persistence error from load failed; why: the enclosing return or state update later reads persistence error, so _load must retain the computed value under that name. + self._persistence_error = "load_failed" + # What: compute rewrite required from true; why: the enclosing return or state update later reads rewrite required, so _load must retain the computed value under that name. + self._rewrite_required = True + # What: return no value from _load; why: _load returns no value to callers that depend on its completed result. + return + # What: gate on invalid and persisted rows and max entries before compact locked; why: _load admits compact locked only for this predicate and excludes the opposite state. + if invalid or self._persisted_rows > self._max_entries: + # What: call self._compact_locked with the declared inputs; why: _load invokes self._compact_locked while performing the enclosing return; the call advances that operation through its result or side effect. + self._compact_locked() + + # What: define _append_locked around record; why: its direct callers call _append_locked for append locked and rely on this exact input and result contract. + def _append_locked(self, record: ActivityRecord) -> None: + # What: gate on persistence path before the computed value; why: _append_locked admits the computed value only for this predicate and excludes the opposite state. + if self._persistence_path is None: + # What: return no value from _append_locked; why: _append_locked returns no value to callers that depend on its completed result. + return + # What: gate on rewrite required before compact locked; why: _append_locked admits compact locked only for this predicate and excludes the opposite state. + if self._rewrite_required: + # What: call self._compact_locked with the declared inputs; why: _append_locked invokes self._compact_locked while performing return; the call advances that operation through its result or side effect. + self._compact_locked() + # What: return no value from _append_locked; why: _append_locked returns no value to callers that depend on its completed result. + return + # What: establish the handler boundary for the protected operation; why: ActivityStore._append_locked routes failures to oserror while preserving cleanup and success flow. + try: + # What: enter the open managed context before target write json dumps record public separators n; why: _append_locked releases this resource or lock after target write json dumps record public separators n on both success and failure paths. + with open(self._persistence_path, "a", encoding="utf-8", newline="\n") as target: + # What: preserve the exact target write json dumps record public separators n literal fragment; why: _append_locked passes this fragment verbatim through target.write(json.dumps(record.public(), separators=(",", ":")) + "\n"), because changing it would alter a protocol payload, serialized fixture, or p. + target.write(json.dumps(record.public(), separators=(",", ":")) + "\n") + # What: call target.flush with the declared inputs; why: _append_locked invokes target.flush while performing os fsync target fileno; the call advances that operation through its result or side effect. + target.flush() + # What: call os.fsync with fileno and target; why: _append_locked invokes os.fsync while performing self persisted rows; the call advances that operation through its result or side effect. + os.fsync(target.fileno()) + # What: compute persisted rows from 1; why: if self persisted rows max self max entries later reads persisted rows, so _append_locked must retain the computed value under that name. + self._persisted_rows += 1 + # What: compute persistence error from the named fixture input; why: self persistence error write failed later reads persistence error, so _append_locked must retain the computed value under that name. + self._persistence_error = None + # What: gate on persisted rows and max and max entries before compact locked; why: _append_locked admits compact locked only for this predicate and excludes the opposite state. + if self._persisted_rows > max(2, self._max_entries * 2): + # What: call self._compact_locked with the declared inputs; why: _append_locked invokes self._compact_locked while performing except oserror; the call advances that operation through its result or side effect. + self._compact_locked() + # What: handle oserror by self persistence error write failed; why: ActivityStore._append_locked converts that failure into this concrete recovery, response, or cleanup behavior. + except OSError: + # What: compute persistence error from write failed; why: the enclosing return or state update later reads persistence error, so _append_locked must retain the computed value under that name. + self._persistence_error = "write_failed" + + # What: define _compact_locked around the current object state; why: its direct callers call _compact_locked for compact locked and rely on this exact input and result contract. + def _compact_locked(self) -> None: + # What: gate on persistence path before the computed value; why: _compact_locked admits the computed value only for this predicate and excludes the opposite state. + if self._persistence_path is None: + # What: return no value from _compact_locked; why: _compact_locked returns no value to callers that depend on its completed result. + return + # What: compute temporary from persistence path and tmp; why: with open temporary w encoding utf 8 later reads temporary, so _compact_locked must retain the computed value under that name. + temporary = self._persistence_path + ".tmp" + # What: establish the handler boundary for the protected operation; why: ActivityStore._compact_locked routes failures to oserror while preserving cleanup and success flow. + try: + # What: enter the open managed context before for record in self records; why: _compact_locked releases this resource or lock after for record in self records on both success and failure paths. + with open(temporary, "w", encoding="utf-8", newline="\n") as target: + # What: iterate across records to perform write and target and dumps and json and public; why: _compact_locked repeats the body only while or for the loop header admits an iteration. + for record in self._records: + # What: preserve the exact target write json dumps record public separators n literal fragment; why: _compact_locked passes this fragment verbatim through target.write(json.dumps(record.public(), separators=(",", ":")) + "\n"), because changing it would alter a protocol payload, serialized fixture. + target.write(json.dumps(record.public(), separators=(",", ":")) + "\n") + # What: call target.flush with the declared inputs; why: _compact_locked invokes target.flush while performing os fsync target fileno; the call advances that operation through its result or side effect. + target.flush() + # What: call os.fsync with fileno and target; why: _compact_locked invokes os.fsync while performing os replace temporary self persistence path; the call advances that operation through its result or side effect. + os.fsync(target.fileno()) + # What: call os.replace with temporary and persistence path; why: _compact_locked invokes os.replace while performing self persisted rows len self records; the call advances that operation through its result or side effect. + os.replace(temporary, self._persistence_path) + # What: compute persisted rows from len and records; why: the enclosing return or state update later reads persisted rows, so _compact_locked must retain the computed value under that name. + self._persisted_rows = len(self._records) + # What: compute persistence error from the named fixture input; why: self persistence error compact failed later reads persistence error, so _compact_locked must retain the computed value under that name. + self._persistence_error = None + # What: compute rewrite required from false; why: the enclosing return or state update later reads rewrite required, so _compact_locked must retain the computed value under that name. + self._rewrite_required = False + # What: handle oserror by self persistence error compact failed; why: ActivityStore._compact_locked converts that failure into this concrete recovery, response, or cleanup behavior. + except OSError: + # What: compute persistence error from compact failed; why: the enclosing return or state update later reads persistence error, so _compact_locked must retain the computed value under that name. + self._persistence_error = "compact_failed" + # What: establish the handler boundary for the protected operation; why: ActivityStore._compact_locked routes failures to oserror while preserving cleanup and success flow. + try: + # What: call os.unlink with temporary; why: _compact_locked invokes os.unlink while performing except oserror; the call advances that operation through its result or side effect. + os.unlink(temporary) + # What: handle oserror by pass; why: ActivityStore._compact_locked converts that failure into this concrete recovery, response, or cleanup behavior. + except OSError: + # What: ignore the anticipated exception handled by this branch; why: _compact_locked continues its retry or cleanup path instead of re-raising that transient failure. + pass diff --git a/python/freetoken/daemon/app.py b/python/freetoken/daemon/app.py index d7a0a53c60..e23233298e 100644 --- a/python/freetoken/daemon/app.py +++ b/python/freetoken/daemon/app.py @@ -9,23 +9,186 @@ from __future__ import annotations import asyncio +# What: import base64 for extract api key using base64; why: _extract_api_key uses base64 b64decode, making that imported dependency available to its named operation. +import base64 +# What: import binascii for extract api key using binascii; why: _extract_api_key uses binascii error, making that imported dependency available to its named operation. +import binascii import collections +# What: import datetime for router performance using datetime and datetime; why: router_performance uses datetime fromisoformat, making that imported dependency available to its named operation. +from datetime import datetime import functools import json import os +# What: import re for module initialization using re; why: module initialization uses re compile, making that imported dependency available to its named operation. +import re import sys +# What: import threading for build app using threading; why: build_app uses threading lock, making that imported dependency available to its named operation. +import threading +# What: import time for forward routed using time; why: forward_routed uses time monotonic, making that imported dependency available to its named operation. +import time +# What: import uuid for forward routed using uuid; why: forward_routed uses uuid uuid4, making that imported dependency available to its named operation. +import uuid from concurrent.futures import ThreadPoolExecutor from typing import Any, Callable +# What: import quote from bytes for escaped path suffix using urllib and parse and quote from bytes; why: _escaped_path_suffix uses quote from bytes, making that imported dependency available to its named operation. +from urllib.parse import quote_from_bytes -from fastapi import Depends, FastAPI, Header, HTTPException, Request -from fastapi.responses import JSONResponse, StreamingResponse +# What: import from fastapi import Depends FastAPI Header HTTPException Query Request; why: this module calls or annotates these symbols in the branch-created operations below. +from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request +# What: import from fastapi responses import HTMLResponse JSONResponse PlainTextResponse Response StreamingResponse; why: this module calls or annotates these symbols in the branch-created operations below. +from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse, Response, StreamingResponse from pydantic import BaseModel from .accounting import AccountingOutboxError, AccountingPrepareError -from .serve_manager import Conflict +# What: import activity store for build app using activity and activity store; why: build_app uses activity store, making that imported dependency available to its named operation. +from .activity import ActivityStore +# What: import from catalog import CatalogError ModelCatalog; why: this module calls or annotates these symbols in the branch-created operations below. +from .catalog import CatalogError, ModelCatalog +# What: import from inference proxy import; why: this module calls or annotates these symbols in the branch-created operations below. +from .inference_proxy import ( + # What: execute RequestModelError; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + RequestModelError, + # What: execute filter request body; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + filter_request_body, + # What: execute open upstream; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + open_upstream, + # What: execute request model; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + request_model, + # What: execute response headers; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + response_headers, +# What: complete the enclosing predicate with from inference proxy import request model error filter request body open upstream request model response headers; why: app groups the supplied clauses as one enclosing predicate expression before its value is consumed. +) +# What: import log ring for build app using logring and log ring; why: build_app uses the log ring annotation in build app, making that imported dependency available to its named operation. +from .logring import LogRing +# What: import performance monitor for build app using performance and performance monitor; why: build_app uses performance monitor, making that imported dependency available to its named operation. +from .performance import PerformanceMonitor +# What: import wait for ready for profile result using readiness and wait for ready; why: profile_result uses wait for ready, making that imported dependency available to its named operation. +from .readiness import wait_for_ready +# What: import from router import RoutingCoordinator RoutingError allocate loopback port; why: this module calls or annotates these symbols in the branch-created operations below. +from .router import RoutingCoordinator, RoutingError, allocate_loopback_port +# What: import from serve manager import Conflict SwitchLaunchError; why: this module calls or annotates these symbols in the branch-created operations below. +from .serve_manager import Conflict, SwitchLaunchError from .version import DAEMON_VERSION +# What: compute http token from compile and re and value and a za z; why: if part raw strip and http token fullmatch part later reads http token, so app must retain the computed value under that name. +_HTTP_TOKEN = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$") +# What: compute default cors headers from content type and authorization and accept and x requested with; why: return default cors headers later reads default cors headers, so app must retain the computed value under that name. +_DEFAULT_CORS_HEADERS = "Content-Type, Authorization, Accept, X-Requested-With" + + +# What: define _cors_request_headers around value; why: its direct callers call _cors_request_headers for cors request headers and rely on this exact input and result contract. +def _cors_request_headers(value: str | None) -> str: + """Echo only syntactically valid HTTP header names in a CORS preflight.""" + # What: document echo only syntactically valid http header in the _cors_request_headers docstring; why: introspection and maintainers read this exact docstring fragment to understand cors request headers behavior without executing it. + # What: gate on value before default cors headers; why: _cors_request_headers admits default cors headers only for this predicate and excludes the opposite state. + if value is None: + # What: return default cors headers from _cors_request_headers; why: _cors_request_headers exposes default cors headers so its caller can continue with the function\'s computed outcome. + return _DEFAULT_CORS_HEADERS + # What: return join and part and raw and split from _cors_request_headers; why: _cors_request_headers exposes join and part and raw and split so its caller can continue with the function\'s computed outcome. + return ", ".join( + # What: call value.split with value; why: _cors_request_headers invokes value.split while performing if part raw strip and http token fullmatch part; the call advances that operation through its result or side effect. + part for raw in value.split(",") + # What: call _HTTP_TOKEN.fullmatch with part; why: _cors_request_headers consumes the _HTTP_TOKEN.fullmatch return value while evaluating if (part := raw.strip()) and _HTTP_TOKEN.fullmatch(part). + if (part := raw.strip()) and _HTTP_TOKEN.fullmatch(part) + # What: complete the operation.join call with part; why: _cors_request_headers groups the supplied clauses as one operation.join call before its value is consumed. + ) + + +# What: define _escaped_path_suffix around raw path and decoded prefix; why: its direct callers call _escaped_path_suffix for escaped path suffix and rely on this exact input and result contract. +def _escaped_path_suffix(raw_path: bytes, decoded_prefix: str) -> str | None: + """Remove a decoded prefix while retaining the suffix's original escaping.""" + # What: document remove a decoded prefix while retaining in the _escaped_path_suffix docstring; why: introspection and maintainers read this exact docstring fragment to understand escaped path suffix behavior without executing it. + # What: compute prefix from encode and decoded prefix and utf 8; why: while raw index len raw path and prefix index later reads prefix, so _escaped_path_suffix must retain the computed value under that name. + prefix = decoded_prefix.encode("utf-8") + # What: compute raw index from 0; why: while raw index len raw path and prefix index later reads raw index, so _escaped_path_suffix must retain the computed value under that name. + raw_index = prefix_index = 0 + # What: iterate across raw index and prefix index and len and raw path and prefix to perform end and raw index; why: _escaped_path_suffix repeats the body only while or for the loop header admits an iteration. + while raw_index < len(raw_path) and prefix_index < len(prefix): + # What: compute end from raw index and 1; why: end raw index later reads end, so _escaped_path_suffix must retain the computed value under that name. + end = raw_index + 1 + # What: compute value from raw path and raw index; why: if value ord later reads value, so _escaped_path_suffix must retain the computed value under that name. + value = raw_path[raw_index] + # What: gate on value and ord before raw index and len and raw path; why: _escaped_path_suffix admits raw index and len and raw path only for this predicate and excludes the opposite state. + if value == ord("%"): + # What: gate on raw index and len and raw path before the computed value; why: _escaped_path_suffix admits the computed value only for this predicate and excludes the opposite state. + if raw_index + 3 > len(raw_path): + # What: reject the malformed or mismatched escaped path; why: the upstream proxy returns no suffix so its caller emits HTTP 400 instead of forwarding ambiguous path bytes. + return None + # What: establish the handler boundary for the protected operation; why: _escaped_path_suffix routes failures to value error while preserving cleanup and success flow. + try: + # What: compute value from int and raw path and raw index and 16 and 1; why: if value prefix prefix index later reads value, so _escaped_path_suffix must retain the computed value under that name. + value = int(raw_path[raw_index + 1:raw_index + 3], 16) + # What: handle value error by return; why: _escaped_path_suffix converts that failure into this concrete recovery, response, or cleanup behavior. + except ValueError: + # What: reject the malformed or mismatched escaped path; why: the upstream proxy returns no suffix so its caller emits HTTP 400 instead of forwarding ambiguous path bytes. + return None + # What: compute end from raw index and 3; why: raw index end later reads end, so _escaped_path_suffix must retain the computed value under that name. + end = raw_index + 3 + # What: gate on value and prefix and prefix index before the computed value; why: _escaped_path_suffix admits the computed value only for this predicate and excludes the opposite state. + if value != prefix[prefix_index]: + # What: reject the malformed or mismatched escaped path; why: the upstream proxy returns no suffix so its caller emits HTTP 400 instead of forwarding ambiguous path bytes. + return None + # What: compute raw index from end; why: suffix raw path raw index later reads raw index, so _escaped_path_suffix must retain the computed value under that name. + raw_index = end + # What: compute prefix index from 1; why: if prefix index len prefix later reads prefix index, so _escaped_path_suffix must retain the computed value under that name. + prefix_index += 1 + # What: gate on prefix index and len and prefix before the computed value; why: _escaped_path_suffix admits the computed value only for this predicate and excludes the opposite state. + if prefix_index != len(prefix): + # What: reject the malformed or mismatched escaped path; why: the upstream proxy returns no suffix so its caller emits HTTP 400 instead of forwarding ambiguous path bytes. + return None + # What: compute suffix from raw path and raw index; why: return suffix decode ascii later reads suffix, so _escaped_path_suffix must retain the computed value under that name. + suffix = raw_path[raw_index:] + # What: establish the handler boundary for the protected operation; why: _escaped_path_suffix routes failures to unicode decode error while preserving cleanup and success flow. + try: + # What: return decode and suffix and ascii from _escaped_path_suffix; why: _escaped_path_suffix exposes decode and suffix and ascii so its caller can continue with the function\'s computed outcome. + return suffix.decode("ascii") + # What: handle unicode decode error by return quote from bytes suffix safe value; why: _escaped_path_suffix converts that failure into this concrete recovery, response, or cleanup behavior. + except UnicodeDecodeError: + # What: return quote from bytes and suffix and value from _escaped_path_suffix; why: _escaped_path_suffix exposes quote from bytes and suffix and value so its caller can continue with the function\'s computed outcome. + return quote_from_bytes(suffix, safe="/%:@!$&'()*+,;=-._~") + + +# What: define _extract_api_key around authorization and x api key; why: its direct callers call _extract_api_key for extract api key and rely on this exact input and result contract. +def _extract_api_key(authorization: str | None, x_api_key: str | None) -> str | None: + """Apply the pinned Basic-password, Bearer, then x-api-key contract.""" + # What: document apply the pinned basic password bearer then in the _extract_api_key docstring; why: introspection and maintainers read this exact docstring fragment to understand extract api key behavior without executing it. + # What: compute bearer key from the named fixture input; why: bearer key credentials or later reads bearer key, so _extract_api_key must retain the computed value under that name. + bearer_key = None + # What: compute basic key from the named fixture input; why: basic key decoded split or later reads basic key, so _extract_api_key must retain the computed value under that name. + basic_key = None + # What: gate on authorization before scheme and separator and credentials and partition and authorization; why: _extract_api_key admits scheme and separator and credentials and partition and authorization only for this predicate and excludes the opposite state. + if authorization: + # What: compute scheme and separator and credentials from partition and authorization and value; why: if separator and scheme lower bearer later reads scheme and separator and credentials, so _extract_api_key must retain the computed value under that name. + scheme, separator, credentials = authorization.partition(" ") + # What: gate on separator and lower and scheme before bearer key and credentials; why: _extract_api_key admits bearer key and credentials only for this predicate and excludes the opposite state. + if separator and scheme.lower() == "bearer": + # What: compute bearer key from credentials; why: return basic key or bearer key or x api key later reads bearer key, so _extract_api_key must retain the computed value under that name. + bearer_key = credentials or None + # What: gate on separator and lower and scheme before decoded and decode and error and value error and basic key; why: _extract_api_key admits decoded and decode and error and value error and basic key only for this predicate and excludes the opposite state. + elif separator and scheme.lower() == "basic": + # What: establish the handler boundary for the protected operation; why: _extract_api_key routes failures to error and value error and binascii while preserving cleanup and success flow. + try: + # What: compute decoded from decode and b64decode and credentials and base64 and utf 8; why: if in decoded later reads decoded, so _extract_api_key must retain the computed value under that name. + decoded = base64.b64decode(credentials, validate=True).decode( + # What: supply errors to operation.decode; why: _extract_api_key binds this surrogateescape value to operation.decode's errors input. + "utf-8", errors="surrogateescape" + # What: complete the operation.decode call with errors; why: _extract_api_key groups the supplied clauses as one operation.decode call before its value is consumed. + ) + # What: handle error and value error and binascii by pass; why: _extract_api_key converts that failure into this concrete recovery, response, or cleanup behavior. + except (binascii.Error, ValueError): + # What: ignore the anticipated exception handled by this branch; why: _extract_api_key continues its retry or cleanup path instead of re-raising that transient failure. + pass + # What: select the remaining branch that performs if in decoded; why: _extract_api_key covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: gate on decoded before basic key and split and decoded; why: _extract_api_key admits basic key and split and decoded only for this predicate and excludes the opposite state. + if ":" in decoded: + # What: compute basic key from split and decoded and 1 and value and 1; why: return basic key or bearer key or x api key later reads basic key, so _extract_api_key must retain the computed value under that name. + basic_key = decoded.split(":", 1)[1] or None + # What: return basic key and bearer key and x api key from _extract_api_key; why: _extract_api_key exposes basic key and bearer key and x api key so its caller can continue with the function\'s computed outcome. + return basic_key or bearer_key or x_api_key + + class StartBody(BaseModel): model: str port: int | None = None @@ -40,6 +203,32 @@ class SwitchBody(StartBody): force: bool = False +# What: define ProfileBody as the owner of its declared state; why: daemon callers use this class boundary so those methods share one profile body state invariant. +class ProfileBody(BaseModel): + # What: compute name from the named fixture input; why: name str later reads name, so app must retain the computed value under that name. + name: str + # What: compute force from false; why: return await run lifecycle pool manager stop bool later reads force, so app must retain the computed value under that name. + force: bool = False + + +# What: define RoutingProfileSelectionBody as the owner of its declared state; why: daemon callers use this class boundary so those methods share one routing profile selection body state invariant. +class RoutingProfileSelectionBody(BaseModel): + # What: compute name from the named fixture input; why: name str later reads name, so app must retain the computed value under that name. + name: str | None + + +# What: define RouterUnloadBody as the owner of its declared state; why: daemon callers use this class boundary so those methods share one router unload body state invariant. +class RouterUnloadBody(BaseModel): + # What: compute name from the named fixture input; why: name str later reads name, so app must retain the computed value under that name. + name: str | None = None + + +# What: define RouterLoadBody as the owner of its declared state; why: daemon callers use this class boundary so those methods share one router load body state invariant. +class RouterLoadBody(BaseModel): + # What: compute name from the named fixture input; why: html lang en head meta charset later reads name, so app must retain the computed value under that name. + name: str + + class AccountingAckBody(BaseModel): receiptId: str @@ -58,6 +247,42 @@ class BenchBody(BaseModel): args: list[str] = [] +# Deliberately dependency-free management view. It never embeds catalog data, +# local paths, tokens, or machine identifiers in the initial HTML response; +# authenticated JSON API calls populate the view only after the operator enters +# a bearer token for this browser session. +# What: embed the exact router ui doctype html router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact html lang en head meta charset router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact title free token swap title style router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact body font px system ui sans serif max width router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact style head body h1 free token swap router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact div class row label bearer key router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact h2 status h2 pre id status router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact h2 activity h2 p rows are router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact script router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact const id document get element by id id headers authorization router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact async function api path opt let router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact function show id value id text content router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact async function refresh try let s router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact refresh onclick refresh reload onclick async router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +# What: embed the exact script body html router-interface fragment; why: the router UI consumer receives this fragment verbatim through router ui, preserving browser markup, style, or script behavior. +_ROUTER_UI = """ + +FreeToken swap

FreeToken swap

Enter a router bearer key to inspect or control this local daemon. The key is kept only in this page's memory.

+
+

Status

Not loaded.

Models

Hardware

Not loaded.

Performance history

Not loaded.
+

Activity

Rows are body-free. Captures may contain prompts and are fetched only when selected.

No capture selected.
+""" + + def _bench_profile_path(gpu_uuid: str | None) -> str | None: # per-GPU profiles and no torch here: the serve's own card when its --gpu names one, else the newest file from freetoken.moe.bench_profile import default_profile_path, latest_profile_path # torch-free @@ -127,12 +352,140 @@ def build_app( started_wall: float = 0.0, wall_now: Callable[[], float] | None = None, shutdown_hook: Callable[[], None] | None = None, + # What: declare the catalog input for build_app; why: build_app consumes catalog during catalog catalog or model catalog empty, so callers must bind it with the other signature inputs. + catalog: ModelCatalog | None = None, + # What: declare the router input for build_app; why: build_app consumes router during router router or routing coordinator, so callers must bind it with the other signature inputs. + router: RoutingCoordinator | None = None, + # What: declare the catalog path input for build_app; why: build_app consumes catalog path during if not catalog path, so callers must bind it with the other signature inputs. + catalog_path: str | None = None, + # What: declare the router ring input for build_app; why: build_app consumes router ring during router ring router ring or log ring capacity, so callers must bind it with the other signature inputs. + router_ring: LogRing | None = None, + # What: declare the catalog watch interval s input for build_app; why: build_app consumes catalog watch interval s during interval s catalog watch interval s if catalog watch interval s else, so callers must bind it with the other signature inputs. + catalog_watch_interval_s: float = 0.0, + # What: declare the activity path input for build_app; why: build_app consumes activity path during activity path, so callers must bind it with the other signature inputs. + activity_path: str | None = None, ) -> FastAPI: import time as _time wall_now = wall_now or _time.time app = FastAPI(title="FreeToken daemon", version=DAEMON_VERSION) + # What: register cors_preflight as HTTP middleware; why: every matching request passes through cors_preflight before the route handler so authentication or accounting wraps the request. + @app.middleware("http") + # What: define cors_preflight around request and call next; why: the registered API client call cors_preflight for cors preflight and rely on this exact input and result contract. + async def cors_preflight(request: Request, call_next): + # Match the pinned compatibility server's side-effect-free global + # preflight contract. Actual requests still pass through normal route + # authentication and lifecycle ownership. + # What: gate on method and request before call next and request; why: cors_preflight admits call next and request only for this predicate and excludes the opposite state. + if request.method != "OPTIONS": + # What: return call next and request from cors_preflight; why: cors_preflight exposes call next and request so its caller can continue with the function\'s computed outcome. + return await call_next(request) + # What: return response and cors request headers and get and headers from cors_preflight; why: cors_preflight exposes response and cors request headers and get and headers so its caller can continue with the function\'s computed outcome. + return Response( + # What: supply status code to Response; why: cors_preflight binds this 204 value to Response's status code input. + status_code=204, + # What: supply headers to Response; why: cors_preflight binds this cors request headers and get and headers and request and access control allow origin value to Response's headers input. + headers={ + # What: map the access control allow origin field as value; why: cors_preflight carries access control allow origin into "Access-Control-Allow-Origin": "*". + "Access-Control-Allow-Origin": "*", + # What: map the access control allow methods field as get and post and put and patch; why: cors_preflight carries access control allow methods into "Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS". + "Access-Control-Allow-Methods": "GET, POST, PUT, PATCH, DELETE, OPTIONS", + # What: map the access control allow headers field as cors request headers and get and headers and request and access control request headers; why: cors_preflight carries access control allow headers into "Access-Control-Allow-Headers": _cors_request_headers(. + "Access-Control-Allow-Headers": _cors_request_headers( + # What: call request.headers.get with access control request headers; why: cors_preflight consumes the request.headers.get return value while evaluating request.headers.get("access-control-request-headers"). + request.headers.get("access-control-request-headers") + # What: complete the _cors_request_headers call with get; why: cors_preflight groups the supplied clauses as one _cors_request_headers call before its value is consumed. + ), + # What: map the access control max age field as 86400; why: cors_preflight carries access control max age into "Access-Control-Max-Age": "86400". + "Access-Control-Max-Age": "86400", + # What: complete the enclosing predicate mapping with access control allow origin and access control allow methods and access control allow headers and access control max age; why: cors_preflight groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + }, + # What: complete the Response call with status code and headers; why: cors_preflight groups the supplied clauses as one Response call before its value is consumed. + ) + # What: compute catalog from catalog and empty and model catalog; why: manager catalog probe default port default serve port later reads catalog, so build_app must retain the computed value under that name. + catalog = catalog or ModelCatalog.empty() + # What: compute router from router and routing coordinator and manager and catalog; why: app router add event handler startup start performance monitor later reads router, so build_app must retain the computed value under that name. + router = router or RoutingCoordinator( + # What: supply default port to RoutingCoordinator; why: build_app binds this default serve port value to RoutingCoordinator's default port input. + manager, catalog, probe, default_port=default_serve_port + # What: complete the RoutingCoordinator call with default port; why: build_app groups the supplied clauses as one RoutingCoordinator call before its value is consumed. + ) + # Keep router events separate from captured engine stdout. Apart from + # making an operator's engine-log view useful, this prevents a noisy child + # from evicting the bounded lifecycle/proxy audit trail. The event payload + # deliberately contains no headers, query strings, request body, or model + # path: those may carry credentials or prompts. + # What: compute router ring from router ring and log ring and 1000; why: app state router ring router ring later reads router ring, so build_app must retain the computed value under that name. + router_ring = router_ring or LogRing(capacity=1000) + # What: compute router ring from router ring; why: router ring append later reads router ring, so build_app must retain the computed value under that name. + app.state.router_ring = router_ring + # What: compute activity store from activity store and activity max entries and activity path and activity session headers; why: app state activity store activity store later reads activity store, so build_app must retain the computed value under that name. + activity_store = ActivityStore( + # What: apply the catalog settings activity max entries portion of activity store; why: build_app uses this clause to evaluate activity store as one grouped value. + catalog.settings.activity_max_entries, + # What: apply the catalog settings capture buffer mb portion of activity store; why: build_app uses this clause to evaluate activity store as one grouped value. + catalog.settings.capture_buffer_mb * 1024 * 1024, + # What: apply the activity path portion of activity store; why: build_app uses this clause to evaluate activity store as one grouped value. + activity_path, + # What: apply the catalog settings activity session headers portion of activity store; why: build_app uses this clause to evaluate activity store as one grouped value. + catalog.settings.activity_session_headers, + # What: complete the ActivityStore call with activity max entries and capture buffer mb and activity path and activity session headers; why: build_app groups the supplied clauses as one ActivityStore call before its value is consumed. + ) + # What: compute activity store from activity store; why: activity store reconfigure later reads activity store, so build_app must retain the computed value under that name. + app.state.activity_store = activity_store + # What: compute performance monitor from performance monitor and performance every s and performance disabled and wall now; why: app state performance monitor performance monitor later reads performance monitor, so build_app must retain the computed value under that name. + performance_monitor = PerformanceMonitor( + # What: call footprint_fn with get and status and manager and pid; why: build_app invokes footprint_fn while performing every s catalog settings performance every s; the call advances that operation through its result or side effect. + lambda: footprint_fn(manager.status().get("pid")), + # What: supply every s to PerformanceMonitor; why: build_app binds this performance every s and settings and catalog value to PerformanceMonitor's every s input. + every_s=catalog.settings.performance_every_s, + # What: supply disabled to PerformanceMonitor; why: build_app binds this performance disabled and settings and catalog value to PerformanceMonitor's disabled input. + disabled=catalog.settings.performance_disabled, + # What: supply wall now to PerformanceMonitor; why: build_app binds this wall now value to PerformanceMonitor's wall now input. + wall_now=wall_now, + # What: complete the PerformanceMonitor call with every s and disabled and wall now; why: build_app groups the supplied clauses as one PerformanceMonitor call before its value is consumed. + ) + # What: compute performance monitor from performance monitor; why: performance monitor start later reads performance monitor, so build_app must retain the computed value under that name. + app.state.performance_monitor = performance_monitor + + # What: define _start_performance_monitor around the current object state; why: its direct callers call _start_performance_monitor for start performance monitor and rely on this exact input and result contract. + async def _start_performance_monitor() -> None: + # What: call performance_monitor.start with the declared inputs; why: _start_performance_monitor invokes performance_monitor.start while performing the enclosing return; the call advances that operation through its result or side effect. + performance_monitor.start() + + # What: define _stop_performance_monitor around the current object state; why: its direct callers call _stop_performance_monitor for stop performance monitor and rely on this exact input and result contract. + async def _stop_performance_monitor() -> None: + # What: call performance_monitor.stop with the declared inputs; why: _stop_performance_monitor invokes performance_monitor.stop while performing the enclosing return; the call advances that operation through its result or side effect. + performance_monitor.stop() + + # What: preserve the exact app router add event handler startup start performance monitor literal fragment; why: build_app passes this fragment verbatim through app.router.add_event_handler("startup", _start_performance_monitor), because changing it would alter a protocol payload, serialized fixture, or public mes. + app.router.add_event_handler("startup", _start_performance_monitor) + # What: preserve the exact app router add event handler shutdown stop performance monitor literal fragment; why: build_app passes this fragment verbatim through app.router.add_event_handler("shutdown", _stop_performance_monitor), because changing it would alter a protocol payload, serialized fixture, or public mes. + app.router.add_event_handler("shutdown", _stop_performance_monitor) + # What: compute inflight lock from lock and threading; why: with inflight lock later reads inflight lock, so build_app must retain the computed value under that name. + inflight_lock = threading.Lock() + # What: initialize inflight as an empty runtime accumulator; why: build_app appends or maps entries into it during inflight request id before consuming the aggregate. + inflight: dict[str, dict] = {} + # What: initialize request reservations as an empty runtime accumulator; why: build_app appends or maps entries into it during if request id in request reservations before consuming the aggregate. + request_reservations: dict[str, dict] = {} + # What: compute watch stop from event and threading; why: app state catalog watch stop watch stop later reads watch stop, so build_app must retain the computed value under that name. + watch_stop = threading.Event() + # What: compute watch lock from lock and threading; why: with watch lock later reads watch lock, so build_app must retain the computed value under that name. + watch_lock = threading.Lock() + # What: compute watch state from bool and catalog watch interval s and catalog path and enabled and interval s; why: watch state last result result later reads watch state, so build_app must retain the computed value under that name. + watch_state = { + # What: map the enabled field as bool and catalog path and catalog watch interval s and 0; why: build_app carries enabled through watch state into watch state last result result. + "enabled": bool(catalog_path and catalog_watch_interval_s > 0), + # What: map the interval s field as catalog watch interval s and 0; why: build_app carries interval s through watch state into watch state last result result. + "intervalS": catalog_watch_interval_s if catalog_watch_interval_s > 0 else None, + # What: map the last result field as the fixture input; why: build_app carries last result through watch state into watch state last result result. + "lastResult": None, + # What: complete the watch_state mapping with enabled and interval s and last result; why: build_app groups the supplied clauses as one watch_state mapping before its value is consumed. + } + # What: compute catalog watch stop from watch stop; why: the enclosing return or state update later reads catalog watch stop, so build_app must retain the computed value under that name. + app.state.catalog_watch_stop = watch_stop + if shutdown_hook is not None: @app.on_event("shutdown") @@ -145,22 +498,255 @@ async def _on_shutdown() -> None: except Exception: # noqa: BLE001 pass - def require_token(x_ft_token: str | None = Header(default=None)) -> None: - if token is not None and x_ft_token != token: - raise HTTPException(status_code=401, detail="invalid or missing X-FT-Token") + # What: define require_token around x ft token and authorization and x api key; why: its direct callers call require_token for require token and rely on this exact input and result contract. + def require_token( + # What: declare the x ft token input for require_token; why: require_token consumes x ft token during if x ft token token, so callers must bind it with the other signature inputs. + x_ft_token: str | None = Header(default=None), + # What: declare the authorization input for require_token; why: require_token consumes authorization during supplied extract api key authorization x api key, so callers must bind it with the other signature inputs. + authorization: str | None = Header(default=None), + # What: declare the x api key input for require_token; why: require_token consumes x api key during supplied extract api key authorization x api key, so callers must bind it with the other signature inputs. + x_api_key: str | None = Header(default=None), + # What: complete the enclosing predicate with group delimiter; why: require_token groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) -> None: + # What: gate on token before x ft token and token and httpexception; why: require_token admits x ft token and token and httpexception only for this predicate and excludes the opposite state. + if token is not None: + # What: gate on x ft token and token before httpexception; why: require_token admits httpexception only for this predicate and excludes the opposite state. + if x_ft_token != token: + # What: raise HTTPException for the caller; why: require_token stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=401, detail="invalid or missing X-FT-Token") + # What: return no value from require_token; why: require_token returns no value to callers that depend on its completed result. + return + # What: compute keys from api keys and settings and catalog and router; why: if keys later reads keys, so require_token must retain the computed value under that name. + keys = router.catalog.settings.api_keys + # What: gate on keys before supplied and extract api key and authorization and x api key; why: require_token admits supplied and extract api key and authorization and x api key only for this predicate and excludes the opposite state. + if keys: + # What: compute supplied from extract api key and authorization and x api key; why: if supplied not in keys later reads supplied, so require_token must retain the computed value under that name. + supplied = _extract_api_key(authorization, x_api_key) + # What: gate on supplied and keys before httpexception; why: require_token admits httpexception only for this predicate and excludes the opposite state. + if supplied not in keys: + # What: raise HTTPException for the caller; why: require_token stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException( + # What: supply status code to HTTPException; why: require_token binds this 401 value to HTTPException's status code input. + status_code=401, + # What: supply detail to HTTPException; why: require_token binds this invalid and or and missing and api value to HTTPException's detail input. + detail="invalid or missing API key", + # What: map the www authenticate field as basic and realm and freetoken swap; why: require_token carries www authenticate into headers={"WWW-Authenticate": 'Basic realm="freetoken-swap"'}. + headers={"WWW-Authenticate": 'Basic realm="freetoken-swap"'}, + # What: complete the HTTPException call with status code and detail and headers; why: require_token groups the supplied clauses as one HTTPException call before its value is consumed. + ) auth = [Depends(require_token)] - async def run(pool: ThreadPoolExecutor, fn, *args): + # What: define require_router_key around authorization and x api key; why: its direct callers call require_router_key for require router key and rely on this exact input and result contract. + def require_router_key( + # What: declare the authorization input for require_router_key; why: require_router_key consumes authorization during supplied extract api key authorization x api key, so callers must bind it with the other signature inputs. + authorization: str | None = Header(default=None), + # What: declare the x api key input for require_router_key; why: require_router_key consumes x api key during supplied extract api key authorization x api key, so callers must bind it with the other signature inputs. + x_api_key: str | None = Header(default=None), + # What: complete the enclosing predicate with group delimiter; why: require_router_key groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) -> None: + # What: compute keys from api keys and settings and catalog and router; why: if not keys later reads keys, so require_router_key must retain the computed value under that name. + keys = router.catalog.settings.api_keys + # What: gate on keys before the computed value; why: require_router_key admits the computed value only for this predicate and excludes the opposite state. + if not keys: + # What: return no value from require_router_key; why: require_router_key returns no value to callers that depend on its completed result. + return + # What: compute supplied from extract api key and authorization and x api key; why: if supplied not in keys later reads supplied, so require_router_key must retain the computed value under that name. + supplied = _extract_api_key(authorization, x_api_key) + # What: gate on supplied and keys before httpexception; why: require_router_key admits httpexception only for this predicate and excludes the opposite state. + if supplied not in keys: + # What: raise HTTPException for the caller; why: require_router_key stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException( + # What: supply status code to HTTPException; why: require_router_key binds this 401 value to HTTPException's status code input. + status_code=401, + # What: supply detail to HTTPException; why: require_router_key binds this invalid and or and missing and api value to HTTPException's detail input. + detail="invalid or missing API key", + # What: map the www authenticate field as basic and realm and freetoken swap; why: require_router_key carries www authenticate into headers={"WWW-Authenticate": 'Basic realm="freetoken-swap"'}. + headers={"WWW-Authenticate": 'Basic realm="freetoken-swap"'}, + # What: complete the HTTPException call with status code and detail and headers; why: require_router_key groups the supplied clauses as one HTTPException call before its value is consumed. + ) + + # What: define run around pool and fn; why: its direct callers call run for run and rely on this exact input and result contract. + async def run(pool: ThreadPoolExecutor, fn, *args, **kwargs): loop = asyncio.get_running_loop() - return await loop.run_in_executor(pool, functools.partial(fn, *args)) + # What: return run in executor and pool and loop and partial from run; why: run exposes run in executor and pool and loop and partial so its caller can continue with the function\'s computed outcome. + return await loop.run_in_executor(pool, functools.partial(fn, *args, **kwargs)) + + # What: define run_to_completion around operation; why: its direct callers call run_to_completion for run to completion and rely on this exact input and result contract. + async def run_to_completion(operation): + """Defer caller cancellation until an ownership transaction is terminal.""" + # What: document defer caller cancellation until an ownership in the run_to_completion docstring; why: introspection and maintainers read this exact docstring fragment to understand run to completion behavior without executing it. + # What: compute task from create task and asyncio and operation; why: return await asyncio shield task later reads task, so run_to_completion must retain the computed value under that name. + task = asyncio.create_task(operation()) + # What: establish the handler boundary for the protected operation; why: run_to_completion routes failures to cancelled error and asyncio while preserving cleanup and success flow. + try: + # What: return shield and task and asyncio from run_to_completion; why: run_to_completion exposes shield and task and asyncio so its caller can continue with the function\'s computed outcome. + return await asyncio.shield(task) + # What: handle cancelled error and asyncio by while true; why: run_to_completion converts that failure into this concrete recovery, response, or cleanup behavior. + except asyncio.CancelledError: + # What: iterate across the computed value to perform cancelled error and base exception and asyncio and shield and task; why: run_to_completion repeats the body only while or for the loop header admits an iteration. + while True: + # What: establish the handler boundary for the protected operation; why: run_to_completion routes failures to cancelled error and asyncio and base exception while preserving cleanup and success flow. + try: + # What: call asyncio.shield with task; why: run_to_completion invokes asyncio.shield while performing except asyncio cancelled error; the call advances that operation through its result or side effect. + await asyncio.shield(task) + # What: handle cancelled error and asyncio by continue; why: run_to_completion converts that failure into this concrete recovery, response, or cleanup behavior. + except asyncio.CancelledError: + # What: apply the continue portion of the enclosing predicate; why: this clause remains in run_to_completion\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: handle base exception by break; why: run_to_completion converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException: + # What: apply the break portion of the enclosing predicate; why: this clause remains in run_to_completion\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: select the remaining branch that performs break; why: run_to_completion covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: apply the break portion of the enclosing predicate; why: this clause remains in run_to_completion\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: re-propagate the active failure to the caller; why: run_to_completion stops this rejected path before it can mutate state, dispatch work, or report success. + raise + + # What: define run_manual_transaction around operation and preempt manual; why: its direct callers call run_manual_transaction for run manual transaction and rely on this exact input and result contract. + async def run_manual_transaction(operation, *, preempt_manual: bool = False): + """Keep manual ownership until the complete transaction reaches a terminal state.""" + # What: document keep manual ownership until the complete in the run_manual_transaction docstring; why: introspection and maintainers read this exact docstring fragment to understand run manual transaction behavior without executing it. + # What: compute owner from begin manual lifecycle and preempt manual; why: router end manual lifecycle owner later reads owner, so run_manual_transaction must retain the computed value under that name. + owner = begin_manual_lifecycle(preempt_manual=preempt_manual) + # What: establish the handler boundary for the protected operation; why: run_manual_transaction routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: return run to completion and operation from run_manual_transaction; why: run_manual_transaction exposes run to completion and operation so its caller can continue with the function\'s computed outcome. + return await run_to_completion(operation) + # What: run router end manual lifecycle owner on every exit path; why: run_manual_transaction performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: call router.end_manual_lifecycle with owner; why: run_manual_transaction invokes router.end_manual_lifecycle while performing the enclosing return; the call advances that operation through its result or side effect. + router.end_manual_lifecycle(owner) + + # What: define acquire_route around name and cancellation and on reserved and apply loading policy and apply routing profile; why: its direct callers call acquire_route for acquire route and rely on this exact input and result contract. + async def acquire_route( + # What: declare the name input for acquire_route; why: acquire_route consumes name during name, so callers must bind it with the other signature inputs. + name: str, + # What: declare the cancellation input for acquire_route; why: acquire_route consumes cancellation during cancellation cancellation or threading event, so callers must bind it with the other signature inputs. + cancellation: threading.Event | None = None, + # What: declare the on reserved input for acquire_route; why: acquire_route consumes on reserved during on reserved, so callers must bind it with the other signature inputs. + on_reserved: Callable[[bool, int], None] | None = None, + # What: mark the remaining parameters as keyword-only; why: acquire_route prevents callers from confusing adjacent lifecycle and timing arguments. + *, + # What: declare the apply loading policy input for acquire_route; why: acquire_route consumes apply loading policy during apply loading policy apply loading policy, so callers must bind it with the other signature inputs. + apply_loading_policy: bool = False, + # What: declare the apply routing profile input for acquire_route; why: acquire_route consumes apply routing profile during apply routing profile apply routing profile, so callers must bind it with the other signature inputs. + apply_routing_profile: bool = True, + # What: complete the enclosing predicate with async def acquire route name str cancellation threading event on reserved callable; why: acquire_route groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + """Keep executor-side admission owned if its HTTP task is cancelled.""" + # What: document keep executor side admission owned if its in the acquire_route docstring; why: introspection and maintainers read this exact docstring fragment to understand acquire route behavior without executing it. + # What: compute loop from get running loop and asyncio; why: future loop run in executor later reads loop, so acquire_route must retain the computed value under that name. + loop = asyncio.get_running_loop() + # What: compute cancellation from cancellation and event and threading; why: cancellation later reads cancellation, so acquire_route must retain the computed value under that name. + cancellation = cancellation or threading.Event() + # What: compute future from run in executor and lifecycle pool and loop and partial; why: shielded asyncio shield future later reads future, so acquire_route must retain the computed value under that name. + future = loop.run_in_executor( + # What: apply the lifecycle pool portion of future; why: acquire_route uses this clause to evaluate future as one grouped value. + lifecycle_pool, + # What: call functools.partial with acquire and router and name and cancellation; why: acquire_route invokes functools.partial while performing router acquire; the call advances that operation through its result or side effect. + functools.partial( + # What: apply the router acquire portion of future; why: acquire_route uses this clause to evaluate future as one grouped value. + router.acquire, + # What: apply the name portion of future; why: acquire_route uses this clause to evaluate future as one grouped value. + name, + # What: apply the cancellation portion of future; why: acquire_route uses this clause to evaluate future as one grouped value. + cancellation, + # What: apply the on reserved portion of future; why: acquire_route uses this clause to evaluate future as one grouped value. + on_reserved, + # What: supply apply loading policy to functools.partial; why: acquire_route binds this apply loading policy value to functools.partial's apply loading policy input. + apply_loading_policy=apply_loading_policy, + # What: supply apply routing profile to functools.partial; why: acquire_route binds this apply routing profile value to functools.partial's apply routing profile input. + apply_routing_profile=apply_routing_profile, + # What: complete the functools.partial call with apply loading policy and apply routing profile; why: acquire_route groups the supplied clauses as one functools.partial call before its value is consumed. + ), + # What: complete the loop.run_in_executor call with lifecycle pool and partial; why: acquire_route groups the supplied clauses as one loop.run_in_executor call before its value is consumed. + ) + # What: compute shielded from shield and future and asyncio; why: return await shielded later reads shielded, so acquire_route must retain the computed value under that name. + shielded = asyncio.shield(future) + # What: establish the handler boundary for the protected operation; why: acquire_route routes failures to cancelled error and asyncio while preserving cleanup and success flow. + try: + # What: return shielded from acquire_route; why: acquire_route exposes shielded so its caller can continue with the function\'s computed outcome. + return await shielded + # What: handle cancelled error and asyncio by router cancel acquire cancellation; why: acquire_route converts that failure into this concrete recovery, response, or cleanup behavior. + except asyncio.CancelledError: + # What: call router.cancel_acquire with cancellation; why: acquire_route invokes router.cancel_acquire while performing try; the call advances that operation through its result or side effect. + router.cancel_acquire(cancellation) + # Retain ownership until the executor-side admission is terminal. + # This retrieves its expected RoutingError before the event loop + # can close and releases a lease if admission won the race. + # What: establish the handler boundary for the protected operation; why: acquire_route routes failures to base exception while preserving cleanup and success flow. + try: + # What: compute orphaned from shield and future and asyncio; why: orphaned release later reads orphaned, so acquire_route must retain the computed value under that name. + orphaned = await asyncio.shield(future) + # What: handle base exception by pass; why: acquire_route converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException: + # What: ignore the anticipated exception handled by this branch; why: acquire_route continues its retry or cleanup path instead of re-raising that transient failure. + pass + # What: select the remaining branch that performs orphaned release; why: acquire_route covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: call orphaned.release with the declared inputs; why: acquire_route invokes orphaned.release while performing raise; the call advances that operation through its result or side effect. + orphaned.release() + # What: re-propagate the active failure to the caller; why: acquire_route stops this rejected path before it can mutate state, dispatch work, or report success. + raise + + # What: define connect_upstream around the current object state; why: its direct callers call connect_upstream for connect upstream and rely on this exact input and result contract. + async def connect_upstream(**kwargs): + """Close a connector result that arrives after its HTTP task disconnects.""" + # What: document close a connector result that arrives in the connect_upstream docstring; why: introspection and maintainers read this exact docstring fragment to understand connect upstream behavior without executing it. + # What: compute loop from get running loop and asyncio; why: future loop run in executor proxy pool functools partial open upstream kwargs later reads loop, so connect_upstream must retain the computed value under that name. + loop = asyncio.get_running_loop() + # What: compute future from run in executor and proxy pool and loop and partial; why: return await asyncio shield future later reads future, so connect_upstream must retain the computed value under that name. + future = loop.run_in_executor(proxy_pool, functools.partial(open_upstream, **kwargs)) + # What: establish the handler boundary for the protected operation; why: connect_upstream routes failures to cancelled error and asyncio while preserving cleanup and success flow. + try: + # What: return shield and future and asyncio from connect_upstream; why: connect_upstream exposes shield and future and asyncio so its caller can continue with the function\'s computed outcome. + return await asyncio.shield(future) + # What: handle cancelled error and asyncio by def close orphaned upstream done; why: connect_upstream converts that failure into this concrete recovery, response, or cleanup behavior. + except asyncio.CancelledError: + # What: define close_orphaned_upstream around done; why: its direct callers call close_orphaned_upstream for close orphaned upstream and rely on this exact input and result contract. + def close_orphaned_upstream(done) -> None: + # What: establish the handler boundary for the protected operation; why: close_orphaned_upstream routes failures to base exception while preserving cleanup and success flow. + try: + # What: compute orphaned from result and done; why: orphaned close later reads orphaned, so close_orphaned_upstream must retain the computed value under that name. + orphaned = done.result() + # What: handle base exception by return; why: close_orphaned_upstream converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException: + # What: return no value from close_orphaned_upstream; why: close_orphaned_upstream returns no value to callers that depend on its completed result. + return + # What: call orphaned.close with the declared inputs; why: close_orphaned_upstream invokes orphaned.close while performing the enclosing return; the call advances that operation through its result or side effect. + orphaned.close() + + # What: call future.add_done_callback with close orphaned upstream; why: connect_upstream invokes future.add_done_callback while performing raise; the call advances that operation through its result or side effect. + future.add_done_callback(close_orphaned_upstream) + # What: re-propagate the active failure to the caller; why: connect_upstream stops this rejected path before it can mutate state, dispatch work, or report success. + raise def resolve_port(explicit: int | None) -> int: + # What: gate on explicit before allocate loopback port; why: resolve_port admits allocate loopback port only for this predicate and excludes the opposite state. + if explicit == 0: + # What: return allocate loopback port from resolve_port; why: resolve_port exposes allocate loopback port so its caller can continue with the function\'s computed outcome. + return allocate_loopback_port() if explicit is not None: return explicit st = manager.status() return st.get("port") or default_serve_port + # What: define begin_manual_lifecycle around preempt manual; why: its direct callers call begin_manual_lifecycle for begin manual lifecycle and rely on this exact input and result contract. + def begin_manual_lifecycle(*, preempt_manual: bool = False) -> object: + """Atomically keep legacy engine controls outside routed ownership.""" + # What: document atomically keep legacy engine controls outside in the begin_manual_lifecycle docstring; why: introspection and maintainers read this exact docstring fragment to understand begin manual lifecycle behavior without executing it. + # What: establish the handler boundary for the protected operation; why: begin_manual_lifecycle routes failures to routing error while preserving cleanup and success flow. + try: + # What: return begin manual lifecycle and router and preempt manual from begin_manual_lifecycle; why: begin_manual_lifecycle exposes begin manual lifecycle and router and preempt manual so its caller can continue with the function\'s computed outcome. + return router.begin_manual_lifecycle(preempt_manual=preempt_manual) + # What: handle routing error by raise httpexception status code exc status code detail str exc; why: begin_manual_lifecycle converts that failure into this concrete recovery, response, or cleanup behavior. + except RoutingError as exc: + # What: raise HTTPException for the caller; why: begin_manual_lifecycle stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=exc.status_code, detail=str(exc)) from exc + def accounting_error(exc: Exception) -> JSONResponse: code = ( "accounting_outbox_failed" @@ -188,33 +774,2169 @@ async def health(): "engineRunning": bool(st.get("running")), } - # ---- engine lifecycle ---- + # What: register GET /ready on the application router; why: clients reach ready's handler only through this method-and-path binding. + @app.get("/ready") + # What: define the /ready control-plane handler; why: supervisors call this handler to learn whether the daemon control plane can accept requests, independent of engine stop receipts or token accounting. + async def ready(): + """Stable router readiness; it never starts a model as a probe side effect.""" + # What: document stable router readiness it never starts in the ready docstring; why: introspection and maintainers read this exact docstring fragment to understand ready behavior without executing it. + # What: compute accepting from run and proxy pool and is ready and probe; why: return jsonresponse status code if accepting else later reads accepting, so ready must retain the computed value under that name. + accepting = await run(proxy_pool, router.is_ready, probe) + # What: return HTTP 200 when accepting and 503 otherwise; why: supervisors use this status and ready boolean to decide whether the daemon control plane may receive traffic. + return JSONResponse(status_code=200 if accepting else 503, content={"ready": accepting}) + + # What: register GET /ui/ on the application router; why: clients reach router_ui's handler only through this method-and-path binding. + @app.get("/ui/") + # What: define router_ui around the current object state; why: the registered API client call router_ui for router ui and rely on this exact input and result contract. + async def router_ui(): + """A static shell; authenticated APIs supply all operational data.""" + # What: document a static shell authenticated apis supply in the router_ui docstring; why: introspection and maintainers read this exact docstring fragment to understand router ui behavior without executing it. + # What: return htmlresponse and router ui from router_ui; why: router_ui exposes htmlresponse and router ui so its caller can continue with the function\'s computed outcome. + return HTMLResponse(_ROUTER_UI) + + # What: define router_event around event; why: its direct callers call router_event for router event and rely on this exact input and result contract. + def router_event(event: str, **fields: Any) -> None: + # What: call router_ring.append with dumps and json and event and fields and event; why: router_event invokes router_ring.append while performing json dumps event event fields separators sort keys; the call advances that operation through its result or side effect. + router_ring.append( + # What: map the event field as event; why: router_event carries event into json.dumps({"event": event, **fields}, separators=(",", ":"), sort_keys=. + json.dumps({"event": event, **fields}, separators=(",", ":"), sort_keys=True), + # What: preserve the exact kind event literal fragment; why: router_event passes this fragment verbatim through kind="event", because changing it would alter a protocol payload, serialized fixture, or public message. + kind="event", + # What: supply ts to wall_now; why: router_event binds this wall now value to wall_now's ts input. + ts=wall_now(), + # What: complete the router_ring.append call with kind and ts; why: router_event groups the supplied clauses as one router_ring.append call before its value is consumed. + ) - @app.post("/engine/start", dependencies=auth) - async def engine_start(body: StartBody): - port = resolve_port(body.port) + # What: execute if catalog settings startup routing profile is not None; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + if catalog.settings.startup_routing_profile is not None: + # What: call router.set_active_routing_profile with startup routing profile and settings and catalog; why: build_app invokes router.set_active_routing_profile while performing if catalog settings preload model is not; the call advances that operation through its result or side effect. + router.set_active_routing_profile(catalog.settings.startup_routing_profile) + + # What: gate on preload model and settings and catalog before name and preload model and on event and settings and lease; why: build_app admits name and preload model and on event and settings and lease only for this predicate and excludes the opposite state. + if catalog.settings.preload_model is not None: + + # What: apply app.on_event behavior to _preload_model; why: Python attaches this named decorator's registration or descriptor semantics to _preload_model. + @app.on_event("startup") + # What: define _preload_model around the current object state; why: the registered API client call _preload_model for preload model and rely on this exact input and result contract. + async def _preload_model() -> None: + # What: compute name from preload model and settings and catalog; why: lease await acquire route name apply routing profile later reads name, so _preload_model must retain the computed value under that name. + name = catalog.settings.preload_model + # What: establish the handler boundary for the protected operation; why: _preload_model routes failures to base exception while preserving cleanup and success flow. + try: + # What: compute lease from acquire route and name and false; why: lease release later reads lease, so _preload_model must retain the computed value under that name. + lease = await acquire_route(name, apply_routing_profile=False) + # What: handle base exception by router event startup preload failed profile name code type exc; why: _preload_model converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException as exc: + # What: preserve the exact router event startup preload failed profile name code type literal fragment; why: _preload_model passes this fragment verbatim through router_event("startup_preload_failed", profile=name, code=type(exc).__na, because changing it would alter a protocol payload, serialized fixt. + router_event("startup_preload_failed", profile=name, code=type(exc).__name__) + # What: return no value from _preload_model; why: _preload_model returns no value to callers that depend on its completed result. + return + # What: call lease.release with the declared inputs; why: _preload_model invokes lease.release while performing router event startup preloaded profile lease profile name; the call advances that operation through its result or side effect. + lease.release() + # What: preserve the exact router event startup preloaded profile lease profile name literal fragment; why: _preload_model passes this fragment verbatim through router_event("startup_preloaded", profile=lease.profile.name), because changing it would alter a protocol payload, serialized fixture, or public m. + router_event("startup_preloaded", profile=lease.profile.name) + + # What: define record_watch around result; why: its direct callers call record_watch for record watch and rely on this exact input and result contract. + def record_watch(result: str) -> None: + # What: enter the watch lock managed context before watch state last result result; why: record_watch releases this resource or lock after watch state last result result on both success and failure paths. + with watch_lock: + # What: compute watch state entry from result; why: watch state last changed at wall now later reads watch state entry, so record_watch must retain the computed value under that name. + watch_state["lastResult"] = result + # What: compute watch state entry from wall now; why: the enclosing return or state update later reads watch state entry, so record_watch must retain the computed value under that name. + watch_state["lastChangedAt"] = wall_now() + + # What: define catalog_watch_snapshot around the current object state; why: its direct callers call catalog_watch_snapshot for catalog watch snapshot and rely on this exact input and result contract. + def catalog_watch_snapshot() -> dict: + # What: enter the watch lock managed context before return dict watch state; why: catalog_watch_snapshot releases this resource or lock after return dict watch state on both success and failure paths. + with watch_lock: + # What: return dict and watch state from catalog_watch_snapshot; why: catalog_watch_snapshot exposes dict and watch state so its caller can continue with the function\'s computed outcome. + return dict(watch_state) + + # What: define catalog_stamp around the current object state; why: its direct callers call catalog_stamp for catalog stamp and rely on this exact input and result contract. + def catalog_stamp() -> tuple[int, int] | None: + # What: gate on catalog path before the computed value; why: catalog_stamp admits the computed value only for this predicate and excludes the opposite state. + if not catalog_path: + # What: return no value from catalog_stamp; why: catalog_stamp returns no value to callers that depend on its completed result. + return None + # What: establish the handler boundary for the protected operation; why: catalog_stamp routes failures to oserror while preserving cleanup and success flow. try: - return await run(lifecycle_pool, manager.start, body.model, port, list(body.args)) - except Conflict as exc: - st = manager.status() + # What: compute stat from stat and catalog path and os; why: return stat st mtime ns stat st size later reads stat, so catalog_stamp must retain the computed value under that name. + stat = os.stat(catalog_path) + # What: handle oserror by return; why: catalog_stamp converts that failure into this concrete recovery, response, or cleanup behavior. + except OSError: + # What: return no value from catalog_stamp; why: catalog_stamp returns no value to callers that depend on its completed result. + return None + # What: return st mtime ns and st size and stat from catalog_stamp; why: catalog_stamp exposes st mtime ns and st size and stat so its caller can continue with the function\'s computed outcome. + return stat.st_mtime_ns, stat.st_size + + # What: define start_catalog_watcher around the current object state; why: its direct callers call start_catalog_watcher for start catalog watcher and rely on this exact input and result contract. + def start_catalog_watcher() -> None: + """Poll a local catalog safely; only a fully validated tree is installed. + + Polling keeps the daemon stdlib-only and cross-platform. A changed + malformed file is remembered until it changes again, avoiding a log + storm while an editor writes it. Active-profile redefinition is still + refused by the coordinator, so a watcher cannot steal a live child. + """ + # What: document poll a local catalog safely only in the start_catalog_watcher docstring; why: introspection and maintainers read this exact docstring fragment to understand start catalog watcher behavior without executing it. + # What: document polling keeps the daemon stdlib only and in the start_catalog_watcher docstring; why: introspection and maintainers read this exact docstring fragment to understand start catalog watcher behavior without executing it. + # What: document malformed file is remembered until it in the start_catalog_watcher docstring; why: introspection and maintainers read this exact docstring fragment to understand start catalog watcher behavior without executing it. + # What: document storm while an editor writes it in the start_catalog_watcher docstring; why: introspection and maintainers read this exact docstring fragment to understand start catalog watcher behavior without executing it. + # What: document refused by the coordinator so a in the start_catalog_watcher docstring; why: introspection and maintainers read this exact docstring fragment to understand start catalog watcher behavior without executing it. + # What: preserve the paragraph boundary in the the start_catalog_watcher docstring; why: introspection and maintainers read this paragraph break to understand start catalog watcher behavior without executing it. + # What: gate on watch state before the computed value; why: start_catalog_watcher admits the computed value only for this predicate and excludes the opposite state. + if not watch_state["enabled"]: + # What: return no value from start_catalog_watcher; why: start_catalog_watcher returns no value to callers that depend on its completed result. + return + # What: compute interval from float and catalog watch interval s; why: while not watch stop wait interval later reads interval, so start_catalog_watcher must retain the computed value under that name. + interval = float(catalog_watch_interval_s) + + # What: define watch around the current object state; why: its direct callers call watch for watch and rely on this exact input and result contract. + def watch() -> None: + # What: compute previous from catalog stamp; why: if changed previous later reads previous, so watch must retain the computed value under that name. + previous = catalog_stamp() + # What: iterate across wait and interval and watch stop to perform changed and catalog stamp; why: watch repeats the body only while or for the loop header admits an iteration. + while not watch_stop.wait(interval): + # What: compute changed from catalog stamp; why: if changed previous later reads changed, so watch must retain the computed value under that name. + changed = catalog_stamp() + # What: gate on changed and previous before the computed value; why: watch admits the computed value only for this predicate and excludes the opposite state. + if changed == previous: + # What: apply the continue portion of the enclosing predicate; why: this clause remains in watch\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: compute previous from changed; why: the enclosing return or state update later reads previous, so watch must retain the computed value under that name. + previous = changed + # What: establish the handler boundary for the protected operation; why: watch routes failures to catalog error and routing error while preserving cleanup and success flow. + try: + # What: compute replacement from load and catalog path and model catalog; why: router replace catalog replacement later reads replacement, so watch must retain the computed value under that name. + replacement = ModelCatalog.load(catalog_path) + # What: call router.replace_catalog with replacement; why: watch invokes router.replace_catalog while performing activity store reconfigure; the call advances that operation through its result or side effect. + router.replace_catalog(replacement) + # What: execute activity store reconfigure; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + activity_store.reconfigure( + # What: apply the replacement settings activity max entries portion of the enclosing predicate; why: this clause remains in watch\'s enclosing expression so its grouping and evaluation order stay intact. + replacement.settings.activity_max_entries, + # What: apply the replacement settings capture buffer mb portion of the enclosing predicate; why: this clause remains in watch\'s enclosing expression so its grouping and evaluation order stay intact. + replacement.settings.capture_buffer_mb * 1024 * 1024, + # What: apply the replacement settings activity session headers portion of the enclosing predicate; why: this clause remains in watch\'s enclosing expression so its grouping and evaluation order stay intact. + replacement.settings.activity_session_headers, + # What: complete the activity_store.reconfigure call with activity max entries and capture buffer mb and activity session headers; why: watch groups the supplied clauses as one activity_store.reconfigure call before its value is consumed. + ) + # What: call performance_monitor.reconfigure; why: app needs this line to preserve the surrounding expression or collection structure. + performance_monitor.reconfigure( + # What: apply the replacement settings performance every s portion of the enclosing predicate; why: this clause remains in watch\'s enclosing expression so its grouping and evaluation order stay intact. + replacement.settings.performance_every_s, + # What: apply the replacement settings performance disabled portion of the enclosing predicate; why: this clause remains in watch\'s enclosing expression so its grouping and evaluation order stay intact. + replacement.settings.performance_disabled, + # What: complete the performance_monitor.reconfigure call with performance every s and performance disabled; why: watch groups the supplied clauses as one performance_monitor.reconfigure call before its value is consumed. + ) + # What: handle catalog error by record watch invalid catalog; why: watch converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError: + # What: preserve the exact record watch invalid catalog literal fragment; why: watch passes this fragment verbatim through record_watch("invalid_catalog"), because changing it would alter a protocol payload, serialized fixture, or public message. + record_watch("invalid_catalog") + # What: preserve the exact router event catalog watch rejected code invalid catalog literal fragment; why: watch passes this fragment verbatim through router_event("catalog_watch_rejected", code="invalid_catalog"), because changing it would alter a protocol payload, serialized fixture, or public me. + router_event("catalog_watch_rejected", code="invalid_catalog") + # What: handle routing error by record watch exc code; why: watch converts that failure into this concrete recovery, response, or cleanup behavior. + except RoutingError as exc: + # What: call record_watch with code and exc; why: watch invokes record_watch while performing router event catalog watch rejected code exc code; the call advances that operation through its result or side effect. + record_watch(exc.code) + # What: preserve the exact router event catalog watch rejected code exc code literal fragment; why: watch passes this fragment verbatim through router_event("catalog_watch_rejected", code=exc.code), because changing it would alter a protocol payload, serialized fixture, or public message. + router_event("catalog_watch_rejected", code=exc.code) + # What: select the remaining branch that performs record watch reloaded; why: watch covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: preserve the exact record watch reloaded literal fragment; why: watch passes this fragment verbatim through record_watch("reloaded"), because changing it would alter a protocol payload, serialized fixture, or public message. + record_watch("reloaded") + # What: preserve the exact router event catalog watch reloaded literal fragment; why: watch passes this fragment verbatim through router_event("catalog_watch_reloaded"), because changing it would alter a protocol payload, serialized fixture, or public message. + router_event("catalog_watch_reloaded") + + # What: compute thread from thread and threading and watch and ft daemon catalog watch and true; why: app state catalog watch thread thread later reads thread, so start_catalog_watcher must retain the computed value under that name. + thread = threading.Thread(target=watch, name="ft-daemon-catalog-watch", daemon=True) + # What: compute catalog watch thread from thread; why: the enclosing return or state update later reads catalog watch thread, so start_catalog_watcher must retain the computed value under that name. + app.state.catalog_watch_thread = thread + # What: call thread.start with the declared inputs; why: start_catalog_watcher invokes thread.start while performing the enclosing return; the call advances that operation through its result or side effect. + thread.start() + + # What: call start_catalog_watcher with the declared inputs; why: build_app invokes start_catalog_watcher while performing if watch state enabled; the call advances that operation through its result or side effect. + start_catalog_watcher() + + # What: gate on watch state before on event and set and app and watch stop; why: build_app admits on event and set and app and watch stop only for this predicate and excludes the opposite state. + if watch_state["enabled"]: + + # What: apply app.on_event behavior to _stop_catalog_watcher; why: Python attaches this named decorator's registration or descriptor semantics to _stop_catalog_watcher. + @app.on_event("shutdown") + # What: define _stop_catalog_watcher around the current object state; why: the registered API client call _stop_catalog_watcher for stop catalog watcher and rely on this exact input and result contract. + async def _stop_catalog_watcher() -> None: + # What: call watch_stop.set with the declared inputs; why: _stop_catalog_watcher invokes watch_stop.set while performing the enclosing return; the call advances that operation through its result or side effect. + watch_stop.set() + + # What: define forward_routed around request and model and path and query and body and apply request filters; why: its direct callers call forward_routed for forward routed and rely on this exact input and result contract. + async def forward_routed( + # What: declare the request input for forward_routed; why: forward_routed consumes request during safe route getattr request scope get route path request method, so callers must bind it with the other signature inputs. + request: Request, + # What: declare the model input for forward_routed; why: forward_routed consumes model during profile model, so callers must bind it with the other signature inputs. + model: str, + # What: mark the remaining parameters as keyword-only; why: forward_routed prevents callers from confusing adjacent lifecycle and timing arguments. + *, + # What: declare the path and query input for forward_routed; why: forward_routed consumes path and query during path and query path and query, so callers must bind it with the other signature inputs. + path_and_query: str, + # What: declare the body input for forward_routed; why: forward_routed consumes body during outbound body body, so callers must bind it with the other signature inputs. + body: bytes, + # What: declare the apply request filters input for forward_routed; why: forward_routed consumes apply request filters during if not apply request filters, so callers must bind it with the other signature inputs. + apply_request_filters: bool = False, + # What: complete the enclosing predicate with async def forward routed request request model str path and query str; why: forward_routed groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + """Select a configured model, then stream the engine response unchanged. + + The lease spans the full downstream iterator. If a client disconnects, + Starlette closes that iterator, which closes the upstream socket and + releases admission for the next model swap. + """ + # What: document select a configured model then stream in the forward_routed docstring; why: introspection and maintainers read this exact docstring fragment to understand forward routed behavior without executing it. + # What: document the lease spans the full downstream in the forward_routed docstring; why: introspection and maintainers read this exact docstring fragment to understand forward routed behavior without executing it. + # What: document starlette closes that iterator which closes in the forward_routed docstring; why: introspection and maintainers read this exact docstring fragment to understand forward routed behavior without executing it. + # What: document releases admission for the next model in the forward_routed docstring; why: introspection and maintainers read this exact docstring fragment to understand forward routed behavior without executing it. + # What: preserve the paragraph boundary in the the forward_routed docstring; why: introspection and maintainers read this paragraph break to understand forward routed behavior without executing it. + # What: compute started from monotonic and time; why: yield observed loading frame f done time monotonic later reads started, so forward_routed must retain the computed value under that name. + started = time.monotonic() + # What: compute request id from hex and get and headers and uuid4; why: if not request id isascii or not request id later reads request id, so forward_routed must retain the computed value under that name. + request_id = request.headers.get("x-ft-request-id") or uuid.uuid4().hex + # What: gate on request id and isascii and len before httpexception; why: forward_routed admits httpexception only for this predicate and excludes the opposite state. + if not request_id.isascii() or not request_id or len(request_id) > 128: + # What: raise HTTPException for the caller; why: forward_routed stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=400, detail="X-FT-Request-ID must be 1 to 128 ASCII characters") + # Use FastAPI's registered route template, not the concrete path or + # query. An upstream passthrough tail can itself contain a signed URL, + # opaque bearer-like value, or tenant identifier. + # What: compute safe route from getattr and method and get and request; why: router event request conflict profile model route safe route later reads safe route, so forward_routed must retain the computed value under that name. + safe_route = getattr(request.scope.get("route"), "path", request.method) + # What: compute admission cancellation from event and threading; why: cancellation admission cancellation later reads admission cancellation, so forward_routed must retain the computed value under that name. + admission_cancellation = threading.Event() + # What: compute capture limit from capture item limit and activity store; why: if capture limit and not capture overflow later reads capture limit, so forward_routed must retain the computed value under that name. + capture_limit = activity_store.capture_item_limit + # What: enter the inflight lock managed context before if request id in request reservations; why: forward_routed releases this resource or lock after if request id in request reservations on both success and failure paths. + with inflight_lock: + # What: gate on request id and request reservations before router event and model and safe route; why: forward_routed admits router event and model and safe route only for this predicate and excludes the opposite state. + if request_id in request_reservations: + # What: preserve the exact router event request conflict profile model route safe route literal fragment; why: forward_routed passes this fragment verbatim through router_event("request_conflict", profile=model, route=safe_route), because changing it would alter a protocol payload, serialized fixture. + router_event("request_conflict", profile=model, route=safe_route) + # What: return jsonresponse and 409 and error and message and type from forward_routed; why: forward_routed exposes jsonresponse and 409 and error and message and type so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: forward_routed binds this 409 value to JSONResponse's status code input. + status_code=409, + # What: map the error field as message and type and request and id and is; why: forward_routed carries error through content={"error": { into except exception as exc the lease must. + content={"error": { + # What: apply the message request id is already active portion of the enclosing predicate; why: this clause remains in forward_routed\'s enclosing expression so its grouping and evaluation order stay intact. + "message": "request id is already active", "type": "request_conflict", + # What: complete the enclosing predicate mapping with error; why: forward_routed groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + }}, + # What: complete the JSONResponse call with status code and content; why: forward_routed groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + # What: compute request reservations entry from model and admission cancellation and profile and cancellation and; why: forward_routed consumes request reservations entry during cancelled before connect request reservations request id cancelled, so request reservations entry value receives the computed val. + request_reservations[request_id] = { + # What: map the profile field as model; why: forward_routed carries profile through request reservations entry into cancelled before connect request reservations request id cancelled. + "profile": model, + # What: map the cancellation field as admission cancellation; why: forward_routed carries cancellation through request reservations entry into cancelled before connect request reservations request id cancelled. + "cancellation": admission_cancellation, + # What: map the cancelled field as false; why: forward_routed carries cancelled through request reservations entry into cancelled before connect request reservations request id cancelled. + "cancelled": False, + # What: complete the request_reservations entry mapping with profile and cancellation and cancelled; why: forward_routed groups the supplied clauses as one request_reservations entry mapping before its value is consumed. + } + + # What: define filtered_body around route lease; why: its direct callers call filtered_body for filtered body and rely on this exact input and result contract. + def filtered_body(route_lease) -> bytes: + # What: gate on apply request filters before body; why: filtered_body admits body only for this predicate and excludes the opposite state. + if not apply_request_filters: + # What: return body from filtered_body; why: filtered_body exposes body so its caller can continue with the function\'s computed outcome. + return body + # What: compute profile from profile and route lease; why: profile drop fields later reads profile, so filtered_body must retain the computed value under that name. + profile = route_lease.profile + # What: compute target model from model id and model and route lease; why: requested model target model later reads target model, so filtered_body must retain the computed value under that name. + target_model = route_lease.model_id or model + # What: return filter request body and body and drop fields and set fields from filtered_body; why: filtered_body exposes filter request body and body and drop fields and set fields so its caller can continue with the function\'s computed outcome. + return filter_request_body( + # What: apply the body portion of the enclosing predicate; why: this clause remains in filtered_body\'s enclosing expression so its grouping and evaluation order stay intact. + body, + # What: apply the profile drop fields portion of the enclosing predicate; why: this clause remains in filtered_body\'s enclosing expression so its grouping and evaluation order stay intact. + profile.drop_fields, + # What: apply the profile set fields portion of the enclosing predicate; why: this clause remains in filtered_body\'s enclosing expression so its grouping and evaluation order stay intact. + profile.set_fields, + # What: apply the profile set fields by id portion of the enclosing predicate; why: this clause remains in filtered_body\'s enclosing expression so its grouping and evaluation order stay intact. + profile.set_fields_by_id, + # What: supply requested model to filter_request_body; why: filtered_body binds this target model value to filter_request_body's requested model input. + requested_model=target_model, + # What: supply rewrite model to filter_request_body; why: filtered_body binds this use model name and profile and target model and selector id value to filter_request_body's rewrite model input. + rewrite_model=( + # What: apply the profile use model name portion of the enclosing predicate; why: this clause remains in filtered_body\'s enclosing expression so its grouping and evaluation order stay intact. + profile.use_model_name + # What: apply the or portion of the enclosing predicate; why: this clause remains in filtered_body\'s enclosing expression so its grouping and evaluation order stay intact. + or ( + # What: apply the target model portion of the enclosing predicate; why: this clause remains in filtered_body\'s enclosing expression so its grouping and evaluation order stay intact. + target_model + # What: apply the if route lease selector id is not portion of the enclosing predicate; why: this clause remains in filtered_body\'s enclosing expression so its grouping and evaluation order stay intact. + if route_lease.selector_id is not None + # What: apply the or route lease routing profile id is not portion of the enclosing predicate; why: this clause remains in filtered_body\'s enclosing expression so its grouping and evaluation order stay intact. + or route_lease.routing_profile_id is not None + # What: apply the else portion of the enclosing predicate; why: this clause remains in filtered_body\'s enclosing expression so its grouping and evaluation order stay intact. + else None + # What: complete the filter_request_body call with requested model and rewrite model; why: filtered_body groups the supplied clauses as one filter_request_body call before its value is consumed. + ) + # What: complete the filter_request_body call with requested model and rewrite model; why: filtered_body groups the supplied clauses as one filter_request_body call before its value is consumed. + ), + # What: complete the filter_request_body call with requested model and rewrite model; why: filtered_body groups the supplied clauses as one filter_request_body call before its value is consumed. + ) + + # What: define lease_event_identity around route lease; why: its direct callers call lease_event_identity for lease event identity and rely on this exact input and result contract. + def lease_event_identity(route_lease) -> dict[str, str]: + # What: map the profile field as name and profile and route lease; why: lease_event_identity carries profile through identity into identity selector route lease selector id. + identity = {"profile": route_lease.profile.name} + # What: gate on selector id and route lease before selector id and identity and route lease; why: lease_event_identity admits selector id and identity and route lease only for this predicate and excludes the opposite state. + if route_lease.selector_id is not None: + # What: compute identity entry from selector id and route lease; why: identity target route lease model id later reads identity entry, so lease_event_identity must retain the computed value under that name. + identity["selector"] = route_lease.selector_id + # What: compute identity entry from model id and route lease; why: identity routing profile route lease routing profile id later reads identity entry, so lease_event_identity must retain the computed value under that name. + identity["target"] = route_lease.model_id + # What: gate on routing profile id and route lease before routing profile id and identity and route lease; why: lease_event_identity admits routing profile id and identity and route lease only for this predicate and excludes the opposite state. + if route_lease.routing_profile_id is not None: + # What: compute identity entry from routing profile id and route lease; why: identity pin route lease pin id later reads identity entry, so lease_event_identity must retain the computed value under that name. + identity["routingProfile"] = route_lease.routing_profile_id + # What: compute identity entry from pin id and route lease; why: identity setdefault target route lease model id later reads identity entry, so lease_event_identity must retain the computed value under that name. + identity["pin"] = route_lease.pin_id + # What: preserve the exact identity setdefault target route lease model id literal fragment; why: lease_event_identity passes this fragment verbatim through identity.setdefault("target", route_lease.model_id), because changing it would alter a protocol payload, serialized fixture, or public message. + identity.setdefault("target", route_lease.model_id) + # What: return identity from lease_event_identity; why: lease_event_identity exposes identity so its caller can continue with the function\'s computed outcome. + return identity + + # What: define loading_frame around text; why: its direct callers call loading_frame for loading frame and rely on this exact input and result contract. + def loading_frame(text: str) -> bytes: + # What: map the choices field as text and delta and reasoning content; why: loading_frame carries choices through payload into payload separators ensure ascii false. + payload = {"choices": [{"delta": {"reasoning_content": text}}]} + # What: return encode and dumps and payload and json and utf 8 from loading_frame; why: loading_frame exposes encode and dumps and payload and json and utf 8 so its caller can continue with the function\'s computed outcome. + return b"data: " + json.dumps( + # What: supply separators to operation.encode; why: loading_frame binds this value and value value to operation.encode's separators input. + payload, separators=(",", ":"), ensure_ascii=False + # What: apply the encode utf 8 b n n portion of the enclosing predicate; why: this clause remains in loading_frame\'s enclosing expression so its grouping and evaluation order stay intact. + ).encode("utf-8") + b"\n\n" + + # What: define loading_error around exc; why: its direct callers call loading_error for loading error and rely on this exact input and result contract. + def loading_error(exc: BaseException) -> bytes: + # What: compute error type from code and isinstance and exc and routing error and upstream unavailable; why: payload dict str any error message later reads error type, so loading_error must retain the computed value under that name. + error_type = exc.code if isinstance(exc, RoutingError) else "upstream_unavailable" + # What: map the error field as error type and str and exc and message and type; why: loading_error carries error through payload into payload recovery exc recovery. + payload: dict[str, Any] = {"error": {"message": str(exc), "type": error_type}} + # What: gate on isinstance and exc and routing error and recovery before recovery and payload and exc; why: loading_error admits recovery and payload and exc only for this predicate and excludes the opposite state. + if isinstance(exc, RoutingError) and exc.recovery is not None: + # What: compute payload entry from recovery and exc; why: json dumps payload separators encode utf 8 later reads payload entry, so loading_error must retain the computed value under that name. + payload["recovery"] = exc.recovery + # What: return encode and dumps and payload and json and utf 8 from loading_error; why: loading_error exposes encode and dumps and payload and json and utf 8 so its caller can continue with the function\'s computed outcome. + return ( + # What: apply the b data portion of the enclosing predicate; why: this clause remains in loading_error\'s enclosing expression so its grouping and evaluation order stay intact. + b"data: " + # What: supply separators to operation.encode; why: loading_error binds this value and value value to operation.encode's separators input. + + json.dumps(payload, separators=(",", ":")).encode("utf-8") + # What: apply the b n ndata done n n portion of the enclosing predicate; why: this clause remains in loading_error\'s enclosing expression so its grouping and evaluation order stay intact. + + b"\n\ndata: [DONE]\n\n" + # What: complete the loading_error signature with exc; why: loading_error groups the supplied clauses as one loading_error signature before its value is consumed. + ) + + # What: map the done field as false; why: forward_routed carries done through abandon state into if abandon state done. + abandon_state = {"done": False} + + # What: define abandon_loading_acquisition around acquisition and record cancellation; why: its direct callers call abandon_loading_acquisition for abandon loading acquisition and rely on this exact input and result contract. + def abandon_loading_acquisition( + # What: declare the acquisition input for abandon_loading_acquisition; why: abandon_loading_acquisition consumes acquisition during acquisition add done callback release if admitted, so callers must bind it with the other signature inputs. + acquisition: asyncio.Task, *, record_cancellation: bool = True + # What: complete the enclosing predicate with group delimiter; why: abandon_loading_acquisition groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) -> None: + """Wake an abandoned admission and release any lease it later returns.""" + # What: document wake an abandoned admission and release in the abandon_loading_acquisition docstring; why: introspection and maintainers read this exact docstring fragment to understand abandon loading acquisition behavior without executing it. + # What: gate on abandon state before the computed value; why: abandon_loading_acquisition admits the computed value only for this predicate and excludes the opposite state. + if abandon_state["done"]: + # What: return no value from abandon_loading_acquisition; why: abandon_loading_acquisition returns no value to callers that depend on its completed result. + return + # What: compute abandon state entry from true; why: the enclosing return or state update later reads abandon state entry, so abandon_loading_acquisition must retain the computed value under that name. + abandon_state["done"] = True + # What: call router.cancel_acquire with admission cancellation; why: abandon_loading_acquisition invokes router.cancel_acquire while performing if record cancellation; the call advances that operation through its result or side effect. + router.cancel_acquire(admission_cancellation) + # What: gate on record cancellation before record cancellation and router; why: abandon_loading_acquisition admits record cancellation and router only for this predicate and excludes the opposite state. + if record_cancellation: + # What: call router.record_cancellation with the declared inputs; why: abandon_loading_acquisition invokes router.record_cancellation while performing def release if admitted done asyncio task; the call advances that operation through its result or side effect. + router.record_cancellation() + + # What: define release_if_admitted around done; why: its direct callers call release_if_admitted for release if admitted and rely on this exact input and result contract. + def release_if_admitted(done: asyncio.Task) -> None: + # What: establish the handler boundary for the protected operation; why: release_if_admitted routes failures to base exception while preserving cleanup and success flow. + try: + # What: compute admitted from result and done; why: admitted release later reads admitted, so release_if_admitted must retain the computed value under that name. + admitted = done.result() + # What: handle base exception by return; why: release_if_admitted converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException: + # What: return no value from release_if_admitted; why: release_if_admitted returns no value to callers that depend on its completed result. + return + # What: call admitted.release with the declared inputs; why: release_if_admitted invokes admitted.release while performing the enclosing return; the call advances that operation through its result or side effect. + admitted.release() + + # What: call acquisition.add_done_callback with release if admitted; why: abandon_loading_acquisition invokes acquisition.add_done_callback while performing the enclosing return; the call advances that operation through its result or side effect. + acquisition.add_done_callback(release_if_admitted) + + # What: define loading_stream around acquisition; why: its direct callers call loading_stream for loading stream and rely on this exact input and result contract. + async def loading_stream(acquisition: asyncio.Task): + """Bridge one admitted cold request into loading SSE, then its real response.""" + # What: document bridge one admitted cold request into in the loading_stream docstring; why: introspection and maintainers read this exact docstring fragment to understand loading stream behavior without executing it. + # What: compute lease from the named fixture input; why: lease acquisition result later reads lease, so loading_stream must retain the computed value under that name. + lease = None + # What: compute upstream from the named fixture input; why: request cancelled before upstream connection later reads upstream, so loading_stream must retain the computed value under that name. + upstream = None + # What: compute first byte at from the named fixture input; why: if first byte at is later reads first byte at, so loading_stream must retain the computed value under that name. + first_byte_at = None + # What: compute byte count from 0; why: byte count len chunk later reads byte count, so loading_stream must retain the computed value under that name. + byte_count = 0 + # What: compute cancelled from false; why: cancelled before connect request reservations request id cancelled later reads cancelled, so loading_stream must retain the computed value under that name. + cancelled = False + # What: compute cancellation recorded from false; why: cancellation recorded later reads cancellation recorded, so loading_stream must retain the computed value under that name. + cancellation_recorded = False + # What: compute last position from the named fixture input; why: last position initial position later reads last position, so loading_stream must retain the computed value under that name. + last_position = None + # What: compute outbound body from body; why: outbound body filtered body lease later reads outbound body, so loading_stream must retain the computed value under that name. + outbound_body = body + # What: compute captured response from bytearray; why: if len captured response len chunk capture limit later reads captured response, so loading_stream must retain the computed value under that name. + captured_response = bytearray() + # What: compute capture overflow from false; why: nonlocal capture overflow later reads capture overflow, so loading_stream must retain the computed value under that name. + capture_overflow = False + + # What: define observed around chunk; why: its direct callers call observed for observed and rely on this exact input and result contract. + def observed(chunk: bytes) -> bytes: + # What: apply the nonlocal capture overflow portion of the enclosing predicate; why: this clause remains in observed\'s enclosing expression so its grouping and evaluation order stay intact. + nonlocal capture_overflow + # What: gate on capture limit and capture overflow before capture limit and capture overflow and extend and chunk and clear; why: observed admits capture limit and capture overflow and extend and chunk and clear only for this predicate and excludes the opposite state. + if capture_limit and not capture_overflow: + # What: gate on capture limit and len and captured response and chunk before extend and chunk and captured response; why: observed admits extend and chunk and captured response only for this predicate and excludes the opposite state. + if len(captured_response) + len(chunk) <= capture_limit: + # What: call captured_response.extend with chunk; why: observed invokes captured_response.extend while performing else; the call advances that operation through its result or side effect. + captured_response.extend(chunk) + # What: select the remaining branch that performs capture overflow; why: observed covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: compute capture overflow from true; why: the enclosing return or state update later reads capture overflow, so observed must retain the computed value under that name. + capture_overflow = True + # What: call captured_response.clear with the declared inputs; why: observed invokes captured_response.clear while performing return chunk; the call advances that operation through its result or side effect. + captured_response.clear() + # What: return chunk from observed; why: observed exposes chunk so its caller can continue with the function\'s computed outcome. + return chunk + # What: establish the handler boundary for the protected operation; why: loading_stream routes failures to cancelled error and asyncio and exception while preserving cleanup and success flow. + try: + # What: preserve the exact yield observed loading frame n literal fragment; why: loading_stream passes this fragment verbatim through yield observed(loading_frame("━━━━━\n")), because changing it would alter a protocol payload, serialized fixture, or public message. + yield observed(loading_frame("━━━━━\n")) + # What: embed the exact yield observed loading frame f freetoken swap loading router-interface fragment; why: the router UI consumer receives this fragment verbatim through yield observed(loading_frame(f"freetoken-swap loading model: {model}\n"), preserving browser markup, style, or script behavior. + yield observed(loading_frame(f"freetoken-swap loading model: {model}\n")) + # What: compute initial position from get and reservation state and queue position; why: if isinstance initial position int later reads initial position, so loading_stream must retain the computed value under that name. + initial_position = reservation_state.get("queuePosition") + # What: gate on isinstance and initial position and int before last position and initial position; why: loading_stream admits last position and initial position only for this predicate and excludes the opposite state. + if isinstance(initial_position, int): + # What: compute last position from initial position; why: if position is not and position later reads last position, so loading_stream must retain the computed value under that name. + last_position = initial_position + # What: execute yield observed loading frame f nQueue position initial position; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + yield observed(loading_frame(f"\nQueue position: #{initial_position} ")) + # What: iterate across done and acquisition to perform position and queue position and admission cancellation and router; why: loading_stream repeats the body only while or for the loop header admits an iteration. + while not acquisition.done(): + # What: compute position from queue position and admission cancellation and router; why: if position is not and position later reads position, so loading_stream must retain the computed value under that name. + position = router.queue_position(admission_cancellation) + # What: gate on position and last position before last position and position; why: loading_stream admits last position and position only for this predicate and excludes the opposite state. + if position is not None and position != last_position: + # What: compute last position from position; why: the enclosing return or state update later reads last position, so loading_stream must retain the computed value under that name. + last_position = position + # What: execute yield observed loading frame f nQueue position position; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + yield observed(loading_frame(f"\nQueue position: #{position} ")) + # What: compute done and from wait and asyncio and acquisition and 0 75; why: if acquisition in done later reads done and, so loading_stream must retain the computed value under that name. + done, _ = await asyncio.wait({acquisition}, timeout=0.75) + # What: gate on acquisition and done before lease and result and acquisition; why: loading_stream admits lease and result and acquisition only for this predicate and excludes the opposite state. + if acquisition in done: + # What: compute lease from result and acquisition; why: if lease is later reads lease, so loading_stream must retain the computed value under that name. + lease = acquisition.result() + # What: select the remaining branch that performs yield observed loading frame; why: loading_stream covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: preserve the exact yield observed loading frame literal fragment; why: loading_stream passes this fragment verbatim through yield observed(loading_frame(".")), because changing it would alter a protocol payload, serialized fixture, or public message. + yield observed(loading_frame(".")) + # What: gate on lease before lease and result and acquisition; why: loading_stream admits lease and result and acquisition only for this predicate and excludes the opposite state. + if lease is None: + # What: compute lease from result and acquisition; why: outbound body filtered body lease later reads lease, so loading_stream must retain the computed value under that name. + lease = acquisition.result() + + # What: preserve the exact yield observed loading frame n literal fragment; why: loading_stream passes this fragment verbatim through yield observed(loading_frame("\n")), because changing it would alter a protocol payload, serialized fixture, or public message. + yield observed(loading_frame("\n")) + # What: embed the exact yield observed loading frame f done time monotonic router-interface fragment; why: the router UI consumer receives this fragment verbatim through yield observed(loading_frame(f"Done! ({time.monotonic() - started:.2f}s), preserving browser markup, style, or script behavior. + yield observed(loading_frame(f"Done! ({time.monotonic() - started:.2f}s)\n")) + # What: preserve the exact yield observed loading frame n literal fragment; why: loading_stream passes this fragment verbatim through yield observed(loading_frame("━━━━━\n")), because changing it would alter a protocol payload, serialized fixture, or public message. + yield observed(loading_frame("━━━━━\n")) + # What: preserve the exact yield observed loading frame n literal fragment; why: loading_stream passes this fragment verbatim through yield observed(loading_frame(" \n")), because changing it would alter a protocol payload, serialized fixture, or public message. + yield observed(loading_frame(" \n")) + + # What: enter the inflight lock managed context before cancelled before connect request reservations request id cancelled; why: loading_stream releases this resource or lock after cancelled before connect request reservations request id cancelled on both success and failure paths. + with inflight_lock: + # What: compute cancelled before connect from request reservations and request id and cancelled; why: if cancelled before connect later reads cancelled before connect, so loading_stream must retain the computed value under that name. + cancelled_before_connect = request_reservations[request_id]["cancelled"] + # What: gate on cancelled before connect before routing error; why: loading_stream admits routing error only for this predicate and excludes the opposite state. + if cancelled_before_connect: + # What: raise RoutingError for the caller; why: loading_stream stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: apply the request cancelled portion of the enclosing predicate; why: this clause remains in loading_stream\'s enclosing expression so its grouping and evaluation order stay intact. + "request_cancelled", + # What: apply the request cancelled before upstream connection portion of the enclosing predicate; why: this clause remains in loading_stream\'s enclosing expression so its grouping and evaluation order stay intact. + "request cancelled before upstream connection", + # What: supply status code to RoutingError; why: loading_stream binds this 409 value to RoutingError's status code input. + status_code=409, + # What: complete the RoutingError call with status code; why: loading_stream groups the supplied clauses as one RoutingError call before its value is consumed. + ) + + # What: compute outbound body from filtered body and lease; why: body outbound body later reads outbound body, so loading_stream must retain the computed value under that name. + outbound_body = filtered_body(lease) + + # What: compute upstream from connect upstream and port and proxy base url and path and query; why: upstream upstream later reads upstream, so loading_stream must retain the computed value under that name. + upstream = await connect_upstream( + # What: supply port to connect_upstream; why: loading_stream binds this port and lease value to connect_upstream's port input. + port=lease.port, + # What: supply base url to connect_upstream; why: loading_stream binds this proxy base url and lease value to connect_upstream's base url input. + base_url=lease.proxy_base_url, + # What: supply path and query to connect_upstream; why: loading_stream binds this path and query value to connect_upstream's path and query input. + path_and_query=path_and_query, + # What: supply headers to dict; why: loading_stream binds this dict and headers and request value to dict's headers input. + headers=dict(request.headers), + # What: supply body to connect_upstream; why: loading_stream binds this outbound body value to connect_upstream's body input. + body=outbound_body, + # What: supply method to connect_upstream; why: loading_stream binds this method and request value to connect_upstream's method input. + method=request.method, + # What: supply timeout s to connect_upstream; why: loading_stream binds this upstream timeout s and profile and router and lease value to connect_upstream's timeout s input. + timeout_s=lease.profile.upstream_timeout_s or router.upstream_timeout_s, + # What: complete the connect_upstream call with port and base url and path and query and headers and body; why: loading_stream groups the supplied clauses as one connect_upstream call before its value is consumed. + ) + # What: enter the inflight lock managed context before cancelled while connecting request reservations request id cancelled; why: loading_stream releases this resource or lock after cancelled while connecting request reservations request id cancelled on both success and failure paths. + with inflight_lock: + # What: compute cancelled while connecting from request reservations and request id and cancelled; why: if not cancelled while connecting later reads cancelled while connecting, so loading_stream must retain the computed value under that name. + cancelled_while_connecting = request_reservations[request_id]["cancelled"] + # What: gate on cancelled while connecting before inflight and request id and name and upstream and profile; why: loading_stream admits inflight and request id and name and upstream and profile only for this predicate and excludes the opposite state. + if not cancelled_while_connecting: + # What: compute inflight entry from name and upstream and profile and lease and profile; why: item inflight get request id later reads inflight entry, so loading_stream must retain the computed value under that name. + inflight[request_id] = { + # What: map the profile field as name and profile and lease; why: loading_stream carries profile through inflight entry into item inflight get request id. + "profile": lease.profile.name, + # What: map the upstream field as upstream; why: loading_stream carries upstream through inflight entry into item inflight get request id. + "upstream": upstream, + # What: map the cancelled field as false; why: loading_stream carries cancelled through inflight entry into item inflight get request id. + "cancelled": False, + # What: complete the inflight entry mapping with profile and upstream and cancelled; why: loading_stream groups the supplied clauses as one inflight entry mapping before its value is consumed. + } + # What: gate on cancelled while connecting before routing error; why: loading_stream admits routing error only for this predicate and excludes the opposite state. + if cancelled_while_connecting: + # What: raise RoutingError for the caller; why: loading_stream stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: apply the request cancelled portion of the enclosing predicate; why: this clause remains in loading_stream\'s enclosing expression so its grouping and evaluation order stay intact. + "request_cancelled", + # What: apply the request cancelled while opening upstream connection portion of the enclosing predicate; why: this clause remains in loading_stream\'s enclosing expression so its grouping and evaluation order stay intact. + "request cancelled while opening upstream connection", + # What: supply status code to RoutingError; why: loading_stream binds this 409 value to RoutingError's status code input. + status_code=409, + # What: complete the RoutingError call with status code; why: loading_stream groups the supplied clauses as one RoutingError call before its value is consumed. + ) + + # What: call router_event with admitted; why: loading_stream invokes router_event while performing admitted lease event identity lease route safe route; the call advances that operation through its result or side effect. + router_event( + # What: preserve the exact admitted lease event identity lease route safe route literal fragment; why: loading_stream passes this fragment verbatim through "admitted", **lease_event_identity(lease), route=safe_route, because changing it would alter a protocol payload, serialized fixture, or public. + "admitted", **lease_event_identity(lease), route=safe_route + # What: complete the router_event call with route; why: loading_stream groups the supplied clauses as one router_event call before its value is consumed. + ) + # What: compute iterator from iter and chunks and upstream; why: return next iterator later reads iterator, so loading_stream must retain the computed value under that name. + iterator = iter(upstream.chunks()) + + # What: define next_chunk around the current object state; why: its direct callers call next_chunk for next chunk and rely on this exact input and result contract. + def next_chunk(): + # What: establish the handler boundary for the protected operation; why: next_chunk routes failures to stop iteration while preserving cleanup and success flow. + try: + # What: return next and iterator and true from next_chunk; why: next_chunk exposes next and iterator and true so its caller can continue with the function\'s computed outcome. + return True, next(iterator) + # What: handle stop iteration by return false b; why: next_chunk converts that failure into this concrete recovery, response, or cleanup behavior. + except StopIteration: + # What: return false from next_chunk; why: next_chunk exposes false so its caller can continue with the function\'s computed outcome. + return False, b"" + + # What: compute loop from get running loop and asyncio; why: has chunk chunk await loop run in executor proxy pool next chunk later reads loop, so loading_stream must retain the computed value under that name. + loop = asyncio.get_running_loop() + # What: iterate across the computed value to perform has chunk and chunk and run in executor and proxy pool and next chunk; why: loading_stream repeats the body only while or for the loop header admits an iteration. + while True: + # What: compute has chunk and chunk from run in executor and proxy pool and next chunk and loop; why: if not has chunk later reads has chunk and chunk, so loading_stream must retain the computed value under that name. + has_chunk, chunk = await loop.run_in_executor(proxy_pool, next_chunk) + # What: gate on has chunk before the computed value; why: loading_stream admits the computed value only for this predicate and excludes the opposite state. + if not has_chunk: + # What: apply the break portion of the enclosing predicate; why: this clause remains in loading_stream\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: gate on first byte at before first byte at and monotonic and time; why: loading_stream admits first byte at and monotonic and time only for this predicate and excludes the opposite state. + if first_byte_at is None: + # What: compute first byte at from monotonic and time; why: ttft s first byte at started if first byte at is later reads first byte at, so loading_stream must retain the computed value under that name. + first_byte_at = time.monotonic() + # What: compute byte count from len and chunk; why: response bytes byte count later reads byte count, so loading_stream must retain the computed value under that name. + byte_count += len(chunk) + # What: call observed with chunk; why: loading_stream invokes observed while performing except asyncio cancelled error; the call advances that operation through its result or side effect. + yield observed(chunk) + # What: handle cancelled error and asyncio by cancelled true; why: loading_stream converts that failure into this concrete recovery, response, or cleanup behavior. + except asyncio.CancelledError: + # What: compute cancelled from true; why: cancelled later reads cancelled, so loading_stream must retain the computed value under that name. + cancelled = True + # What: call abandon_loading_acquisition with acquisition; why: loading_stream invokes abandon_loading_acquisition while performing cancellation recorded; the call advances that operation through its result or side effect. + abandon_loading_acquisition(acquisition) + # What: compute cancellation recorded from true; why: acquisition record cancellation not cancellation recorded later reads cancellation recorded, so loading_stream must retain the computed value under that name. + cancellation_recorded = True + # What: preserve the exact router event request cancelled profile model route safe route literal fragment; why: loading_stream passes this fragment verbatim through router_event("request_cancelled", profile=model, route=safe_route), because changing it would alter a protocol payload, serialized fixture. + router_event("request_cancelled", profile=model, route=safe_route) + # What: re-propagate the active failure to the caller; why: loading_stream stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: handle exception by if isinstance exc routing error; why: loading_stream converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: + # What: gate on isinstance and exc and routing error before router event and model and safe route and code and exc; why: loading_stream admits router event and model and safe route and code and exc only for this predicate and excludes the opposite state. + if isinstance(exc, RoutingError): + # What: preserve the exact router event admission failed profile model route safe route literal fragment; why: loading_stream passes this fragment verbatim through router_event("admission_failed", profile=model, route=safe_route, code=e, because changing it would alter a protocol payload, serialize. + router_event("admission_failed", profile=model, route=safe_route, code=exc.code) + # What: select the remaining branch that performs router event upstream connect failed profile model route safe route; why: loading_stream covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: preserve the exact router event upstream connect failed profile model route safe route literal fragment; why: loading_stream passes this fragment verbatim through router_event("upstream_connect_failed", profile=model, route=safe_route), because changing it would alter a protocol payload, se. + router_event("upstream_connect_failed", profile=model, route=safe_route) + # What: call observed with loading error and exc; why: loading_stream invokes observed while performing finally; the call advances that operation through its result or side effect. + yield observed(loading_error(exc)) + # What: run ended time monotonic on every exit path; why: loading_stream performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: compute ended from monotonic and time; why: duration s ended started later reads ended, so loading_stream must retain the computed value under that name. + ended = time.monotonic() + # Starlette may finalize an async response iterator with + # ``GeneratorExit`` rather than injecting ``CancelledError``. + # A downstream that disappears must still synchronously wake + # and cancel any queued ownership. + # What: gate on done and acquisition before cancelled; why: loading_stream admits cancelled only for this predicate and excludes the opposite state. + if not acquisition.done(): + # What: compute cancelled from true; why: cancelled later reads cancelled, so loading_stream must retain the computed value under that name. + cancelled = True + # What: call abandon_loading_acquisition with acquisition; why: loading_stream invokes abandon_loading_acquisition while performing acquisition record cancellation not cancellation recorded; the call advances that operation through its result or side effect. + abandon_loading_acquisition( + # What: supply record cancellation to abandon_loading_acquisition; why: loading_stream binds this cancellation recorded value to abandon_loading_acquisition's record cancellation input. + acquisition, record_cancellation=not cancellation_recorded + # What: complete the abandon_loading_acquisition call with record cancellation; why: loading_stream groups the supplied clauses as one abandon_loading_acquisition call before its value is consumed. + ) + # What: gate on lease before lease and base exception and cancelled and result and cancellation recorded; why: loading_stream admits lease and base exception and cancelled and result and cancellation recorded only for this predicate and excludes the opposite state. + elif lease is None: + # What: establish the handler boundary for the protected operation; why: loading_stream routes failures to base exception while preserving cleanup and success flow. + try: + # What: compute lease from result and acquisition; why: if lease is not later reads lease, so loading_stream must retain the computed value under that name. + lease = acquisition.result() + # What: handle base exception by pass; why: loading_stream converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException: + # What: ignore the anticipated exception handled by this branch; why: next_chunk continues its retry or cleanup path instead of re-raising that transient failure. + pass + # What: select the remaining branch that performs cancelled; why: loading_stream covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: compute cancelled from true; why: cancelled later reads cancelled, so loading_stream must retain the computed value under that name. + cancelled = True + # What: gate on cancellation recorded before record cancellation and router; why: loading_stream admits record cancellation and router only for this predicate and excludes the opposite state. + if not cancellation_recorded: + # What: call router.record_cancellation with the declared inputs; why: loading_stream invokes router.record_cancellation while performing if upstream is not; the call advances that operation through its result or side effect. + router.record_cancellation() + # What: gate on upstream before close and upstream; why: loading_stream admits close and upstream only for this predicate and excludes the opposite state. + if upstream is not None: + # What: call upstream.close with the declared inputs; why: loading_stream invokes upstream.close while performing with inflight lock; the call advances that operation through its result or side effect. + upstream.close() + # What: enter the inflight lock managed context before reservation request reservations get request id; why: loading_stream releases this resource or lock after reservation request reservations get request id on both success and failure paths. + with inflight_lock: + # What: compute reservation from get and request id and request reservations; why: or bool reservation get cancelled later reads reservation, so loading_stream must retain the computed value under that name. + reservation = request_reservations.get(request_id, {}) + # What: compute item from get and request id and inflight; why: or bool item get cancelled later reads item, so loading_stream must retain the computed value under that name. + item = inflight.get(request_id, {}) + # What: compute cancelled from cancelled and bool and get and reservation; why: cancelled later reads cancelled, so loading_stream must retain the computed value under that name. + cancelled = ( + # What: apply the cancelled portion of cancelled; why: loading_stream uses this clause to evaluate cancelled as one grouped value. + cancelled + # What: call bool with get and reservation and cancelled; why: loading_stream invokes bool while performing or bool item get cancelled; the call advances that operation through its result or side effect. + or bool(reservation.get("cancelled")) + # What: call bool with get and item and cancelled; why: loading_stream consumes the bool return value while evaluating or bool(item.get("cancelled")). + or bool(item.get("cancelled")) + # What: complete the cancelled expression with cancelled cancelled or bool reservation get cancelled or bool item get; why: loading_stream groups the supplied clauses as one cancelled expression before its value is consumed. + ) + # What: gate on upstream and get and item before pop and request id and inflight; why: loading_stream admits pop and request id and inflight only for this predicate and excludes the opposite state. + if upstream is not None and item.get("upstream") is upstream: + # What: call inflight.pop with request id and the named fixture input; why: loading_stream invokes inflight.pop while performing request reservations pop request id; the call advances that operation through its result or side effect. + inflight.pop(request_id, None) + # What: call request_reservations.pop with request id and the named fixture input; why: loading_stream invokes request_reservations.pop while performing if lease is not; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: gate on lease before ttft s and first byte at and started; why: loading_stream admits ttft s and first byte at and started only for this predicate and excludes the opposite state. + if lease is not None: + # What: compute ttft s from first byte at and started; why: ttft s ttft s later reads ttft s, so loading_stream must retain the computed value under that name. + ttft_s = (first_byte_at - started) if first_byte_at is not None else None + # What: call router.record_stream with the declared inputs; why: loading_stream invokes router.record_stream while performing ttft s ttft s; the call advances that operation through its result or side effect. + router.record_stream( + # What: supply ttft s to router.record_stream; why: loading_stream binds this ttft s value to router.record_stream's ttft s input. + ttft_s=ttft_s, + # What: supply duration s to router.record_stream; why: loading_stream binds this ended and started value to router.record_stream's duration s input. + duration_s=ended - started, + # What: supply response bytes to router.record_stream; why: loading_stream binds this byte count value to router.record_stream's response bytes input. + response_bytes=byte_count, + # What: supply completed to router.record_stream; why: loading_stream binds this cancelled value to router.record_stream's completed input. + completed=not cancelled, + # What: complete the router.record_stream call with ttft s and duration s and response bytes and completed; why: loading_stream groups the supplied clauses as one router.record_stream call before its value is consumed. + ) + # What: call lease.release with the declared inputs; why: loading_stream invokes lease.release while performing router event; the call advances that operation through its result or side effect. + lease.release() + # What: call router_event with request finished; why: loading_stream invokes router_event while performing request finished; the call advances that operation through its result or side effect. + router_event( + # What: preserve the exact request finished literal fragment; why: loading_stream passes this fragment verbatim through "request_finished", because changing it would alter a protocol payload, serialized fixture, or public message. + "request_finished", + # What: supply expanded arguments to lease_event_identity; why: loading_stream binds this lease event identity and lease value to lease_event_identity's expanded input. + **lease_event_identity(lease), + # What: supply route to router_event; why: loading_stream binds this safe route value to router_event's route input. + route=safe_route, + # What: supply status to router_event; why: loading_stream binds this status and upstream and 200 value to router_event's status input. + status=upstream.status if upstream is not None else 200, + # What: supply cancelled to router_event; why: loading_stream binds this cancelled value to router_event's cancelled input. + cancelled=cancelled, + # What: supply response bytes to router_event; why: loading_stream binds this byte count value to router_event's response bytes input. + responseBytes=byte_count, + # What: complete the router_event call with route and status and cancelled and response bytes; why: loading_stream groups the supplied clauses as one router_event call before its value is consumed. + ) + # What: call asyncio.get_running_loop with the declared inputs; why: loading_stream invokes asyncio.get_running_loop while performing proxy pool; the call advances that operation through its result or side effect. + await asyncio.get_running_loop().run_in_executor( + # What: apply the proxy pool portion of the enclosing predicate; why: this clause remains in loading_stream\'s enclosing expression so its grouping and evaluation order stay intact. + proxy_pool, + # What: call functools.partial with record and activity store; why: loading_stream invokes functools.partial while performing activity store record; the call advances that operation through its result or side effect. + functools.partial( + # What: apply the activity store record portion of the enclosing predicate; why: this clause remains in loading_stream\'s enclosing expression so its grouping and evaluation order stay intact. + activity_store.record, + # What: supply model to functools.partial; why: loading_stream binds this name and profile and lease value to functools.partial's model input. + model=lease.profile.name, + # What: supply route to functools.partial; why: loading_stream binds this safe route value to functools.partial's route input. + route=safe_route, + # What: supply method to functools.partial; why: loading_stream binds this method and request value to functools.partial's method input. + method=request.method, + # What: supply status to functools.partial; why: loading_stream binds this status and upstream and 200 value to functools.partial's status input. + status=upstream.status if upstream is not None else 200, + # What: supply started to functools.partial; why: loading_stream binds this started value to functools.partial's started input. + started=started, + # What: supply ttft s to functools.partial; why: loading_stream binds this ttft s value to functools.partial's ttft s input. + ttft_s=ttft_s, + # What: supply response bytes to functools.partial; why: loading_stream binds this byte count value to functools.partial's response bytes input. + response_bytes=byte_count, + # What: supply cancelled to functools.partial; why: loading_stream binds this cancelled value to functools.partial's cancelled input. + cancelled=cancelled, + # What: supply request headers to dict; why: loading_stream binds this dict and headers and request value to dict's request headers input. + request_headers=dict(request.headers), + # What: supply request body to functools.partial; why: loading_stream binds this outbound body value to functools.partial's request body input. + request_body=outbound_body, + # What: supply response headers to functools.partial; why: loading_stream binds this headers and upstream value to functools.partial's response headers input. + response_headers=upstream.headers if upstream is not None else {}, + # What: supply response body to functools.partial; why: loading_stream binds this capture overflow and bytes and captured response and upstream value to functools.partial's response body input. + response_body=( + # What: apply the if upstream is or capture overflow portion of the enclosing predicate; why: this clause remains in loading_stream\'s enclosing expression so its grouping and evaluation order stay intact. + None if upstream is None or capture_overflow + # What: call bytes with captured response; why: loading_stream consumes the bytes return value while evaluating else bytes(captured_response). + else bytes(captured_response) + # What: complete the functools.partial call with model and route and method and status and started; why: loading_stream groups the supplied clauses as one functools.partial call before its value is consumed. + ), + # What: complete the functools.partial call with model and route and method and status and started; why: loading_stream groups the supplied clauses as one functools.partial call before its value is consumed. + ), + # What: complete the operation.run_in_executor call with proxy pool and partial; why: loading_stream groups the supplied clauses as one operation.run_in_executor call before its value is consumed. + ) + + # What: compute loading eligible from false; why: loading eligible isinstance request doc dict and request doc get later reads loading eligible, so forward_routed must retain the computed value under that name. + loading_eligible = False + # What: gate on path and url and request before request doc and loads and body and unicode decode error and jsondecode error; why: forward_routed admits request doc and loads and body and unicode decode error and jsondecode error only for this predicate and excludes the opposite state. + if request.url.path == "/v1/chat/completions": + # What: establish the handler boundary for the protected operation; why: forward_routed routes failures to unicode decode error and jsondecode error and json while preserving cleanup and success flow. + try: + # What: compute request doc from loads and body and json; why: request doc later reads request doc, so forward_routed must retain the computed value under that name. + request_doc = json.loads(body) + # What: handle unicode decode error and jsondecode error and json by request doc; why: forward_routed converts that failure into this concrete recovery, response, or cleanup behavior. + except (UnicodeDecodeError, json.JSONDecodeError): + # What: compute request doc from the named fixture input; why: loading eligible isinstance request doc dict and request doc get later reads request doc, so forward_routed must retain the computed value under that name. + request_doc = None + # What: compute loading eligible from isinstance and request doc and dict and get and true; why: if loading eligible later reads loading eligible, so forward_routed must retain the computed value under that name. + loading_eligible = isinstance(request_doc, dict) and request_doc.get("stream") is True + + # What: gate on loading eligible before loop and get running loop and asyncio; why: forward_routed admits loop and get running loop and asyncio only for this predicate and excludes the opposite state. + if loading_eligible: + # What: compute loop from get running loop and asyncio; why: loop call soon threadsafe reserved set later reads loop, so forward_routed must retain the computed value under that name. + loop = asyncio.get_running_loop() + # What: compute reserved from event and asyncio; why: loop call soon threadsafe reserved set later reads reserved, so forward_routed must retain the computed value under that name. + reserved = asyncio.Event() + # What: initialize reservation state as an empty runtime accumulator; why: forward_routed appends or maps entries into it during reservation state loading required loading required before consuming the aggregate. + reservation_state: dict[str, Any] = {} + + # What: define on_reserved around loading required and queue position; why: its direct callers call on_reserved for on reserved and rely on this exact input and result contract. + def on_reserved(loading_required: bool, queue_position: int) -> None: + # What: compute reservation state entry from loading required; why: reservation state queue position queue position later reads reservation state entry, so on_reserved must retain the computed value under that name. + reservation_state["loadingRequired"] = loading_required + # What: compute reservation state entry from queue position; why: the enclosing return or state update later reads reservation state entry, so on_reserved must retain the computed value under that name. + reservation_state["queuePosition"] = queue_position + # What: call loop.call_soon_threadsafe with set and reserved; why: on_reserved invokes loop.call_soon_threadsafe while performing the enclosing return; the call advances that operation through its result or side effect. + loop.call_soon_threadsafe(reserved.set) + + # What: compute acquisition from create task and asyncio and acquire route and model; why: acquisition reservation wait return when asyncio first completed later reads acquisition, so forward_routed must retain the computed value under that name. + acquisition = asyncio.create_task( + # What: call acquire_route with model and admission cancellation and on reserved; why: forward_routed invokes acquire_route while performing model; the call advances that operation through its result or side effect. + acquire_route( + # What: apply the model portion of acquisition; why: forward_routed uses this clause to evaluate acquisition as one grouped value. + model, + # What: apply the admission cancellation portion of acquisition; why: forward_routed uses this clause to evaluate acquisition as one grouped value. + admission_cancellation, + # What: apply the on reserved portion of acquisition; why: forward_routed uses this clause to evaluate acquisition as one grouped value. + on_reserved, + # What: supply apply loading policy to acquire_route; why: forward_routed binds this true value to acquire_route's apply loading policy input. + apply_loading_policy=True, + # What: complete the acquire_route call with apply loading policy; why: forward_routed groups the supplied clauses as one acquire_route call before its value is consumed. + ) + # What: complete the asyncio.create_task call with acquire route; why: forward_routed groups the supplied clauses as one asyncio.create_task call before its value is consumed. + ) + # What: compute reservation wait from create task and asyncio and wait and reserved; why: acquisition reservation wait return when asyncio first completed later reads reservation wait, so forward_routed must retain the computed value under that name. + reservation_wait = asyncio.create_task(reserved.wait()) + # What: establish the handler boundary for the protected operation; why: forward_routed routes failures to routing error and cancelled error and asyncio and base exception while preserving cleanup and success flow. + try: + # What: compute done and from wait and asyncio and acquisition and reservation wait; why: if acquisition in done later reads done and, so forward_routed must retain the computed value under that name. + done, _ = await asyncio.wait( + # What: supply return when to asyncio.wait; why: forward_routed binds this first completed and asyncio value to asyncio.wait's return when input. + {acquisition, reservation_wait}, return_when=asyncio.FIRST_COMPLETED + # What: complete the asyncio.wait call with return when; why: forward_routed groups the supplied clauses as one asyncio.wait call before its value is consumed. + ) + # What: gate on acquisition and done before cancel and reservation wait; why: forward_routed admits cancel and reservation wait only for this predicate and excludes the opposite state. + if acquisition in done: + # What: call reservation_wait.cancel with the declared inputs; why: forward_routed invokes reservation_wait.cancel while performing lease acquisition result; the call advances that operation through its result or side effect. + reservation_wait.cancel() + # What: compute lease from result and acquisition; why: lease await acquisition later reads lease, so forward_routed must retain the computed value under that name. + lease = acquisition.result() + # What: select the remaining branch that performs if reservation state loading required; why: forward_routed covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: gate on reservation state before streaming response and owned and inflight lock and call and scope; why: forward_routed admits streaming response and owned and inflight lock and call and scope only for this predicate and excludes the opposite state. + if reservation_state["loadingRequired"]: + # What: define AdmissionOwnedStreamingResponse as the owner of __call__; why: daemon callers use this class boundary so those methods share one admission owned streaming response state invariant. + class AdmissionOwnedStreamingResponse(StreamingResponse): + # What: define __call__ around scope and receive and send; why: its direct callers call __call__ for call and rely on this exact input and result contract. + async def __call__(self, scope, receive, send) -> None: + # What: establish the handler boundary for the protected operation; why: AdmissionOwnedStreamingResponse.__call__ routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: call operation.__call__ with scope and receive and send; why: __call__ invokes operation.__call__ while performing finally; the call advances that operation through its result or side effect. + await super().__call__(scope, receive, send) + # What: run async generator finalization can be deferred on every exit path; why: __call__ performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # Async-generator finalization can be deferred + # beyond response termination. The response is + # the ownership barrier for both unstarted and + # suspended loading iterators. + # What: enter the inflight lock managed context before owned request id in request reservations; why: __call__ releases this resource or lock after owned request id in request reservations on both success and failure paths. + with inflight_lock: + # What: compute owned from request id and request reservations; why: if owned later reads owned, so __call__ must retain the computed value under that name. + owned = request_id in request_reservations + # What: call request_reservations.pop with request id and the named fixture input; why: __call__ invokes request_reservations.pop while performing if owned; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: gate on owned before abandon loading acquisition and acquisition; why: __call__ admits abandon loading acquisition and acquisition only for this predicate and excludes the opposite state. + if owned: + # What: call abandon_loading_acquisition with acquisition; why: __call__ invokes abandon_loading_acquisition while performing the enclosing return; the call advances that operation through its result or side effect. + abandon_loading_acquisition(acquisition) + + # What: return admission owned streaming response and loading stream and acquisition and request id and 200 from; why: forward_routed exposes admission owned streaming response and loading stream and acquisition and request id and 200 so its caller can continue with the function\'s computed outcome. + return AdmissionOwnedStreamingResponse( + # What: call loading_stream with acquisition; why: forward_routed invokes loading_stream while performing status code; the call advances that operation through its result or side effect. + loading_stream(acquisition), + # What: supply status code to AdmissionOwnedStreamingResponse; why: forward_routed binds this 200 value to AdmissionOwnedStreamingResponse's status code input. + status_code=200, + # What: supply headers to AdmissionOwnedStreamingResponse; why: forward_routed binds this request id and cache control and connection and x ft request id and no cache value to AdmissionOwnedStreamingResponse's headers input. + headers={ + # What: map the cache control field as no cache; why: forward_routed carries cache control through "Cache-Control": "no-cache" into except exception as exc the lease must. + "Cache-Control": "no-cache", + # What: map the connection field as keep alive; why: forward_routed carries connection through "Connection": "keep-alive" into except exception as exc the lease must. + "Connection": "keep-alive", + # What: map the x ft request id field as request id; why: forward_routed carries x ft request id through "X-FT-Request-ID": request_id into except exception as exc the lease must. + "X-FT-Request-ID": request_id, + # What: complete the enclosing predicate mapping with cache control and connection and x ft request id; why: forward_routed groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + }, + # What: supply media type to AdmissionOwnedStreamingResponse; why: forward_routed binds this text and event stream value to AdmissionOwnedStreamingResponse's media type input. + media_type="text/event-stream", + # What: complete the AdmissionOwnedStreamingResponse call with status code and headers and media type; why: forward_routed groups the supplied clauses as one AdmissionOwnedStreamingResponse call before its value is consumed. + ) + # What: compute lease from acquisition; why: lease later reads lease, so forward_routed must retain the computed value under that name. + lease = await acquisition + # What: handle routing error by reservation wait cancel; why: forward_routed converts that failure into this concrete recovery, response, or cleanup behavior. + except RoutingError as exc: + # What: call reservation_wait.cancel with the declared inputs; why: forward_routed invokes reservation_wait.cancel while performing with inflight lock; the call advances that operation through its result or side effect. + reservation_wait.cancel() + # What: enter the inflight lock managed context before request reservations pop request id; why: forward_routed releases this resource or lock after request reservations pop request id on both success and failure paths. + with inflight_lock: + # What: call request_reservations.pop with request id and the named fixture input; why: forward_routed invokes request_reservations.pop while performing router event admission failed profile model route safe route; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: preserve the exact router event admission failed profile model route safe route literal fragment; why: forward_routed passes this fragment verbatim through router_event("admission_failed", profile=model, route=safe_route, code=e, because changing it would alter a protocol payload, serialized fi. + router_event("admission_failed", profile=model, route=safe_route, code=exc.code) + # What: map the error field as code and str and exc and message and type; why: forward_routed carries error through content into content recovery exc recovery. + content = {"error": {"message": str(exc), "type": exc.code}} + # What: gate on recovery and exc before recovery and content and exc; why: forward_routed admits recovery and content and exc only for this predicate and excludes the opposite state. + if exc.recovery is not None: + # What: compute content entry from recovery and exc; why: content content later reads content entry, so forward_routed must retain the computed value under that name. + content["recovery"] = exc.recovery + # What: return jsonresponse and status code and content and exc and 429 from forward_routed; why: forward_routed exposes jsonresponse and status code and content and exc and 429 so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: forward_routed binds this status code and exc value to JSONResponse's status code input. + status_code=exc.status_code, + # What: supply content to JSONResponse; why: forward_routed binds this content value to JSONResponse's content input. + content=content, + # What: map the retry after field as 1; why: forward_routed carries retry after through headers={"Retry-After": "1"} if exc.status_code == 429 else None into except exception as exc the lease must. + headers={"Retry-After": "1"} if exc.status_code == 429 else None, + # What: complete the JSONResponse call with status code and content and headers; why: forward_routed groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + # What: handle cancelled error and asyncio by reservation wait cancel; why: forward_routed converts that failure into this concrete recovery, response, or cleanup behavior. + except asyncio.CancelledError: + # What: call reservation_wait.cancel with the declared inputs; why: forward_routed invokes reservation_wait.cancel while performing with inflight lock; the call advances that operation through its result or side effect. + reservation_wait.cancel() + # What: enter the inflight lock managed context before request reservations pop request id; why: forward_routed releases this resource or lock after request reservations pop request id on both success and failure paths. + with inflight_lock: + # What: call request_reservations.pop with request id and the named fixture input; why: forward_routed invokes request_reservations.pop while performing abandon loading acquisition acquisition; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: call abandon_loading_acquisition with acquisition; why: forward_routed invokes abandon_loading_acquisition while performing raise; the call advances that operation through its result or side effect. + abandon_loading_acquisition(acquisition) + # What: re-propagate the active failure to the caller; why: forward_routed stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: handle base exception by reservation wait cancel; why: forward_routed converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException: + # What: call reservation_wait.cancel with the declared inputs; why: forward_routed invokes reservation_wait.cancel while performing with inflight lock; the call advances that operation through its result or side effect. + reservation_wait.cancel() + # What: enter the inflight lock managed context before request reservations pop request id; why: forward_routed releases this resource or lock after request reservations pop request id on both success and failure paths. + with inflight_lock: + # What: call request_reservations.pop with request id and the named fixture input; why: forward_routed invokes request_reservations.pop while performing if not acquisition done; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: gate on done and acquisition before abandon loading acquisition and acquisition; why: forward_routed admits abandon loading acquisition and acquisition only for this predicate and excludes the opposite state. + if not acquisition.done(): + # What: supply record cancellation to abandon_loading_acquisition; why: forward_routed binds this false value to abandon_loading_acquisition's record cancellation input. + abandon_loading_acquisition(acquisition, record_cancellation=False) + # What: re-propagate the active failure to the caller; why: forward_routed stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: run if not reservation wait done on every exit path; why: forward_routed performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: gate on done and reservation wait before cancel and reservation wait; why: forward_routed admits cancel and reservation wait only for this predicate and excludes the opposite state. + if not reservation_wait.done(): + # What: call reservation_wait.cancel with the declared inputs; why: forward_routed invokes reservation_wait.cancel while performing else; the call advances that operation through its result or side effect. + reservation_wait.cancel() + # What: select the remaining branch that performs lease; why: forward_routed covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: compute lease from the named fixture input; why: if lease is later reads lease, so forward_routed must retain the computed value under that name. + lease = None + # What: establish the handler boundary for the protected operation; why: forward_routed routes failures to routing error and cancelled error and asyncio and base exception while preserving cleanup and success flow. + try: + # What: gate on lease before lease and acquire route and model and admission cancellation; why: forward_routed admits lease and acquire route and model and admission cancellation only for this predicate and excludes the opposite state. + if lease is None: + # What: compute lease from acquire route and model and admission cancellation; why: lease release later reads lease, so forward_routed must retain the computed value under that name. + lease = await acquire_route(model, admission_cancellation) + # What: handle routing error by with inflight lock; why: forward_routed converts that failure into this concrete recovery, response, or cleanup behavior. + except RoutingError as exc: + # What: enter the inflight lock managed context before request reservations pop request id; why: forward_routed releases this resource or lock after request reservations pop request id on both success and failure paths. + with inflight_lock: + # What: call request_reservations.pop with request id and the named fixture input; why: forward_routed invokes request_reservations.pop while performing router event admission failed profile model route safe route; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: preserve the exact router event admission failed profile model route safe route literal fragment; why: forward_routed passes this fragment verbatim through router_event("admission_failed", profile=model, route=safe_route, code=e, because changing it would alter a protocol payload, serialized fixtur. + router_event("admission_failed", profile=model, route=safe_route, code=exc.code) + # What: map the error field as code and str and exc and message and type; why: forward_routed carries error through content into content recovery exc recovery. + content = {"error": {"message": str(exc), "type": exc.code}} + # What: gate on recovery and exc before recovery and content and exc; why: forward_routed admits recovery and content and exc only for this predicate and excludes the opposite state. + if exc.recovery is not None: + # What: compute content entry from recovery and exc; why: content content later reads content entry, so forward_routed must retain the computed value under that name. + content["recovery"] = exc.recovery + # What: return jsonresponse and status code and content and exc and 429 from forward_routed; why: forward_routed exposes jsonresponse and status code and content and exc and 429 so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: forward_routed binds this status code and exc value to JSONResponse's status code input. + status_code=exc.status_code, + # What: supply content to JSONResponse; why: forward_routed binds this content value to JSONResponse's content input. + content=content, + # What: map the retry after field as 1; why: forward_routed carries retry after through headers={"Retry-After": "1"} if exc.status_code == 429 else None into except exception as exc the lease must. + headers={"Retry-After": "1"} if exc.status_code == 429 else None, + # What: complete the JSONResponse call with status code and content and headers; why: forward_routed groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + # What: handle cancelled error and asyncio by with inflight lock; why: forward_routed converts that failure into this concrete recovery, response, or cleanup behavior. + except asyncio.CancelledError: + # What: enter the inflight lock managed context before request reservations pop request id; why: forward_routed releases this resource or lock after request reservations pop request id on both success and failure paths. + with inflight_lock: + # What: call request_reservations.pop with request id and the named fixture input; why: forward_routed invokes request_reservations.pop while performing router cancel acquire admission cancellation; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: call router.cancel_acquire with admission cancellation; why: forward_routed invokes router.cancel_acquire while performing router record cancellation; the call advances that operation through its result or side effect. + router.cancel_acquire(admission_cancellation) + # What: call router.record_cancellation with the declared inputs; why: forward_routed invokes router.record_cancellation while performing router event request cancelled profile model route safe route; the call advances that operation through its result or side effect. + router.record_cancellation() + # What: preserve the exact router event request cancelled profile model route safe route literal fragment; why: forward_routed passes this fragment verbatim through router_event("request_cancelled", profile=model, route=safe_route), because changing it would alter a protocol payload, serialized fixture, or. + router_event("request_cancelled", profile=model, route=safe_route) + # What: re-propagate the active failure to the caller; why: forward_routed stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: handle base exception by with inflight lock; why: forward_routed converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException: + # What: enter the inflight lock managed context before request reservations pop request id; why: forward_routed releases this resource or lock after request reservations pop request id on both success and failure paths. + with inflight_lock: + # What: call request_reservations.pop with request id and the named fixture input; why: forward_routed invokes request_reservations.pop while performing raise; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: re-propagate the active failure to the caller; why: forward_routed stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: enter the inflight lock managed context before cancelled before connect request reservations request id cancelled; why: forward_routed releases this resource or lock after cancelled before connect request reservations request id cancelled on both success and failure paths. + with inflight_lock: + # What: compute cancelled before connect from request reservations and request id and cancelled; why: if cancelled before connect later reads cancelled before connect, so forward_routed must retain the computed value under that name. + cancelled_before_connect = request_reservations[request_id]["cancelled"] + # What: gate on cancelled before connect before pop and request id and request reservations; why: forward_routed admits pop and request id and request reservations only for this predicate and excludes the opposite state. + if cancelled_before_connect: + # What: call request_reservations.pop with request id and the named fixture input; why: forward_routed invokes request_reservations.pop while performing if cancelled before connect; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: gate on cancelled before connect before release and lease; why: forward_routed admits release and lease only for this predicate and excludes the opposite state. + if cancelled_before_connect: + # What: call lease.release with the declared inputs; why: forward_routed invokes lease.release while performing return jsonresponse; the call advances that operation through its result or side effect. + lease.release() + # What: return jsonresponse and 409 and error and message and type from forward_routed; why: forward_routed exposes jsonresponse and 409 and error and message and type so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: forward_routed binds this 409 value to JSONResponse's status code input. + status_code=409, + # What: map the error field as message and type and request and cancelled and before; why: forward_routed carries error through content={"error": { into except exception as exc the lease must. + content={"error": { + # What: apply the message request cancelled before upstream connection portion of the enclosing predicate; why: this clause remains in forward_routed\'s enclosing expression so its grouping and evaluation order stay intact. + "message": "request cancelled before upstream connection", + # What: apply the type request cancelled portion of the enclosing predicate; why: this clause remains in forward_routed\'s enclosing expression so its grouping and evaluation order stay intact. + "type": "request_cancelled", + # What: complete the enclosing predicate mapping with error; why: forward_routed groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + }}, + # What: complete the JSONResponse call with status code and content; why: forward_routed groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + # What: establish the handler boundary for the protected operation; why: forward_routed routes failures to request model error while preserving cleanup and success flow. + try: + # What: compute outbound body from filtered body and lease; why: body outbound body later reads outbound body, so forward_routed must retain the computed value under that name. + outbound_body = filtered_body(lease) + # What: handle request model error by lease release; why: forward_routed converts that failure into this concrete recovery, response, or cleanup behavior. + except RequestModelError as exc: + # What: call lease.release with the declared inputs; why: forward_routed invokes lease.release while performing with inflight lock; the call advances that operation through its result or side effect. + lease.release() + # What: enter the inflight lock managed context before request reservations pop request id; why: forward_routed releases this resource or lock after request reservations pop request id on both success and failure paths. + with inflight_lock: + # What: call request_reservations.pop with request id and the named fixture input; why: forward_routed invokes request_reservations.pop while performing return jsonresponse; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: return jsonresponse and str and exc and 400 and error from forward_routed; why: forward_routed exposes jsonresponse and str and exc and 400 and error so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: forward_routed binds this 400 value to JSONResponse's status code input. + status_code=400, + # What: map the error field as str and exc and message and type and invalid request; why: forward_routed carries error through content={"error": {"message": str(exc), "type": "invalid_request"}} into except exception as exc the lease must. + content={"error": {"message": str(exc), "type": "invalid_request"}}, + # What: complete the JSONResponse call with status code and content; why: forward_routed groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + try: + # What: compute upstream from connect upstream and port and proxy base url and path and query; why: upstream upstream later reads upstream, so forward_routed must retain the computed value under that name. + upstream = await connect_upstream( + # What: supply port to connect_upstream; why: forward_routed binds this port and lease value to connect_upstream's port input. + port=lease.port, + # What: supply base url to connect_upstream; why: forward_routed binds this proxy base url and lease value to connect_upstream's base url input. + base_url=lease.proxy_base_url, + # What: supply path and query to connect_upstream; why: forward_routed binds this path and query value to connect_upstream's path and query input. + path_and_query=path_and_query, + # What: supply headers to dict; why: forward_routed binds this dict and headers and request value to dict's headers input. + headers=dict(request.headers), + # What: supply body to connect_upstream; why: forward_routed binds this outbound body value to connect_upstream's body input. + body=outbound_body, + # What: supply method to connect_upstream; why: forward_routed binds this method and request value to connect_upstream's method input. + method=request.method, + # What: supply timeout s to connect_upstream; why: forward_routed binds this upstream timeout s and profile and router and lease value to connect_upstream's timeout s input. + timeout_s=lease.profile.upstream_timeout_s or router.upstream_timeout_s, + # What: complete the connect_upstream call with port and base url and path and query and headers and body; why: forward_routed groups the supplied clauses as one connect_upstream call before its value is consumed. + ) + # What: handle cancelled error and asyncio by lease release; why: forward_routed converts that failure into this concrete recovery, response, or cleanup behavior. + except asyncio.CancelledError: + # What: call lease.release with the declared inputs; why: forward_routed invokes lease.release while performing with inflight lock; the call advances that operation through its result or side effect. + lease.release() + # What: enter the inflight lock managed context before request reservations pop request id; why: forward_routed releases this resource or lock after request reservations pop request id on both success and failure paths. + with inflight_lock: + # What: call request_reservations.pop with request id and the named fixture input; why: forward_routed invokes request_reservations.pop while performing router record cancellation; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: call router.record_cancellation with the declared inputs; why: forward_routed invokes router.record_cancellation while performing router event request cancelled profile model route safe route; the call advances that operation through its result or side effect. + router.record_cancellation() + # What: preserve the exact router event request cancelled profile model route safe route literal fragment; why: forward_routed passes this fragment verbatim through router_event("request_cancelled", profile=model, route=safe_route), because changing it would alter a protocol payload, serialized fixture, or. + router_event("request_cancelled", profile=model, route=safe_route) + # What: re-propagate the active failure to the caller; why: forward_routed stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: handle exception by lease release; why: forward_routed converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: # the lease must not strand a pending swap on connect failure + # What: call lease.release with the declared inputs; why: forward_routed invokes lease.release while performing with inflight lock; the call advances that operation through its result or side effect. + lease.release() + # What: enter the inflight lock managed context before request reservations pop request id; why: forward_routed releases this resource or lock after request reservations pop request id on both success and failure paths. + with inflight_lock: + # What: call request_reservations.pop with request id and the named fixture input; why: forward_routed invokes request_reservations.pop while performing router event upstream connect failed profile model route safe route; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: preserve the exact router event upstream connect failed profile model route safe route literal fragment; why: forward_routed passes this fragment verbatim through router_event("upstream_connect_failed", profile=model, route=safe_route), because changing it would alter a protocol payload, serialized. + router_event("upstream_connect_failed", profile=model, route=safe_route) + # What: map the error field as str and exc and message and type and upstream unavailable; why: forward_routed carries error into return JSONResponse(status_code=502, content={"error": {"message": str(e. + return JSONResponse(status_code=502, content={"error": {"message": str(exc), "type": "upstream_unavailable"}}) + # What: handle base exception by lease release; why: forward_routed converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException: + # What: call lease.release with the declared inputs; why: forward_routed invokes lease.release while performing with inflight lock; the call advances that operation through its result or side effect. + lease.release() + # What: enter the inflight lock managed context before request reservations pop request id; why: forward_routed releases this resource or lock after request reservations pop request id on both success and failure paths. + with inflight_lock: + # What: call request_reservations.pop with request id and the named fixture input; why: forward_routed invokes request_reservations.pop while performing raise; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: re-propagate the active failure to the caller; why: forward_routed stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: enter the inflight lock managed context before cancelled while connecting request reservations request id cancelled; why: forward_routed releases this resource or lock after cancelled while connecting request reservations request id cancelled on both success and failure paths. + with inflight_lock: + # What: compute cancelled while connecting from request reservations and request id and cancelled; why: if cancelled while connecting later reads cancelled while connecting, so forward_routed must retain the computed value under that name. + cancelled_while_connecting = request_reservations[request_id]["cancelled"] + # What: gate on cancelled while connecting before pop and request id and request reservations; why: forward_routed admits pop and request id and request reservations only for this predicate and excludes the opposite state. + if cancelled_while_connecting: + # What: call request_reservations.pop with request id and the named fixture input; why: forward_routed invokes request_reservations.pop while performing else; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: select the remaining branch that performs inflight request id; why: forward_routed covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: compute inflight entry from name and upstream and profile and lease and profile; why: item inflight get request id later reads inflight entry, so forward_routed must retain the computed value under that name. + inflight[request_id] = { + # What: map the profile field as name and profile and lease; why: forward_routed carries profile through inflight entry into item inflight get request id. + "profile": lease.profile.name, + # What: map the upstream field as upstream; why: forward_routed carries upstream through inflight entry into item inflight get request id. + "upstream": upstream, + # What: map the cancelled field as false; why: forward_routed carries cancelled through inflight entry into item inflight get request id. + "cancelled": False, + # What: complete the inflight entry mapping with profile and upstream and cancelled; why: forward_routed groups the supplied clauses as one inflight entry mapping before its value is consumed. + } + # What: gate on cancelled while connecting before close and upstream; why: forward_routed admits close and upstream only for this predicate and excludes the opposite state. + if cancelled_while_connecting: + # What: call upstream.close with the declared inputs; why: forward_routed invokes upstream.close while performing lease release; the call advances that operation through its result or side effect. + upstream.close() + # What: call lease.release with the declared inputs; why: forward_routed invokes lease.release while performing return jsonresponse; the call advances that operation through its result or side effect. + lease.release() + # What: return jsonresponse and 409 and error and message and type from forward_routed; why: forward_routed exposes jsonresponse and 409 and error and message and type so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: forward_routed binds this 409 value to JSONResponse's status code input. + status_code=409, + # What: map the error field as message and type and request and cancelled and while; why: forward_routed carries error into content={"error": {. + content={"error": { + # What: apply the message request cancelled while opening upstream portion of the enclosing predicate; why: this clause remains in forward_routed\'s enclosing expression so its grouping and evaluation order stay intact. + "message": "request cancelled while opening upstream connection", + # What: apply the type request cancelled portion of the enclosing predicate; why: this clause remains in forward_routed\'s enclosing expression so its grouping and evaluation order stay intact. + "type": "request_cancelled", + # What: complete the enclosing predicate mapping with error; why: forward_routed groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + }}, + # What: complete the JSONResponse call with status code and content; why: forward_routed groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + # What: preserve the exact router event admitted lease event identity lease route safe route literal fragment; why: forward_routed passes this fragment verbatim through router_event("admitted", **lease_event_identity(lease), route=safe_route, because changing it would alter a protocol payload, serialized fixtu. + router_event("admitted", **lease_event_identity(lease), route=safe_route) + + # What: define stream_response around the current object state; why: its direct callers call stream_response for stream response and rely on this exact input and result contract. + def stream_response(): + # What: compute first byte at from the named fixture input; why: if first byte at is later reads first byte at, so stream_response must retain the computed value under that name. + first_byte_at = None + # What: compute byte count from 0; why: byte count len chunk later reads byte count, so stream_response must retain the computed value under that name. + byte_count = 0 + # What: compute captured response from bytearray; why: if len captured response len chunk capture limit later reads captured response, so stream_response must retain the computed value under that name. + captured_response = bytearray() + # What: compute capture overflow from false; why: if capture limit and not capture overflow later reads capture overflow, so stream_response must retain the computed value under that name. + capture_overflow = False + # What: establish the handler boundary for the protected operation; why: stream_response routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: iterate across chunks and upstream to perform first byte at and monotonic and time; why: stream_response repeats the body only while or for the loop header admits an iteration. + for chunk in upstream.chunks(): + # What: gate on first byte at before first byte at and monotonic and time; why: stream_response admits first byte at and monotonic and time only for this predicate and excludes the opposite state. + if first_byte_at is None: + # What: compute first byte at from monotonic and time; why: ttft s first byte at started if first byte at is later reads first byte at, so stream_response must retain the computed value under that name. + first_byte_at = time.monotonic() + # What: compute byte count from len and chunk; why: response bytes byte count later reads byte count, so stream_response must retain the computed value under that name. + byte_count += len(chunk) + # What: gate on capture limit and capture overflow before capture limit and capture overflow and extend and chunk and clear; why: stream_response admits capture limit and capture overflow and extend and chunk and clear only for this predicate and excludes the opposite state. + if capture_limit and not capture_overflow: + # What: gate on capture limit and len and captured response and chunk before extend and chunk and captured response; why: stream_response admits extend and chunk and captured response only for this predicate and excludes the opposite state. + if len(captured_response) + len(chunk) <= capture_limit: + # What: call captured_response.extend with chunk; why: stream_response invokes captured_response.extend while performing else; the call advances that operation through its result or side effect. + captured_response.extend(chunk) + # What: select the remaining branch that performs capture overflow; why: stream_response covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: compute capture overflow from true; why: response body if capture overflow else bytes captured response later reads capture overflow, so stream_response must retain the computed value under that name. + capture_overflow = True + # What: call captured_response.clear with the declared inputs; why: stream_response invokes captured_response.clear while performing yield chunk; the call advances that operation through its result or side effect. + captured_response.clear() + # What: apply the yield chunk portion of the enclosing predicate; why: this clause remains in stream_response\'s enclosing expression so its grouping and evaluation order stay intact. + yield chunk + # What: run ended time monotonic on every exit path; why: stream_response performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: compute ended from monotonic and time; why: duration s ended started later reads ended, so stream_response must retain the computed value under that name. + ended = time.monotonic() + # What: enter the inflight lock managed context before item inflight get request id; why: stream_response releases this resource or lock after item inflight get request id on both success and failure paths. + with inflight_lock: + # What: compute item from get and request id and inflight; why: cancelled bool item get cancelled later reads item, so stream_response must retain the computed value under that name. + item = inflight.get(request_id, {}) + # What: compute cancelled from bool and get and item and cancelled; why: completed not cancelled later reads cancelled, so stream_response must retain the computed value under that name. + cancelled = bool(item.get("cancelled")) + # What: gate on upstream and get and item before pop and request id and inflight; why: stream_response admits pop and request id and inflight only for this predicate and excludes the opposite state. + if item.get("upstream") is upstream: + # What: call inflight.pop with request id and the named fixture input; why: stream_response invokes inflight.pop while performing request reservations pop request id; the call advances that operation through its result or side effect. + inflight.pop(request_id, None) + # What: call request_reservations.pop with request id and the named fixture input; why: stream_response invokes request_reservations.pop while performing ttft s first byte at started if first byte at is; the call advances that operation through its result or side effect. + request_reservations.pop(request_id, None) + # What: compute ttft s from first byte at and started; why: ttft s ttft s later reads ttft s, so stream_response must retain the computed value under that name. + ttft_s = (first_byte_at - started) if first_byte_at is not None else None + # What: call router.record_stream with the declared inputs; why: stream_response invokes router.record_stream while performing ttft s ttft s; the call advances that operation through its result or side effect. + router.record_stream( + # What: supply ttft s to router.record_stream; why: stream_response binds this ttft s value to router.record_stream's ttft s input. + ttft_s=ttft_s, + # What: supply duration s to router.record_stream; why: stream_response binds this ended and started value to router.record_stream's duration s input. + duration_s=ended - started, + # What: supply response bytes to router.record_stream; why: stream_response binds this byte count value to router.record_stream's response bytes input. + response_bytes=byte_count, + # What: supply completed to router.record_stream; why: stream_response binds this cancelled value to router.record_stream's completed input. + completed=not cancelled, + # What: complete the router.record_stream call with ttft s and duration s and response bytes and completed; why: stream_response groups the supplied clauses as one router.record_stream call before its value is consumed. + ) + # What: call lease.release with the declared inputs; why: stream_response invokes lease.release while performing router event; the call advances that operation through its result or side effect. + lease.release() + # What: call router_event with request finished; why: stream_response invokes router_event while performing request finished; the call advances that operation through its result or side effect. + router_event( + # What: preserve the exact request finished literal fragment; why: stream_response passes this fragment verbatim through "request_finished", because changing it would alter a protocol payload, serialized fixture, or public message. + "request_finished", + # What: supply expanded arguments to lease_event_identity; why: stream_response binds this lease event identity and lease value to lease_event_identity's expanded input. + **lease_event_identity(lease), + # What: supply route to router_event; why: stream_response binds this safe route value to router_event's route input. + route=safe_route, + # What: supply status to router_event; why: stream_response binds this status and upstream value to router_event's status input. + status=upstream.status, + # What: supply cancelled to router_event; why: stream_response binds this cancelled value to router_event's cancelled input. + cancelled=cancelled, + # What: supply response bytes to router_event; why: stream_response binds this byte count value to router_event's response bytes input. + responseBytes=byte_count, + # What: complete the router_event call with route and status and cancelled and response bytes; why: stream_response groups the supplied clauses as one router_event call before its value is consumed. + ) + # What: call activity_store.record with the declared inputs; why: stream_response invokes activity_store.record while performing model lease profile name; the call advances that operation through its result or side effect. + activity_store.record( + # What: supply model to activity_store.record; why: stream_response binds this name and profile and lease value to activity_store.record's model input. + model=lease.profile.name, + # What: supply route to activity_store.record; why: stream_response binds this safe route value to activity_store.record's route input. + route=safe_route, + # What: supply method to activity_store.record; why: stream_response binds this method and request value to activity_store.record's method input. + method=request.method, + # What: supply status to activity_store.record; why: stream_response binds this status and upstream value to activity_store.record's status input. + status=upstream.status, + # What: supply started to activity_store.record; why: stream_response binds this started value to activity_store.record's started input. + started=started, + # What: supply ttft s to activity_store.record; why: stream_response binds this ttft s value to activity_store.record's ttft s input. + ttft_s=ttft_s, + # What: supply response bytes to activity_store.record; why: stream_response binds this byte count value to activity_store.record's response bytes input. + response_bytes=byte_count, + # What: supply cancelled to activity_store.record; why: stream_response binds this cancelled value to activity_store.record's cancelled input. + cancelled=cancelled, + # What: supply request headers to activity_store.record; why: stream_response binds this headers and request value to activity_store.record's request headers input. + request_headers=request.headers, + # What: supply request body to activity_store.record; why: stream_response binds this outbound body value to activity_store.record's request body input. + request_body=outbound_body, + # What: supply response headers to activity_store.record; why: stream_response binds this headers and upstream value to activity_store.record's response headers input. + response_headers=upstream.headers, + # What: supply response body to bytes; why: stream_response binds this capture overflow and bytes and captured response value to bytes's response body input. + response_body=None if capture_overflow else bytes(captured_response), + # What: complete the activity_store.record call with model and route and method and status and started; why: stream_response groups the supplied clauses as one activity_store.record call before its value is consumed. + ) + + # What: compute headers from response headers and headers and upstream; why: headers x ft request id request id later reads headers, so forward_routed must retain the computed value under that name. + headers = response_headers(upstream.headers) + # What: compute headers entry from request id; why: headers headers later reads headers entry, so forward_routed must retain the computed value under that name. + headers["X-FT-Request-ID"] = request_id + # What: return streaming response and stream response and status and headers from forward_routed; why: forward_routed exposes streaming response and stream response and status and headers so its caller can continue with the function\'s computed outcome. + return StreamingResponse( + # What: call stream_response with the declared inputs; why: forward_routed invokes stream_response while performing status code upstream status; the call advances that operation through its result or side effect. + stream_response(), + # What: supply status code to StreamingResponse; why: forward_routed binds this status and upstream value to StreamingResponse's status code input. + status_code=upstream.status, + # What: supply headers to StreamingResponse; why: forward_routed binds this headers value to StreamingResponse's headers input. + headers=headers, + # What: supply media type to upstream.headers.get; why: forward_routed binds this get and headers and upstream and content type value to upstream.headers.get's media type input. + media_type=upstream.headers.get("Content-Type"), + # What: complete the StreamingResponse call with status code and headers and media type; why: forward_routed groups the supplied clauses as one StreamingResponse call before its value is consumed. + ) + + # What: define route_inference around request; why: its direct callers call route_inference for route inference and rely on this exact input and result contract. + async def route_inference(request: Request): + # What: compute body from body and request; why: model request model body later reads body, so route_inference must retain the computed value under that name. + body = await request.body() + # What: establish the handler boundary for the protected operation; why: route_inference routes failures to request model error and catalog error while preserving cleanup and success flow. + try: + # What: compute model from request model and body; why: if not router has routable id model later reads model, so route_inference must retain the computed value under that name. + model = request_model(body) + # What: gate on has routable id and model and router before catalog error and model; why: route_inference admits catalog error and model only for this predicate and excludes the opposite state. + if not router.has_routable_id(model): + # What: raise CatalogError for the caller; why: route_inference stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"unknown model profile {model!r}") + # What: handle request model error by raise httpexception status code 400 detail str exc; why: route_inference converts that failure into this concrete recovery, response, or cleanup behavior. + except RequestModelError as exc: + # What: raise HTTPException for the caller; why: route_inference stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=400, detail=str(exc)) from exc + # What: handle catalog error by return jsonresponse; why: route_inference converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError as exc: + # What: return jsonresponse and str and exc and 404 and error from route_inference; why: route_inference exposes jsonresponse and str and exc and 404 and error so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: route_inference binds this 404 value to JSONResponse's status code input. + status_code=404, + # What: map the error field as str and exc and message and type and unknown model; why: route_inference carries error into content={"error": {"message": str(exc), "type": "unknown_model"}}. + content={"error": {"message": str(exc), "type": "unknown_model"}}, + # What: complete the JSONResponse call with status code and content; why: route_inference groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + # What: compute suffix from query and url and request and value and value; why: path and query request url path suffix later reads suffix, so route_inference must retain the computed value under that name. + suffix = f"?{request.url.query}" if request.url.query else "" + # What: return forward routed and request and model and body from route_inference; why: route_inference exposes forward routed and request and model and body so its caller can continue with the function\'s computed outcome. + return await forward_routed( + # What: apply the request portion of the enclosing predicate; why: this clause remains in route_inference\'s enclosing expression so its grouping and evaluation order stay intact. + request, + # What: apply the model portion of the enclosing predicate; why: this clause remains in route_inference\'s enclosing expression so its grouping and evaluation order stay intact. + model, + # What: supply path and query to forward_routed; why: route_inference binds this path and suffix and url and request value to forward_routed's path and query input. + path_and_query=request.url.path + suffix, + # What: supply body to forward_routed; why: route_inference binds this body value to forward_routed's body input. + body=body, + # What: supply apply request filters to forward_routed; why: route_inference binds this true value to forward_routed's apply request filters input. + apply_request_filters=True, + # What: complete the forward_routed call with path and query and body and apply request filters; why: route_inference groups the supplied clauses as one forward_routed call before its value is consumed. + ) + + # FreeToken's supported inference surface. All routes use the same native + # admission and proxy path so an OpenAI or Anthropic client cannot bypass + # lifecycle, accounting, readiness, or cancellation ownership. + # What: register POST /v1/chat/completions on the application router; why: clients reach inference_proxy's handler only through this method-and-path binding. + @app.post("/v1/chat/completions", dependencies=[Depends(require_router_key)]) + # What: register POST /v1/completions on the application router; why: clients reach inference_proxy's handler only through this method-and-path binding. + @app.post("/v1/completions", dependencies=[Depends(require_router_key)]) + # What: register POST /v1/responses on the application router; why: clients reach inference_proxy's handler only through this method-and-path binding. + @app.post("/v1/responses", dependencies=[Depends(require_router_key)]) + # What: register POST /v1/messages on the application router; why: clients reach inference_proxy's handler only through this method-and-path binding. + @app.post("/v1/messages", dependencies=[Depends(require_router_key)]) + # What: register POST /v1/messages/count_tokens on the application router; why: clients reach inference_proxy's handler only through this method-and-path binding. + @app.post("/v1/messages/count_tokens", dependencies=[Depends(require_router_key)]) + # What: define inference_proxy around request; why: the registered API client call inference_proxy for inference proxy and rely on this exact input and result contract. + async def inference_proxy(request: Request): + # What: return route inference and request from inference_proxy; why: inference_proxy exposes route inference and request so its caller can continue with the function\'s computed outcome. + return await route_inference(request) + + # FreeToken's Responses implementation is deliberately stateless. Keep its + # registered resource routes available at the stable daemon URL, but do not + # activate an arbitrary model for a request that carries no model identity. + # The envelope matches the engine contract and remains behind inference auth. + # What: register GET /v1/responses/{response_id} on the application router; why: clients reach stateless_response_not_found's handler only through this method-and-path binding. + @app.get("/v1/responses/{response_id}", dependencies=[Depends(require_router_key)]) + # What: register POST the configured path on the application router; why: clients reach stateless_response_not_found's handler only through this method-and-path binding. + @app.post( + # What: apply the v1 responses response id cancel portion of the enclosing predicate; why: this clause remains in stateless_response_not_found\'s enclosing expression so its grouping and evaluation order stay intact. + "/v1/responses/{response_id}/cancel", + # What: supply dependencies to Depends; why: stateless_response_not_found binds this depends and require router key value to Depends's dependencies input. + dependencies=[Depends(require_router_key)], + # What: complete the app.post call with dependencies; why: stateless_response_not_found groups the supplied clauses as one app.post call before its value is consumed. + ) + # What: define stateless_response_not_found around response id; why: the registered API client call stateless_response_not_found for stateless response not found and rely on this exact input and result contract. + async def stateless_response_not_found(response_id: str): + # What: return jsonresponse and response id and 404 and error and message from stateless_response_not_found; why: stateless_response_not_found exposes jsonresponse and response id and 404 and error and message so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: stateless_response_not_found binds this 404 value to JSONResponse's status code input. + status_code=404, + # What: supply content to JSONResponse; why: stateless_response_not_found binds this response id and error and message and type and code value to JSONResponse's content input. + content={ + # What: apply the error portion of the enclosing predicate; why: this clause remains in stateless_response_not_found\'s enclosing expression so its grouping and evaluation order stay intact. + "error": { + # What: map the message field as response id and response and not and found and stateless; why: stateless_response_not_found carries message into "message": f"response {response_id!r} not found (stateless server)". + "message": f"response {response_id!r} not found (stateless server)", + # What: map the type field as invalid request error; why: stateless_response_not_found carries type into "type": "invalid_request_error". + "type": "invalid_request_error", + # What: map the code field as the fixture input; why: stateless_response_not_found carries code into "code": None. + "code": None, + # What: complete the enclosing predicate mapping with message and type and code; why: stateless_response_not_found groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + # What: complete the enclosing predicate mapping with error; why: stateless_response_not_found groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + }, + # What: complete the JSONResponse call with status code and content; why: stateless_response_not_found groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + + # What: register GET /models on the application router; why: clients reach openai_model_list's handler only through this method-and-path binding. + @app.get("/models", dependencies=[Depends(require_router_key)]) + # What: register GET /v1/models on the application router; why: clients reach openai_model_list's handler only through this method-and-path binding. + @app.get("/v1/models", dependencies=[Depends(require_router_key)]) + # What: define openai_model_list around request; why: the registered API client call openai_model_list for openai model list and rely on this exact input and result contract. + async def openai_model_list(request: Request): + """OpenAI-compatible public metadata without exposing local model paths.""" + # What: document open ai compatible public metadata without exposing local in the openai_model_list docstring; why: introspection and maintainers read this exact docstring fragment to understand openai model list behavior without executing it. + # What: evaluate and capture catalog snapshot loaded profiles active routing profile; why: the enclosing qualifier uses the captured result in its next validation or artifact step. + catalog_snapshot, loaded_profiles, active_routing_profile = ( + # What: call router.public_model_listing_snapshot with the declared inputs; why: openai_model_list consumes the router.public_model_listing_snapshot return value while evaluating router.public_model_listing_snapshot(). + router.public_model_listing_snapshot() + # What: execute the grouped source fragment; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + ) + # What: compute created from int and time; why: created created later reads created, so openai_model_list must retain the computed value under that name. + created = int(time.time()) + # What: initialize data as an empty runtime accumulator; why: openai_model_list appends or maps entries into it during data append record before consuming the aggregate. + data = [] + # What: iterate across listed model ids and catalog snapshot to perform selector and model id and catalog snapshot; why: openai_model_list repeats the body only while or for the loop header admits an iteration. + for model_id in catalog_snapshot.listed_model_ids(): + # What: compute selector from selector and model id and catalog snapshot; why: profile if selector is not else later reads selector, so openai_model_list must retain the computed value under that name. + selector = catalog_snapshot.selector(model_id) + # What: compute profile from selector and get and model id and catalog snapshot; why: loaded profile name in loaded profiles later reads profile, so openai_model_list must retain the computed value under that name. + profile = None if selector is not None else catalog_snapshot.get(model_id) + # What: gate on selector before loaded and name and loaded profiles and profile; why: openai_model_list admits loaded and name and loaded profiles and profile only for this predicate and excludes the opposite state. + if selector is None: + # What: compute loaded from name and loaded profiles and profile; why: loaded any later reads loaded, so openai_model_list must retain the computed value under that name. + loaded = profile.name in loaded_profiles + # What: select the remaining branch that performs targets selector targets if selector strategy pin else; why: openai_model_list covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: compute targets from targets and strategy and selector and pin and 1; why: catalog snapshot get target name in loaded profiles for later reads targets, so openai_model_list must retain the computed value under that name. + targets = selector.targets[:1] if selector.strategy == "pin" else selector.targets + # What: compute loaded from any and name and loaded profiles and target; why: value loaded if loaded else unloaded later reads loaded, so openai_model_list must retain the computed value under that name. + loaded = any( + # What: call catalog_snapshot.get with target; why: openai_model_list consumes the catalog_snapshot.get return value while evaluating catalog_snapshot.get(target).name in loaded_profiles for target in targe. + catalog_snapshot.get(target).name in loaded_profiles for target in targets + # What: complete the any call with name; why: openai_model_list groups the supplied clauses as one any call before its value is consumed. + ) + # What: compute record from model id and created and loaded and id and object; why: record name selector display name later reads record, so openai_model_list must retain the computed value under that name. + record = { + # What: map the id field as model id; why: openai_model_list carries id through record into record name selector display name. + "id": model_id, + # What: map the object field as model; why: openai_model_list carries object through record into record name selector display name. + "object": "model", + # What: map the created field as created; why: openai_model_list carries created through record into record name selector display name. + "created": created, + # What: map the owned by field as freetoken; why: openai_model_list carries owned by through record into record name selector display name. + "owned_by": "freetoken", + # What: apply the status portion of record; why: openai_model_list uses this clause to evaluate record as one grouped value. + "status": { + # What: map the value field as loaded and loaded and unloaded; why: openai_model_list carries value through record into record name selector display name. + "value": "loaded" if loaded else "unloaded" + # What: complete the record mapping with value; why: openai_model_list groups the supplied clauses as one record mapping before its value is consumed. + }, + # What: complete the record mapping with id and object and created and owned by and status; why: openai_model_list groups the supplied clauses as one record mapping before its value is consumed. + } + # What: gate on selector before display name and selector and record; why: openai_model_list admits display name and selector and record only for this predicate and excludes the opposite state. + if selector is not None: + # What: gate on display name and selector before display name and record and selector; why: openai_model_list admits display name and record and selector only for this predicate and excludes the opposite state. + if selector.display_name: + # What: compute record entry from display name and selector; why: record description selector description later reads record entry, so openai_model_list must retain the computed value under that name. + record["name"] = selector.display_name + # What: gate on description and selector before description and record and selector; why: openai_model_list admits description and record and selector only for this predicate and excludes the opposite state. + if selector.description: + # What: compute record entry from description and selector; why: record meta freetoken selector metadata later reads record entry, so openai_model_list must retain the computed value under that name. + record["description"] = selector.description + # What: compute selector metadata from metadata and selector; why: selector metadata update later reads selector metadata, so openai_model_list must retain the computed value under that name. + selector_metadata = selector.metadata() + # What: call selector_metadata.update with strategy and selector and list and targets and type; why: openai_model_list invokes selector_metadata.update while performing type selector; the call advances that operation through its result or side effect. + selector_metadata.update({ + # What: map the type field as selector; why: openai_model_list carries type into "type": "selector". + "type": "selector", + # What: map the strategy field as strategy and selector; why: openai_model_list carries strategy into "strategy": selector.strategy. + "strategy": selector.strategy, + # What: map the targets field as list and targets and selector; why: openai_model_list carries targets into "targets": list(selector.targets). + "targets": list(selector.targets), + # What: complete the selector_metadata.update call with strategy; why: openai_model_list groups the supplied clauses as one selector_metadata.update call before its value is consumed. + }) + # What: map the freetoken field as selector metadata; why: openai_model_list carries freetoken through record entry into record name profile display name strip. + record["meta"] = {"freetoken": selector_metadata} + # What: gate on profile before display name and profile and record and strip; why: openai_model_list admits display name and profile and record and strip only for this predicate and excludes the opposite state. + if profile is not None: + # What: gate on display name and profile before record and strip and display name and profile; why: openai_model_list admits record and strip and display name and profile only for this predicate and excludes the opposite state. + if profile.display_name: + # What: compute record entry from strip and display name and profile; why: record description profile description strip later reads record entry, so openai_model_list must retain the computed value under that name. + record["name"] = profile.display_name.strip() + # What: gate on description and profile before record and strip and description and profile; why: openai_model_list admits record and strip and description and profile only for this predicate and excludes the opposite state. + if profile.description: + # What: compute record entry from strip and description and profile; why: record update capability fields later reads record entry, so openai_model_list must retain the computed value under that name. + record["description"] = profile.description.strip() + # What: compute capability fields from model listing fields and capabilities and profile; why: record update capability fields later reads capability fields, so openai_model_list must retain the computed value under that name. + capability_fields = profile.capabilities.model_listing_fields() + # What: call record.update with capability fields; why: openai_model_list invokes record.update while performing metadata profile metadata; the call advances that operation through its result or side effect. + record.update(capability_fields) + # What: compute metadata from metadata and profile; why: metadata pop key later reads metadata, so openai_model_list must retain the computed value under that name. + metadata = profile.metadata() + # What: gate on empty and capabilities and profile before key and pop and metadata; why: openai_model_list admits key and pop and metadata only for this predicate and excludes the opposite state. + if not profile.capabilities.empty(): + # What: iterate across the computed value to perform pop and key and metadata; why: openai_model_list repeats the body only while or for the loop header admits an iteration. + for key in ( + # What: apply the architecture capabilities supported parameters portion of the enclosing predicate; why: this clause remains in openai_model_list\'s enclosing expression so its grouping and evaluation order stay intact. + "architecture", "capabilities", "supported_parameters", + # What: apply the context length context window portion of the enclosing predicate; why: this clause remains in openai_model_list\'s enclosing expression so its grouping and evaluation order stay intact. + "context_length", "context_window", + # What: complete the enclosing predicate collection with architecture and capabilities and supported parameters and context length; why: openai_model_list groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. + ): + # What: call metadata.pop with key and the named fixture input; why: openai_model_list invokes metadata.pop while performing if model id profile name; the call advances that operation through its result or side effect. + metadata.pop(key, None) + # What: gate on model id and name and profile before internal metadata; why: openai_model_list admits internal metadata only for this predicate and excludes the opposite state. + if model_id == profile.name: + # What: map the type field as model; why: openai_model_list carries type through internal metadata into internal metadata aliases list profile aliases. + internal_metadata = {"type": "model"} + # What: gate on aliases and profile before internal metadata and list and aliases and profile; why: openai_model_list admits internal metadata and list and aliases and profile only for this predicate and excludes the opposite state. + if profile.aliases: + # What: compute internal metadata entry from list and aliases and profile; why: internal metadata type alias model id profile name later reads internal metadata entry, so openai_model_list must retain the computed value under that name. + internal_metadata["aliases"] = list(profile.aliases) + # What: select the remaining branch that performs internal metadata type alias model id profile name; why: openai_model_list covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: map the type field as alias; why: openai_model_list carries type through internal metadata into metadata update internal metadata. + internal_metadata = {"type": "alias", "modelID": profile.name} + # What: call metadata.update with internal metadata; why: openai_model_list invokes metadata.update while performing record setdefault meta freetoken metadata; the call advances that operation through its result or side effect. + metadata.update(internal_metadata) + # What: compute result entry from metadata; why: the enclosing return or state update later reads result entry, so openai_model_list must retain the computed value under that name. + record.setdefault("meta", {})["freetoken"] = metadata + # What: call data.append with record; why: openai_model_list invokes data.append while performing routing profile; the call advances that operation through its result or side effect. + data.append(record) + # What: compute routing profile from active routing profile and routing profile and catalog snapshot; why: catalog snapshot routing profile active routing profile later reads routing profile, so openai_model_list must retain the computed value under that name. + routing_profile = ( + # What: call catalog_snapshot.routing_profile with active routing profile; why: openai_model_list invokes catalog_snapshot.routing_profile while performing if active routing profile is not else; the call advances that operation through its result or side effect. + catalog_snapshot.routing_profile(active_routing_profile) + # What: apply the if active routing profile is not else portion of routing profile; why: openai_model_list uses this clause to evaluate routing profile as one grouped value. + if active_routing_profile is not None else None + # What: complete the routing_profile expression with routing profile catalog snapshot routing profile active routing profile if active routing profile is not else; why: openai_model_list groups the supplied clauses as one routing_profile expression before its value is consumed. + ) + # What: gate on routing profile before pins and pin and target and routing profile and append; why: openai_model_list admits pins and pin and target and routing profile and append only for this predicate and excludes the opposite state. + if routing_profile is not None: + # What: iterate across pins and routing profile to perform target and has routable id and pin and catalog snapshot; why: openai_model_list repeats the body only while or for the loop header admits an iteration. + for pin, target in routing_profile.pins: + # What: gate on target and has routable id and pin and catalog snapshot before the computed value; why: openai_model_list admits the computed value only for this predicate and excludes the opposite state. + if target is None or catalog_snapshot.has_routable_id(pin): + # What: apply the continue portion of the enclosing predicate; why: this clause remains in openai_model_list\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: call data.append with pin and created and id and object and created; why: openai_model_list invokes data.append while performing id pin; the call advances that operation through its result or side effect. + data.append({ + # What: map the id field as pin; why: openai_model_list carries id into "id": pin. + "id": pin, + # What: map the object field as model; why: openai_model_list carries object into "object": "model". + "object": "model", + # What: map the created field as created; why: openai_model_list carries created into "created": created. + "created": created, + # What: map the owned by field as freetoken; why: openai_model_list carries owned by into "owned_by": "freetoken". + "owned_by": "freetoken", + # What: map the value field as unloaded; why: openai_model_list carries value into "status": {"value": "unloaded"}. + "status": {"value": "unloaded"}, + # What: map the freetoken field as type and profile; why: openai_model_list carries freetoken into "meta": {"freetoken": {"type": "profile"}}. + "meta": {"freetoken": {"type": "profile"}}, + # What: complete the data.append call with pin; why: openai_model_list groups the supplied clauses as one data.append call before its value is consumed. + }) + # What: compute response from jsonresponse and data and object and data and list; why: response headers access control allow origin origin later reads response, so openai_model_list must retain the computed value under that name. + response = JSONResponse(content={ + # What: map the object field as list; why: openai_model_list carries object through response into response headers access control allow origin origin. + "object": "list", + # What: map the data field as data; why: openai_model_list carries data through response into response headers access control allow origin origin. + "data": data, + # What: complete the JSONResponse call with content; why: openai_model_list groups the supplied clauses as one JSONResponse call before its value is consumed. + }) + # What: gate on origin and get and headers and request before origin and headers and response; why: openai_model_list admits origin and headers and response only for this predicate and excludes the opposite state. + if origin := request.headers.get("origin"): + # What: compute headers entry from origin; why: the enclosing return or state update later reads headers entry, so openai_model_list must retain the computed value under that name. + response.headers["Access-Control-Allow-Origin"] = origin + # What: return response from openai_model_list; why: openai_model_list exposes response so its caller can continue with the function\'s computed outcome. + return response + + # What: apply app.api_route behavior to upstream_proxy; why: Python attaches this named decorator's registration or descriptor semantics to upstream_proxy. + @app.api_route( + # What: apply the upstream upstream path path portion of the enclosing predicate; why: this clause remains in upstream_proxy\'s enclosing expression so its grouping and evaluation order stay intact. + "/upstream/{upstream_path:path}", + # What: supply methods to app.api_route; why: upstream_proxy binds this get and post and put and patch and delete value to app.api_route's methods input. + methods=["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"], + # What: supply dependencies to Depends; why: upstream_proxy binds this depends and require router key value to Depends's dependencies input. + dependencies=[Depends(require_router_key)], + # What: complete the app.api_route call with methods and dependencies; why: upstream_proxy groups the supplied clauses as one app.api_route call before its value is consumed. + ) + # What: define upstream_proxy around request and upstream path; why: the registered API client call upstream_proxy for upstream proxy and rely on this exact input and result contract. + async def upstream_proxy(request: Request, upstream_path: str): + # What: establish the handler boundary for the protected operation; why: upstream_proxy routes failures to catalog error while preserving cleanup and success flow. + try: + # What: compute source model and model and and remaining path from resolve upstream path and upstream path and ro; why: upstream_proxy consumes source model and model and and remaining path during escaped path suffix raw path f upstream source model, so source model and model and and remaining path value r. + source_model, model, _, remaining_path = router.resolve_upstream_path( + # What: apply the upstream path portion of source model and model and and remaining path; why: upstream_proxy uses this clause to evaluate source model and model and and remaining path as one grouped value. + upstream_path + # What: complete the router.resolve_upstream_path call with upstream path; why: upstream_proxy groups the supplied clauses as one router.resolve_upstream_path call before its value is consumed. + ) + # What: handle catalog error by return jsonresponse; why: upstream_proxy converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError as exc: + # What: return jsonresponse and str and exc and 404 and error from upstream_proxy; why: upstream_proxy exposes jsonresponse and str and exc and 404 and error so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: upstream_proxy binds this 404 value to JSONResponse's status code input. + status_code=404, + # What: map the error field as str and exc and message and type and unknown model; why: upstream_proxy carries error into content={"error": {"message": str(exc), "type": "unknown_model"}}. + content={"error": {"message": str(exc), "type": "unknown_model"}}, + # What: complete the JSONResponse call with status code and content; why: upstream_proxy groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + # The daemon alone may call prepare-stop. Exposing it through an + # arbitrary passthrough would bypass durable accounting and leave a + # misleading routing lease behind. + # What: compute normalized from lstrip and remaining path and value; why: if normalized v1 admin prepare stop later reads normalized, so upstream_proxy must retain the computed value under that name. + normalized = remaining_path.lstrip("/") + # What: gate on normalized before httpexception; why: upstream_proxy admits httpexception only for this predicate and excludes the opposite state. + if normalized == "v1/admin/prepare-stop": + # What: raise HTTPException for the caller; why: upstream_proxy stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=403, detail="upstream prepare-stop is daemon-managed") + # What: gate on any and profile is resident and model and endswith and suffix before jsonresponse and model; why: upstream_proxy admits jsonresponse and model only for this predicate and excludes the opposite state. + if ( + # What: call any with endswith and suffix and upstream no activation suffixes and remaining path; why: upstream_proxy invokes any while performing remaining path endswith suffix; the call advances that operation through its result or side effect. + any( + # What: call remaining_path.endswith with suffix; why: upstream_proxy invokes remaining_path.endswith while performing for suffix in router catalog settings upstream no activation suffixes; the call advances that operation through its result or side effect. + remaining_path.endswith(suffix) + # What: apply the for suffix in router catalog settings upstream no activation suffixes portion of the enclosing predicate; why: this clause remains in upstream_proxy\'s enclosing expression so its grouping and evaluation order stay intact. + for suffix in router.catalog.settings.upstream_no_activation_suffixes + # What: complete the any call with endswith; why: upstream_proxy groups the supplied clauses as one any call before its value is consumed. + ) + # What: call router.profile_is_resident with model; why: upstream_proxy consumes the router.profile_is_resident return value while evaluating and not router.profile_is_resident(model). + and not router.profile_is_resident(model) + # What: complete the enclosing predicate with if any remaining path endswith suffix for suffix in router catalog settings upstream no activation suffixes and; why: upstream_proxy groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): return JSONResponse( status_code=409, content={ - "error": str(exc), - "code": "serve_conflict", - "currentModel": st.get("model"), - "currentPort": st.get("port"), + # What: apply the error portion of the enclosing predicate; why: this clause remains in upstream_proxy\'s enclosing expression so its grouping and evaluation order stay intact. + "error": { + # What: map the message field as model and model and is and not and loaded; why: upstream_proxy carries message into "message": (. + "message": ( + # What: embed the exact f model model r is not router-interface fragment; why: the router UI consumer receives this fragment verbatim through f"model {model!r} is not loaded; path matches ", preserving browser markup, style, or script behavior. + # What: preserve the exact router upstream no activation suffixes literal fragment; why: upstream_proxy passes this fragment verbatim through f"model {model!r} is not loaded; path matches ", because changing it would alter a protocol payload, serialized fixture, or public message. + f"model {model!r} is not loaded; path matches " + "router.upstream_no_activation_suffixes" + # What: complete the enclosing predicate mapping with message and type; why: upstream_proxy groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + ), + # What: map the type field as model not loaded; why: upstream_proxy carries type into "type": "model_not_loaded". + "type": "model_not_loaded", + # What: complete the enclosing predicate mapping with message and type; why: upstream_proxy groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } }, ) - except Exception as exc: # noqa: BLE001 — never propagate a 500-as-crash - raise HTTPException(status_code=500, detail=f"start failed: {exc}") + # What: compute raw path from get and scope and request and raw path; why: escaped path suffix raw path f upstream source model later reads raw path, so upstream_proxy must retain the computed value under that name. + raw_path = request.scope.get("raw_path") + # What: compute escaped path from isinstance and raw path and bytes and escaped path suffix; why: if escaped path is later reads escaped path, so upstream_proxy must retain the computed value under that name. + escaped_path = ( + # What: call _escaped_path_suffix with raw path and source model and upstream; why: upstream_proxy invokes _escaped_path_suffix while performing if isinstance raw path bytes else; the call advances that operation through its result or side effect. + _escaped_path_suffix(raw_path, f"/upstream/{source_model}") + # What: call isinstance with raw path and bytes; why: upstream_proxy consumes the isinstance return value while evaluating if isinstance(raw_path, bytes) else None. + if isinstance(raw_path, bytes) else None + # What: complete the escaped_path expression with escaped path escaped path suffix raw path f upstream source model if isinstance raw path; why: upstream_proxy groups the supplied clauses as one escaped_path expression before its value is consumed. + ) + # What: gate on escaped path before httpexception; why: upstream_proxy admits httpexception only for this predicate and excludes the opposite state. + if escaped_path is None: + # What: raise HTTPException for the caller; why: upstream_proxy stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=400, detail="invalid escaped upstream path") + # What: gate on escaped path before escaped path; why: upstream_proxy admits escaped path only for this predicate and excludes the opposite state. + if not escaped_path: + # What: compute escaped path from value; why: path and query escaped path suffix later reads escaped path, so upstream_proxy must retain the computed value under that name. + escaped_path = "/" + # What: compute raw query from get and scope and request and query string; why: suffix f raw query decode ascii if raw query later reads raw query, so upstream_proxy must retain the computed value under that name. + raw_query = request.scope.get("query_string", b"") + # What: compute suffix from raw query and decode and value and value and ascii; why: path and query escaped path suffix later reads suffix, so upstream_proxy must retain the computed value under that name. + suffix = f"?{raw_query.decode('ascii')}" if raw_query else "" + # What: return forward routed and request and model and escaped path from upstream_proxy; why: upstream_proxy exposes forward routed and request and model and escaped path so its caller can continue with the function\'s computed outcome. + return await forward_routed( + # What: apply the request portion of the enclosing predicate; why: this clause remains in upstream_proxy\'s enclosing expression so its grouping and evaluation order stay intact. + request, + # What: apply the model portion of the enclosing predicate; why: this clause remains in upstream_proxy\'s enclosing expression so its grouping and evaluation order stay intact. + model, + # What: supply path and query to forward_routed; why: upstream_proxy binds this escaped path and suffix value to forward_routed's path and query input. + path_and_query=escaped_path + suffix, + # What: supply body to request.body; why: upstream_proxy binds this body and request value to request.body's body input. + body=await request.body(), + # What: supply apply request filters to operation.lower; why: upstream_proxy binds this lower and get and headers and request and application value to operation.lower's apply request filters input. + apply_request_filters="application/json" in request.headers.get( + # What: apply the content type portion of the enclosing predicate; why: this clause remains in upstream_proxy\'s enclosing expression so its grouping and evaluation order stay intact. + "content-type", "" + # What: apply the lower portion of the enclosing predicate; why: this clause remains in upstream_proxy\'s enclosing expression so its grouping and evaluation order stay intact. + ).lower(), + # What: complete the forward_routed call with path and query and body and apply request filters; why: upstream_proxy groups the supplied clauses as one forward_routed call before its value is consumed. + ) - @app.post("/engine/stop", dependencies=auth) - async def engine_stop(body: StopBody | None = None): + # What: register GET /router/status on the application router; why: clients reach router_status's handler only through this method-and-path binding. + @app.get("/router/status", dependencies=auth) + # What: define router_status around the current object state; why: the registered API client call router_status for router status and rely on this exact input and result contract. + async def router_status(): + # What: map the catalog watch field as catalog watch snapshot; why: router_status carries catalog watch into return {**router.status(), "catalogWatch": catalog_watch_snapshot()}. + return {**router.status(), "catalogWatch": catalog_watch_snapshot()} + + # What: register GET /router/models on the application router; why: clients reach router_models's handler only through this method-and-path binding. + @app.get("/router/models", dependencies=auth) + # What: define router_models around the current object state; why: the registered API client call router_models for router models and rely on this exact input and result contract. + async def router_models(): + """Configured profiles annotated with the sole engine's live residency.""" + # What: document configured profiles annotated with the sole in the router_models docstring; why: introspection and maintainers read this exact docstring fragment to understand router models behavior without executing it. + # What: compute catalog snapshot and route state from control plane snapshot and router; why: for profile in catalog snapshot public later reads catalog snapshot and route state, so router_models must retain the computed value under that name. + catalog_snapshot, route_state = router.control_plane_snapshot() + # What: compute engine from status and manager; why: profile name active and bool engine get later reads engine, so router_models must retain the computed value under that name. + engine = manager.status() + # What: compute active from route state and active profile; why: profile name active and bool engine get later reads active, so router_models must retain the computed value under that name. + active = route_state["activeProfile"] + # What: compute active identity matches from route state and active identity matches engine; why: profile name active and bool engine get later reads active identity matches, so router_models must retain the computed value under that name. + active_identity_matches = route_state["activeIdentityMatchesEngine"] + # What: initialize data as an empty runtime accumulator; why: router_models appends or maps entries into it during data append profile before consuming the aggregate. + data = [] + # What: iterate across public and catalog snapshot to perform profile and dict; why: router_models repeats the body only while or for the loop header admits an iteration. + for profile in catalog_snapshot.public(): + # What: compute profile from dict and profile; why: profile configured later reads profile, so router_models must retain the computed value under that name. + profile = dict(profile) + # What: compute profile entry from true; why: profile resident later reads profile entry, so router_models must retain the computed value under that name. + profile["configured"] = True + # What: compute profile entry from active identity matches and active and bool and profile; why: profile name active and bool engine get later reads profile entry, so router_models must retain the computed value under that name. + profile["resident"] = ( + # What: call bool with get and engine and running; why: router_models consumes the bool return value while evaluating profile["name"] == active and bool(engine.get("running")) and active_ide. + profile["name"] == active and bool(engine.get("running")) and active_identity_matches + # What: complete the profile entry expression with profile resident profile name equals active and bool engine get; why: router_models groups the supplied clauses as one profile entry expression before its value is consumed. + ) + # What: compute profile entry from profile and route state and 0 and resident and active requests; why: data append profile later reads profile entry, so router_models must retain the computed value under that name. + profile["activeRequests"] = route_state["activeRequests"] if profile["resident"] else 0 + # What: call data.append with profile; why: router_models invokes data.append while performing return; the call advances that operation through its result or side effect. + data.append(profile) + # What: return data and public selectors and public routing profiles and route state from router_models; why: router_models exposes data and public selectors and public routing profiles and route state so its caller can continue with the function\'s computed outcome. + return { + # What: map the data field as data; why: router_models carries data into "data": data. + "data": data, + # What: map the selectors field as public selectors and catalog snapshot; why: router_models carries selectors into "selectors": catalog_snapshot.public_selectors(). + "selectors": catalog_snapshot.public_selectors(), + # What: map the routing profiles field as public routing profiles and catalog snapshot; why: router_models carries routing profiles into "routingProfiles": catalog_snapshot.public_routing_profiles(). + "routingProfiles": catalog_snapshot.public_routing_profiles(), + # What: map the active routing profile field as route state and active routing profile; why: router_models carries active routing profile into "activeRoutingProfile": route_state["activeRoutingProfile"]. + "activeRoutingProfile": route_state["activeRoutingProfile"], + # What: map the capacity field as route state and capacity; why: router_models carries capacity into "capacity": route_state["capacity"]. + "capacity": route_state["capacity"], + # What: complete the enclosing predicate mapping with data and selectors and routing profiles and active routing profile and capacity; why: router_models groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + # What: register GET /router/profiles on the application router; why: clients reach router_profiles's handler only through this method-and-path binding. + @app.get("/router/profiles", dependencies=auth) + # What: define router_profiles around the current object state; why: the registered API client call router_profiles for router profiles and rely on this exact input and result contract. + async def router_profiles(): + # What: compute catalog snapshot and route state from control plane snapshot and router; why: data catalog snapshot public later reads catalog snapshot and route state, so router_profiles must retain the computed value under that name. + catalog_snapshot, route_state = router.control_plane_snapshot() + # What: return public and public selectors and public routing profiles and route state from router_profiles; why: router_profiles exposes public and public selectors and public routing profiles and route state so its caller can continue with the function\'s computed outcome. + return { + # What: map the data field as public and catalog snapshot; why: router_profiles carries data into "data": catalog_snapshot.public(). + "data": catalog_snapshot.public(), + # What: map the selectors field as public selectors and catalog snapshot; why: router_profiles carries selectors into "selectors": catalog_snapshot.public_selectors(). + "selectors": catalog_snapshot.public_selectors(), + # What: map the routing profiles field as public routing profiles and catalog snapshot; why: router_profiles carries routing profiles into "routingProfiles": catalog_snapshot.public_routing_profiles(). + "routingProfiles": catalog_snapshot.public_routing_profiles(), + # What: map the active routing profile field as route state and active routing profile; why: router_profiles carries active routing profile into "activeRoutingProfile": route_state["activeRoutingProfile"]. + "activeRoutingProfile": route_state["activeRoutingProfile"], + # What: map the active profile field as route state and active profile; why: router_profiles carries active profile into "activeProfile": route_state["activeProfile"]. + "activeProfile": route_state["activeProfile"], + # What: complete the enclosing predicate mapping with data and selectors and routing profiles and active routing profile and active profile; why: router_profiles groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + # What: register PUT /router/profiles/active on the application router; why: clients reach set_active_routing_profile's handler only through this method-and-path binding. + @app.put("/router/profiles/active", dependencies=auth) + # What: define set_active_routing_profile around body; why: the registered API client call set_active_routing_profile for set active routing profile and rely on this exact input and result contract. + async def set_active_routing_profile(body: RoutingProfileSelectionBody): + # What: establish the handler boundary for the protected operation; why: set_active_routing_profile routes failures to routing error while preserving cleanup and success flow. + try: + # What: compute active from set active routing profile and name and router and body; why: router event routing profile changed routing profile active later reads active, so set_active_routing_profile must retain the computed value under that name. + active = router.set_active_routing_profile(body.name) + # What: handle routing error by return jsonresponse; why: set_active_routing_profile converts that failure into this concrete recovery, response, or cleanup behavior. + except RoutingError as exc: + # What: return jsonresponse and status code and exc and code from set_active_routing_profile; why: set_active_routing_profile exposes jsonresponse and status code and exc and code so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: set_active_routing_profile binds this status code and exc value to JSONResponse's status code input. + status_code=exc.status_code, + # What: map the error field as code and str and exc and message and type; why: set_active_routing_profile carries error into content={"error": {"message": str(exc), "type": exc.code}}. + content={"error": {"message": str(exc), "type": exc.code}}, + # What: complete the JSONResponse call with status code and content; why: set_active_routing_profile groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + # What: preserve the exact router event routing profile changed routing profile active literal fragment; why: set_active_routing_profile passes this fragment verbatim through router_event("routing_profile_changed", routingProfile=active), because changing it would alter a protocol payload, serialized fixture. + router_event("routing_profile_changed", routingProfile=active) + # What: map the active field as active; why: set_active_routing_profile carries active into return {"active": active}. + return {"active": active} + + # What: register GET /router/hardware on the application router; why: clients reach router_hardware's handler only through this method-and-path binding. + @app.get("/router/hardware", dependencies=auth) + # What: define router_hardware around the current object state; why: the registered API client call router_hardware for router hardware and rely on this exact input and result contract. + async def router_hardware(): + """Small, privacy-preserving local memory view for the management UI.""" + # What: document small privacy preserving local memory view for in the router_hardware docstring; why: introspection and maintainers read this exact docstring fragment to understand router hardware behavior without executing it. + # What: compute engine from status and manager; why: footprint await run proxy pool footprint fn engine get later reads engine, so router_hardware must retain the computed value under that name. + engine = manager.status() + # What: compute footprint from run and proxy pool and footprint fn and get; why: memory footprint later reads footprint, so router_hardware must retain the computed value under that name. + footprint = await run(proxy_pool, footprint_fn, engine.get("pid")) + # What: return footprint and bool and get and engine and engine from router_hardware; why: router_hardware exposes footprint and bool and get and engine and engine so its caller can continue with the function\'s computed outcome. + return { + # What: apply the engine portion of the enclosing predicate; why: this clause remains in router_hardware\'s enclosing expression so its grouping and evaluation order stay intact. + "engine": { + # What: map the running field as bool and get and engine and running; why: router_hardware carries running into "running": bool(engine.get("running")). + "running": bool(engine.get("running")), + # What: map the pid field as get and engine and pid; why: router_hardware carries pid into "pid": engine.get("pid"). + "pid": engine.get("pid"), + # What: map the port field as get and engine and port; why: router_hardware carries port into "port": engine.get("port"). + "port": engine.get("port"), + # What: complete the enclosing predicate mapping with running and pid and port; why: router_hardware groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + }, + # What: map the memory field as footprint; why: router_hardware carries memory into "memory": footprint. + "memory": footprint, + # What: complete the enclosing predicate mapping with engine and memory; why: router_hardware groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + # What: register GET /api/performance on the application router; why: clients reach router_performance's handler only through this method-and-path binding. + @app.get("/api/performance", dependencies=auth) + # What: register GET /router/performance on the application router; why: clients reach router_performance's handler only through this method-and-path binding. + @app.get("/router/performance", dependencies=auth) + # What: define router_performance around after; why: the registered API client call router_performance for router performance and rely on this exact input and result contract. + async def router_performance(after: str | None = Query(default=None)): + # What: compute parsed after from the named fixture input; why: parsed after datetime fromisoformat after replace z later reads parsed after, so router_performance must retain the computed value under that name. + parsed_after = None + # What: gate on after before fullmatch and after and httpexception and re; why: router_performance admits fullmatch and after and httpexception and re only for this predicate and excludes the opposite state. + if after is not None: + # What: gate on fullmatch and after and re before httpexception; why: router_performance admits httpexception only for this predicate and excludes the opposite state. + if not re.fullmatch( + # What: apply the r d d d t d portion of the enclosing predicate; why: this clause remains in router_performance\'s enclosing expression so its grouping and evaluation order stay intact. + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})", + # What: apply the after portion of the enclosing predicate; why: this clause remains in router_performance\'s enclosing expression so its grouping and evaluation order stay intact. + after, + # What: complete the re.fullmatch call with after; why: router_performance groups the supplied clauses as one re.fullmatch call before its value is consumed. + ): + # What: raise HTTPException for the caller; why: router_performance stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException( + # What: supply status code to HTTPException; why: router_performance binds this 400 value to HTTPException's status code input. + status_code=400, detail="invalid 'after' timestamp, use RFC3339 format" + # What: complete the HTTPException call with status code and detail; why: router_performance groups the supplied clauses as one HTTPException call before its value is consumed. + ) + # What: establish the handler boundary for the protected operation; why: router_performance routes failures to value error while preserving cleanup and success flow. + try: + # What: compute parsed after from fromisoformat and datetime and replace and after and z; why: result performance monitor current after parsed after later reads parsed after, so router_performance must retain the computed value under that name. + parsed_after = datetime.fromisoformat(after.replace("Z", "+00:00")) + # What: handle value error by raise httpexception; why: router_performance converts that failure into this concrete recovery, response, or cleanup behavior. + except ValueError as exc: + # What: raise HTTPException for the caller; why: router_performance stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException( + # What: supply status code to HTTPException; why: router_performance binds this 400 value to HTTPException's status code input. + status_code=400, detail="invalid 'after' timestamp, use RFC3339 format" + # What: apply the from exc portion of the enclosing predicate; why: this clause remains in router_performance\'s enclosing expression so its grouping and evaluation order stay intact. + ) from exc + # What: compute result from current and performance monitor and parsed after; why: if not result enabled later reads result, so router_performance must retain the computed value under that name. + result = performance_monitor.current(after=parsed_after) + # What: gate on result before jsonresponse; why: router_performance admits jsonresponse only for this predicate and excludes the opposite state. + if not result["enabled"]: + # What: map the enabled field as false; why: router_performance carries enabled into return JSONResponse(status_code=503, content={"enabled": False}). + return JSONResponse(status_code=503, content={"enabled": False}) + # What: return result from router_performance; why: router_performance exposes result so its caller can continue with the function\'s computed outcome. + return result + + # What: register GET /router/requests on the application router; why: clients reach router_requests's handler only through this method-and-path binding. + @app.get("/router/requests", dependencies=auth) + # What: define router_requests around the current object state; why: the registered API client call router_requests for router requests and rely on this exact input and result contract. + async def router_requests(): + # What: enter the inflight lock managed context before data id request id profile item profile; why: router_requests releases this resource or lock after data id request id profile item profile on both success and failure paths. + with inflight_lock: + # What: map the id field as request id; why: router_requests carries id through data into return data data. + data = [{"id": request_id, "profile": item["profile"]} + # What: call request_reservations.items with the declared inputs; why: router_requests invokes request_reservations.items while performing return data data; the call advances that operation through its result or side effect. + for request_id, item in request_reservations.items()] + # What: map the data field as data; why: router_requests carries data into return {"data": data}. + return {"data": data} + + # What: register POST /router/requests/{request_id}/cancel on the application router; why: clients reach router_cancel's handler only through this method-and-path binding. + @app.post("/router/requests/{request_id}/cancel", dependencies=auth) + # What: define router_cancel around request id; why: the registered API client call router_cancel for router cancel and rely on this exact input and result contract. + async def router_cancel(request_id: str): + # What: enter the inflight lock managed context before item inflight get request id; why: router_cancel releases this resource or lock after item inflight get request id on both success and failure paths. + with inflight_lock: + # What: compute item from get and request id and inflight; why: if item is not and cancellation later reads item, so router_cancel must retain the computed value under that name. + item = inflight.get(request_id) + # What: compute reservation from get and request id and request reservations; why: if reservation is not and not later reads reservation, so router_cancel must retain the computed value under that name. + reservation = request_reservations.get(request_id) + # What: gate on reservation before reservation; why: router_cancel admits reservation only for this predicate and excludes the opposite state. + if reservation is not None and not reservation["cancelled"]: + # What: compute reservation entry from true; why: cancellation reservation cancellation later reads reservation entry, so router_cancel must retain the computed value under that name. + reservation["cancelled"] = True + # What: compute cancellation from reservation and cancellation; why: cancellation later reads cancellation, so router_cancel must retain the computed value under that name. + cancellation = reservation["cancellation"] + # What: select the remaining branch that performs cancellation; why: router_cancel covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: compute cancellation from the named fixture input; why: if item is not and cancellation later reads cancellation, so router_cancel must retain the computed value under that name. + cancellation = None + # What: gate on item and cancellation before item; why: router_cancel admits item only for this predicate and excludes the opposite state. + if item is not None and cancellation is not None: + # What: compute item entry from true; why: if item is later reads item entry, so router_cancel must retain the computed value under that name. + item["cancelled"] = True + # What: gate on cancellation before the computed value; why: router_cancel admits the computed value only for this predicate and excludes the opposite state. + if cancellation is None: + # What: map the cancelled field as false; why: router_cancel carries cancelled into return {"cancelled": False, "reason": "not_found"}. + return {"cancelled": False, "reason": "not_found"} + # What: gate on item before cancel acquire and cancellation and router; why: router_cancel admits cancel acquire and cancellation and router only for this predicate and excludes the opposite state. + if item is None: + # What: call router.cancel_acquire with cancellation; why: router_cancel invokes router.cancel_acquire while performing profile reservation profile; the call advances that operation through its result or side effect. + router.cancel_acquire(cancellation) + # What: compute profile from reservation and profile; why: profile item profile later reads profile, so router_cancel must retain the computed value under that name. + profile = reservation["profile"] + # What: select the remaining branch that performs item upstream close; why: router_cancel covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: preserve the exact item upstream close literal fragment; why: router_cancel passes this fragment verbatim through item["upstream"].close(), because changing it would alter a protocol payload, serialized fixture, or public message. + item["upstream"].close() + # What: compute profile from item and profile; why: router event request cancelled profile profile later reads profile, so router_cancel must retain the computed value under that name. + profile = item["profile"] + # What: call router.record_cancellation with the declared inputs; why: router_cancel invokes router.record_cancellation while performing router event request cancelled profile profile; the call advances that operation through its result or side effect. + router.record_cancellation() + # What: preserve the exact router event request cancelled profile profile literal fragment; why: router_cancel passes this fragment verbatim through router_event("request_cancelled", profile=profile), because changing it would alter a protocol payload, serialized fixture, or public message. + router_event("request_cancelled", profile=profile) + # What: map the cancelled field as true; why: router_cancel carries cancelled into return {"cancelled": True, "id": request_id}. + return {"cancelled": True, "id": request_id} + + # What: register GET /router/logs on the application router; why: clients reach router_logs's handler only through this method-and-path binding. + @app.get("/router/logs", dependencies=auth) + # What: define router_logs around request and since; why: the registered API client call router_logs for router logs and rely on this exact input and result contract. + async def router_logs(request: Request, since: int = 0): + """Bounded lifecycle/proxy event stream, separate from engine stdout.""" + # What: document bounded lifecycle proxy event stream separate in the router_logs docstring; why: introspection and maintainers read this exact docstring fragment to understand router logs behavior without executing it. + # What: return log stream and request and router ring and since from router_logs; why: router_logs exposes log stream and request and router ring and since so its caller can continue with the function\'s computed outcome. + return _log_stream(request, router_ring, since) + + # What: register GET /router/activity on the application router; why: clients reach router_activity's handler only through this method-and-path binding. + @app.get("/router/activity", dependencies=auth) + # What: define router_activity around limit and before id and model; why: the registered API client call router_activity for router activity and rely on this exact input and result contract. + async def router_activity( + # What: declare the limit input for router_activity; why: router_activity consumes limit during return activity store list limit limit before id before id, so callers must bind it with the other signature inputs. + limit: int = Query(default=100, ge=1, le=999), + # What: declare the before id input for router_activity; why: router_activity consumes before id during return activity store list limit limit before id before id, so callers must bind it with the other signature inputs. + before_id: int | None = Query(default=None, ge=1, alias="beforeId"), + # What: declare the model input for router_activity; why: router_activity consumes model during return activity store list limit limit before id before id, so callers must bind it with the other signature inputs. + model: str | None = None, + # What: complete the enclosing predicate with app get router activity dependencies auth async def router activity limit; why: router_activity groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: return list and activity store and limit and before id from router_activity; why: router_activity exposes list and activity store and limit and before id so its caller can continue with the function\'s computed outcome. + return activity_store.list(limit=limit, before_id=before_id, model=model) + + # What: register GET /router/activity/stats on the application router; why: clients reach router_activity_stats's handler only through this method-and-path binding. + @app.get("/router/activity/stats", dependencies=auth) + # What: define router_activity_stats around model; why: the registered API client call router_activity_stats for router activity stats and rely on this exact input and result contract. + async def router_activity_stats(model: str | None = None): + # What: return stats and activity store and model from router_activity_stats; why: router_activity_stats exposes stats and activity store and model so its caller can continue with the function\'s computed outcome. + return activity_store.stats(model=model) + + # What: register GET /router/captures/{capture_id} on the application router; why: clients reach router_capture's handler only through this method-and-path binding. + @app.get("/router/captures/{capture_id}", dependencies=auth) + # What: define router_capture around capture id; why: the registered API client call router_capture for router capture and rely on this exact input and result contract. + async def router_capture(capture_id: int): + # What: compute capture from capture and capture id and activity store; why: if capture is later reads capture, so router_capture must retain the computed value under that name. + capture = activity_store.capture(capture_id) + # What: gate on capture before httpexception; why: router_capture admits httpexception only for this predicate and excludes the opposite state. + if capture is None: + # What: raise HTTPException for the caller; why: router_capture stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=404, detail="capture not found") + # What: return capture from router_capture; why: router_capture exposes capture so its caller can continue with the function\'s computed outcome. + return capture + + # What: register GET /metrics on the application router; why: clients reach router_metrics's handler only through this method-and-path binding. + @app.get("/metrics", dependencies=auth) + # What: define router_metrics around the current object state; why: the registered API client call router_metrics for router metrics and rely on this exact input and result contract. + async def router_metrics(): + # What: return plain text response and prometheus and router and text and plain from router_metrics; why: router_metrics exposes plain text response and prometheus and router and text and plain so its caller can continue with the function\'s computed outcome. + return PlainTextResponse(router.prometheus(), media_type="text/plain; version=0.0.4") + + # What: register POST /router/unload on the application router; why: clients reach router_unload's handler only through this method-and-path binding. + @app.post("/router/unload", dependencies=auth) + # What: define router_unload around body; why: the registered API client call router_unload for router unload and rely on this exact input and result contract. + async def router_unload(body: RouterUnloadBody | None = None): try: - return await run(lifecycle_pool, manager.stop, None, bool(body and body.force)) + # What: compute unloaded from run and lifecycle pool and evict idle and router; why: return unloaded unloaded router router status later reads unloaded, so router_unload must retain the computed value under that name. + unloaded = await run(lifecycle_pool, router.evict_idle, body.name if body else None) except (AccountingPrepareError, AccountingOutboxError) as exc: return accounting_error(exc) + # What: map the unloaded field as unloaded; why: router_unload carries unloaded into return {"unloaded": unloaded, "router": router.status()}. + return {"unloaded": unloaded, "router": router.status()} + + # What: register POST /router/load on the application router; why: clients reach router_load's handler only through this method-and-path binding. + @app.post("/router/load", dependencies=auth) + # What: define router_load around body; why: the registered API client call router_load for router load and rely on this exact input and result contract. + async def router_load(body: RouterLoadBody): + """Activate one profile without inventing a synthetic inference request. + + The short lease still uses the identical admission, readiness, switch, + accounting, and rollback transaction as automatic routing. Releasing it + afterwards permits the configured idle-TTL policy to apply normally. + """ + # What: document activate one profile without inventing a in the router_load docstring; why: introspection and maintainers read this exact docstring fragment to understand router load behavior without executing it. + # What: document the short lease still uses the in the router_load docstring; why: introspection and maintainers read this exact docstring fragment to understand router load behavior without executing it. + # What: document accounting and rollback transaction as automatic in the router_load docstring; why: introspection and maintainers read this exact docstring fragment to understand router load behavior without executing it. + # What: document afterwards permits the configured idle ttl policy in the router_load docstring; why: introspection and maintainers read this exact docstring fragment to understand router load behavior without executing it. + # What: preserve the paragraph boundary in the the router_load docstring; why: introspection and maintainers read this paragraph break to understand router load behavior without executing it. + # What: establish the handler boundary for the protected operation; why: router_load routes failures to routing error while preserving cleanup and success flow. + try: + # What: compute lease from acquire route and name and body and false; why: result profile lease profile name port lease port pid later reads lease, so router_load must retain the computed value under that name. + lease = await acquire_route(body.name, apply_routing_profile=False) + # What: handle routing error by router event management load failed profile body name code exc code; why: router_load converts that failure into this concrete recovery, response, or cleanup behavior. + except RoutingError as exc: + # What: preserve the exact router event management load failed profile body name code exc code literal fragment; why: router_load passes this fragment verbatim through router_event("management_load_failed", profile=body.name, code=exc.code), because changing it would alter a protocol payload, serialized fi. + router_event("management_load_failed", profile=body.name, code=exc.code) + # What: map the error field as code and str and exc and message and type; why: router_load carries error through content into content recovery exc recovery. + content = {"error": {"message": str(exc), "type": exc.code}} + # What: gate on recovery and exc before recovery and content and exc; why: router_load admits recovery and content and exc only for this predicate and excludes the opposite state. + if exc.recovery is not None: + # What: compute content entry from recovery and exc; why: content content later reads content entry, so router_load must retain the computed value under that name. + content["recovery"] = exc.recovery + # What: return jsonresponse and status code and content and exc and 429 from router_load; why: router_load exposes jsonresponse and status code and content and exc and 429 so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: router_load binds this status code and exc value to JSONResponse's status code input. + status_code=exc.status_code, + # What: supply content to JSONResponse; why: router_load binds this content value to JSONResponse's content input. + content=content, + # What: map the retry after field as 1; why: router_load carries retry after into headers={"Retry-After": "1"} if exc.status_code == 429 else None. + headers={"Retry-After": "1"} if exc.status_code == 429 else None, + # What: complete the JSONResponse call with status code and content and headers; why: router_load groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + # What: establish the handler boundary for the protected operation; why: router_load routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: map the profile field as name and profile and lease; why: router_load carries profile through result into return result router router status. + result = {"profile": lease.profile.name, "port": lease.port, "pid": lease.pid} + # What: run lease release on every exit path; why: router_load performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: call lease.release with the declared inputs; why: router_load invokes lease.release while performing router event management loaded profile lease profile name; the call advances that operation through its result or side effect. + lease.release() + # What: preserve the exact router event management loaded profile lease profile name literal fragment; why: router_load passes this fragment verbatim through router_event("management_loaded", profile=lease.profile.name), because changing it would alter a protocol payload, serialized fixture, or public message. + router_event("management_loaded", profile=lease.profile.name) + # What: map the router field as status and router; why: router_load carries router into return {**result, "router": router.status()}. + return {**result, "router": router.status()} + + # What: register POST /router/reload on the application router; why: clients reach router_reload's handler only through this method-and-path binding. + @app.post("/router/reload", dependencies=auth) + # What: define router_reload around the current object state; why: the registered API client call router_reload for router reload and rely on this exact input and result contract. + async def router_reload(): + # What: gate on catalog path before httpexception; why: router_reload admits httpexception only for this predicate and excludes the opposite state. + if not catalog_path: + # What: raise HTTPException for the caller; why: router_reload stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=409, detail="catalog reload requires --catalog") + # What: establish the handler boundary for the protected operation; why: router_reload routes failures to catalog error and routing error while preserving cleanup and success flow. + try: + # What: compute replacement from run and proxy pool and load and catalog path; why: await run lifecycle pool router replace catalog replacement later reads replacement, so router_reload must retain the computed value under that name. + replacement = await run(proxy_pool, ModelCatalog.load, catalog_path) + # What: call run with lifecycle pool and replace catalog and router and replacement; why: router_reload invokes run while performing await run; the call advances that operation through its result or side effect. + await run(lifecycle_pool, router.replace_catalog, replacement) + # What: call run with proxy pool and reconfigure and activity store and activity max entries and settings and replacement; why: router_reload invokes run while performing proxy pool; the call advances that operation through its result or side effect. + await run( + # What: apply the proxy pool portion of the enclosing predicate; why: this clause remains in router_reload\'s enclosing expression so its grouping and evaluation order stay intact. + proxy_pool, + # What: apply the activity store reconfigure portion of the enclosing predicate; why: this clause remains in router_reload\'s enclosing expression so its grouping and evaluation order stay intact. + activity_store.reconfigure, + # What: apply the replacement settings activity max entries portion of the enclosing predicate; why: this clause remains in router_reload\'s enclosing expression so its grouping and evaluation order stay intact. + replacement.settings.activity_max_entries, + # What: apply the replacement settings capture buffer mb portion of the enclosing predicate; why: this clause remains in router_reload\'s enclosing expression so its grouping and evaluation order stay intact. + replacement.settings.capture_buffer_mb * 1024 * 1024, + # What: apply the replacement settings activity session headers portion of the enclosing predicate; why: this clause remains in router_reload\'s enclosing expression so its grouping and evaluation order stay intact. + replacement.settings.activity_session_headers, + # What: complete the run call with proxy pool and reconfigure and activity max entries and capture buffer mb and activity session headers; why: router_reload groups the supplied clauses as one run call before its value is consumed. + ) + # What: call run with proxy pool and reconfigure and performance monitor and performance every s and settings and replacement; why: router_reload invokes run while performing proxy pool; the call advances that operation through its result or side effect. + await run( + # What: apply the proxy pool portion of the enclosing predicate; why: this clause remains in router_reload\'s enclosing expression so its grouping and evaluation order stay intact. + proxy_pool, + # What: apply the performance monitor reconfigure portion of the enclosing predicate; why: this clause remains in router_reload\'s enclosing expression so its grouping and evaluation order stay intact. + performance_monitor.reconfigure, + # What: apply the replacement settings performance every s portion of the enclosing predicate; why: this clause remains in router_reload\'s enclosing expression so its grouping and evaluation order stay intact. + replacement.settings.performance_every_s, + # What: apply the replacement settings performance disabled portion of the enclosing predicate; why: this clause remains in router_reload\'s enclosing expression so its grouping and evaluation order stay intact. + replacement.settings.performance_disabled, + # What: complete the run call with proxy pool and reconfigure and performance every s and performance disabled; why: router_reload groups the supplied clauses as one run call before its value is consumed. + ) + # What: handle catalog error by raise httpexception status code 400 detail str exc; why: router_reload converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError as exc: + # What: raise HTTPException for the caller; why: router_reload stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=400, detail=str(exc)) from exc + # What: handle routing error by return jsonresponse; why: router_reload converts that failure into this concrete recovery, response, or cleanup behavior. + except RoutingError as exc: + # What: return jsonresponse and status code and exc and code from router_reload; why: router_reload exposes jsonresponse and status code and exc and code so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: router_reload binds this status code and exc value to JSONResponse's status code input. + status_code=exc.status_code, + # What: map the error field as code and str and exc and message and type; why: router_reload carries error into content={"error": {"message": str(exc), "type": exc.code}}. + content={"error": {"message": str(exc), "type": exc.code}}, + # What: complete the JSONResponse call with status code and content; why: router_reload groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + # What: preserve the exact record watch reloaded literal fragment; why: router_reload passes this fragment verbatim through record_watch("reloaded"), because changing it would alter a protocol payload, serialized fixture, or public message. + record_watch("reloaded") + # What: map the reloaded field as true; why: router_reload carries reloaded into return {"reloaded": True, "models": router.catalog.public()}. + return {"reloaded": True, "models": router.catalog.public()} + + # ---- engine lifecycle ---- + + # What: define profile_request around name; why: its direct callers call profile_request for profile request and rely on this exact input and result contract. + def profile_request(name: str) -> tuple[str, int, list[str]]: + # What: compute profile from get and name and catalog and router; why: return profile model resolve port profile port list profile args later reads profile, so profile_request must retain the computed value under that name. + profile = router.catalog.get(name) + # What: return model and profile and resolve port and port from profile_request; why: profile_request exposes model and profile and resolve port and port so its caller can continue with the function\'s computed outcome. + return profile.model, resolve_port(profile.port), list(profile.args) + + # What: define profile_result around name and result and port; why: its direct callers call profile_result for profile result and rely on this exact input and result contract. + def profile_result(name: str, result: dict, port: int): + # What: compute profile from get and name and catalog and router; why: timeout s profile ready timeout s later reads profile, so profile_result must retain the computed value under that name. + profile = router.catalog.get(name) + # What: compute readiness from wait for ready and manager and probe and port; why: content result profile name readiness readiness later reads readiness, so profile_result must retain the computed value under that name. + readiness = wait_for_ready( + # What: apply the manager portion of readiness; why: profile_result uses this clause to evaluate readiness as one grouped value. + manager, + # What: apply the probe portion of readiness; why: profile_result uses this clause to evaluate readiness as one grouped value. + probe, + # What: supply pid to result.get; why: profile_result binds this get and result and pid value to result.get's pid input. + pid=result.get("pid"), + # What: supply port to wait_for_ready; why: profile_result binds this port value to wait_for_ready's port input. + port=port, + # What: supply timeout s to wait_for_ready; why: profile_result binds this ready timeout s and profile value to wait_for_ready's timeout s input. + timeout_s=profile.ready_timeout_s, + # What: supply path to wait_for_ready; why: profile_result binds this check endpoint and profile value to wait_for_ready's path input. + path=profile.check_endpoint, + # What: complete the wait_for_ready call with pid and port and timeout s and path; why: profile_result groups the supplied clauses as one wait_for_ready call before its value is consumed. + ) + + # What: map the profile field as name; why: profile_result carries profile through content into return jsonresponse status code 503 content content. + content = {**result, "profile": name, "readiness": readiness} + # What: gate on readiness before jsonresponse and content; why: profile_result admits jsonresponse and content only for this predicate and excludes the opposite state. + if not readiness["ready"]: + # What: return jsonresponse and content and 503 from profile_result; why: profile_result exposes jsonresponse and content and 503 so its caller can continue with the function\'s computed outcome. + return JSONResponse(status_code=503, content=content) + # What: return content from profile_result; why: profile_result exposes content so its caller can continue with the function\'s computed outcome. + return content + + # What: apply app.exception_handler behavior to switch_launch_error; why: Python attaches this named decorator's registration or descriptor semantics to switch_launch_error. + @app.exception_handler(SwitchLaunchError) + # What: define switch_launch_error around request and exc; why: the registered API client call switch_launch_error for switch launch error and rely on this exact input and result contract. + async def switch_launch_error(request: Request, exc: SwitchLaunchError): + # What: return jsonresponse and rollback and accounting and str from switch_launch_error; why: switch_launch_error exposes jsonresponse and rollback and accounting and str so its caller can continue with the function\'s computed outcome. + return JSONResponse(status_code=503, content={ + # What: map the code field as switch launch failed; why: switch_launch_error carries code into "code": "switch_launch_failed", "error": str(exc). + "code": "switch_launch_failed", "error": str(exc), + # What: map the rollback field as rollback and exc; why: switch_launch_error carries rollback into "rollback": exc.rollback, "accounting": exc.accounting. + "rollback": exc.rollback, "accounting": exc.accounting, + # What: complete the JSONResponse call with status code and content; why: switch_launch_error groups the supplied clauses as one JSONResponse call before its value is consumed. + }) + + # What: register POST /engine/start on the application router; why: clients reach engine_start's handler only through this method-and-path binding. + @app.post("/engine/start", dependencies=auth) + # What: define engine_start around body; why: the registered API client call engine_start for engine start and rely on this exact input and result contract. + async def engine_start(body: StartBody): + # What: define operation around the current object state; why: its direct callers call operation for operation and rely on this exact input and result contract. + async def operation(): + # What: establish the handler boundary for the protected operation; why: operation routes failures to conflict and exception while preserving cleanup and success flow. + try: + # What: compute port from resolve port and port and body; why: return await run lifecycle pool manager start body model later reads port, so operation must retain the computed value under that name. + port = resolve_port(body.port) + # What: return run and lifecycle pool and start and model from operation; why: operation exposes run and lifecycle pool and start and model so its caller can continue with the function\'s computed outcome. + return await run(lifecycle_pool, manager.start, body.model, port, list(body.args)) + # What: handle conflict by st manager status; why: operation converts that failure into this concrete recovery, response, or cleanup behavior. + except Conflict as exc: + # What: compute st from status and manager; why: current model st get model later reads st, so operation must retain the computed value under that name. + st = manager.status() + # What: return jsonresponse and str and exc and get from operation; why: operation exposes jsonresponse and str and exc and get so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: operation binds this 409 value to JSONResponse's status code input. + status_code=409, + # What: supply content to JSONResponse; why: operation binds this str and exc and get and st and error value to JSONResponse's content input. + content={ + # What: map the error field as str and exc; why: operation carries error into "error": str(exc). + "error": str(exc), + # What: map the code field as serve conflict; why: operation carries code into "code": "serve_conflict". + "code": "serve_conflict", + # What: map the current model field as get and st and model; why: operation carries current model into "currentModel": st.get("model"). + "currentModel": st.get("model"), + # What: map the current port field as get and st and port; why: operation carries current port into "currentPort": st.get("port"). + "currentPort": st.get("port"), + # What: execute the grouped source fragment; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + }, + # What: complete the JSONResponse call with status code and content; why: operation groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + # What: handle exception by raise httpexception status code 500 detail f start; why: operation converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: # noqa: BLE001 — never propagate a 500-as-crash + # What: raise HTTPException for the caller; why: operation stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=500, detail=f"start failed: {exc}") + + # What: return run manual transaction and operation from engine_start; why: engine_start exposes run manual transaction and operation so its caller can continue with the function\'s computed outcome. + return await run_manual_transaction(operation) + + # What: register POST /engine/stop on the application router; why: clients reach engine_stop's handler only through this method-and-path binding. + @app.post("/engine/stop", dependencies=auth) + # What: define engine_stop around body; why: the registered API client call engine_stop for engine stop and rely on this exact input and result contract. + async def engine_stop(body: StopBody | None = None): + # What: define operation around the current object state; why: its direct callers call operation for operation and rely on this exact input and result contract. + async def operation(): + # What: establish the handler boundary for the protected operation; why: operation routes failures to accounting prepare error and accounting outbox error while preserving cleanup and success flow. + try: + # What: return run and lifecycle pool and stop and manager from operation; why: operation exposes run and lifecycle pool and stop and manager so its caller can continue with the function\'s computed outcome. + return await run(lifecycle_pool, manager.stop, None, bool(body and body.force)) + # What: execute except AccountingPrepareError AccountingOutboxError as exc; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + except (AccountingPrepareError, AccountingOutboxError) as exc: + # What: return accounting error and exc from operation; why: operation exposes accounting error and exc so its caller can continue with the function\'s computed outcome. + return accounting_error(exc) + + # What: return run manual transaction and operation and true from engine_stop; why: engine_stop exposes run manual transaction and operation and true so its caller can continue with the function\'s computed outcome. + return await run_manual_transaction(operation, preempt_manual=True) @app.post("/shutdown", dependencies=auth) async def shutdown_daemon(request: Request, body: StopBody | None = None): @@ -222,34 +2944,210 @@ async def shutdown_daemon(request: Request, body: StopBody | None = None): # leave the ~18GB serve orphaned, THEN bring the daemon down. We reply before uvicorn # actually stops (it notices should_exit within ~0.1s) so the client still gets a clean 200. try: - stopped = await run(lifecycle_pool, manager.shutdown, None, bool(body and body.force)) - except (AccountingPrepareError, AccountingOutboxError) as exc: - return accounting_error(exc) - req = getattr(request.app.state, "request_shutdown", None) - if req is not None: - req() - return { - "stopping": True, - "already": stopped.get("already", False), - "accounting": stopped.get("accounting"), - } + # What: compute owner from begin shutdown and router; why: owner later reads owner, so shutdown_daemon must retain the computed value under that name. + owner = router.begin_shutdown() + # What: handle routing error by return jsonresponse; why: shutdown_daemon converts that failure into this concrete recovery, response, or cleanup behavior. + except RoutingError as exc: + # What: return jsonresponse and status code and exc and code from shutdown_daemon; why: shutdown_daemon exposes jsonresponse and status code and exc and code so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: shutdown_daemon binds this status code and exc value to JSONResponse's status code input. + status_code=exc.status_code, + # What: map the error field as code and str and exc and message and type; why: shutdown_daemon carries error into content={"error": {"message": str(exc), "type": exc.code}}. + content={"error": {"message": str(exc), "type": exc.code}}, + # What: complete the JSONResponse call with status code and content; why: shutdown_daemon groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + + # What: define operation around the current object state; why: its direct callers call operation for operation and rely on this exact input and result contract. + async def operation(): + # What: establish the handler boundary for the protected operation; why: operation routes failures to accounting prepare error and accounting outbox error while preserving cleanup and success flow. + try: + # What: compute stopped from run and lifecycle pool and finish shutdown and owner; why: already stopped get already later reads stopped, so operation must retain the computed value under that name. + stopped = await run( + # What: apply the lifecycle pool portion of stopped; why: operation uses this clause to evaluate stopped as one grouped value. + lifecycle_pool, + # What: apply the router finish shutdown portion of stopped; why: operation uses this clause to evaluate stopped as one grouped value. + router.finish_shutdown, + # What: apply the owner portion of stopped; why: operation uses this clause to evaluate stopped as one grouped value. + owner, + # What: apply the grouped expression portion of stopped; why: operation uses this clause to evaluate stopped as one grouped value. + None, + # What: call bool with body and force; why: operation consumes the bool return value while evaluating bool(body and body.force). + bool(body and body.force), + # What: complete the run call with lifecycle pool and finish shutdown and owner and bool; why: operation groups the supplied clauses as one run call before its value is consumed. + ) + # What: execute except AccountingPrepareError AccountingOutboxError as exc; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + except (AccountingPrepareError, AccountingOutboxError) as exc: + # What: return accounting error and exc from operation; why: operation exposes accounting error and exc so its caller can continue with the function\'s computed outcome. + return accounting_error(exc) + # What: compute req from getattr and state and app and request and request shutdown; why: if req is not later reads req, so operation must retain the computed value under that name. + req = getattr(request.app.state, "request_shutdown", None) + # What: gate on req before req; why: operation admits req only for this predicate and excludes the opposite state. + if req is not None: + # What: call req with the declared inputs; why: operation invokes req while performing return; the call advances that operation through its result or side effect. + req() + # What: return get and stopped and stopping and already and accounting from operation; why: operation exposes get and stopped and stopping and already and accounting so its caller can continue with the function\'s computed outcome. + return { + # What: map the stopping field as true; why: operation carries stopping into "stopping": True. + "stopping": True, + # What: map the already field as get and stopped and already and false; why: operation carries already into "already": stopped.get("already", False). + "already": stopped.get("already", False), + # What: map the accounting field as get and stopped and accounting; why: operation carries accounting into "accounting": stopped.get("accounting"). + "accounting": stopped.get("accounting"), + # What: complete the enclosing predicate mapping with stopping and already and accounting; why: operation groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + # What: return run to completion and operation from shutdown_daemon; why: shutdown_daemon exposes run to completion and operation so its caller can continue with the function\'s computed outcome. + return await run_to_completion(operation) @app.post("/engine/switch", dependencies=auth) async def engine_switch(body: SwitchBody): - port = resolve_port(body.port) - try: - return await run( - lifecycle_pool, - manager.switch, - body.model, - port, - list(body.args), - body.force, - ) - except (AccountingPrepareError, AccountingOutboxError) as exc: - return accounting_error(exc) - except Exception as exc: # noqa: BLE001 - raise HTTPException(status_code=500, detail=f"switch failed: {exc}") + # What: define operation around the current object state; why: its direct callers call operation for operation and rely on this exact input and result contract. + async def operation(): + # What: establish the handler boundary for the protected operation; why: operation routes failures to accounting prepare error and accounting outbox error and exception while preserving cleanup and success flow. + try: + # What: compute port from resolve port and port and body; why: port later reads port, so operation must retain the computed value under that name. + port = resolve_port(body.port) + # What: return run and lifecycle pool and switch and model from operation; why: operation exposes run and lifecycle pool and switch and model so its caller can continue with the function\'s computed outcome. + return await run( + # What: apply the lifecycle pool portion of the enclosing predicate; why: this clause remains in operation\'s enclosing expression so its grouping and evaluation order stay intact. + lifecycle_pool, + # What: apply the manager switch portion of the enclosing predicate; why: this clause remains in operation\'s enclosing expression so its grouping and evaluation order stay intact. + manager.switch, + # What: apply the body model portion of the enclosing predicate; why: this clause remains in operation\'s enclosing expression so its grouping and evaluation order stay intact. + body.model, + # What: apply the port portion of the enclosing predicate; why: this clause remains in operation\'s enclosing expression so its grouping and evaluation order stay intact. + port, + # What: call list with args and body; why: operation invokes list while performing body force; the call advances that operation through its result or side effect. + list(body.args), + # What: apply the body force portion of the enclosing predicate; why: this clause remains in operation\'s enclosing expression so its grouping and evaluation order stay intact. + body.force, + # What: complete the run call with lifecycle pool and switch and model and port and list; why: operation groups the supplied clauses as one run call before its value is consumed. + ) + # What: execute except AccountingPrepareError AccountingOutboxError as exc; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + except (AccountingPrepareError, AccountingOutboxError) as exc: + # What: return accounting error and exc from operation; why: operation exposes accounting error and exc so its caller can continue with the function\'s computed outcome. + return accounting_error(exc) + # What: handle exception by if isinstance exc switch launch error; why: operation converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: # noqa: BLE001 + # What: gate on isinstance and exc and switch launch error before the computed value; why: operation admits the computed value only for this predicate and excludes the opposite state. + if isinstance(exc, SwitchLaunchError): + # What: re-propagate the active failure to the caller; why: operation stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: raise HTTPException for the caller; why: operation stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=500, detail=f"switch failed: {exc}") + + # What: return run manual transaction and operation from engine_switch; why: engine_switch exposes run manual transaction and operation so its caller can continue with the function\'s computed outcome. + return await run_manual_transaction(operation) + + # What: register POST /engine/start-profile on the application router; why: clients reach engine_start_profile's handler only through this method-and-path binding. + @app.post("/engine/start-profile", dependencies=auth) + # What: define engine_start_profile around body; why: the registered API client call engine_start_profile for engine start profile and rely on this exact input and result contract. + async def engine_start_profile(body: ProfileBody): + # What: define operation around the current object state; why: its direct callers call operation for operation and rely on this exact input and result contract. + async def operation(): + # What: establish the handler boundary for the protected operation; why: operation routes failures to catalog error and conflict and exception while preserving cleanup and success flow. + try: + # What: compute model and port and args from profile request and name and body; why: result await run lifecycle pool manager start model later reads model and port and args, so operation must retain the computed value under that name. + model, port, args = profile_request(body.name) + # What: compute result from run and lifecycle pool and start and model; why: return await run proxy pool profile result body name later reads result, so operation must retain the computed value under that name. + result = await run(lifecycle_pool, manager.start, model, port, args) + # What: return run and proxy pool and profile result and name from operation; why: operation exposes run and proxy pool and profile result and name so its caller can continue with the function\'s computed outcome. + return await run(proxy_pool, profile_result, body.name, result, port) + # What: handle catalog error by raise httpexception status code 404 detail str exc; why: operation converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError as exc: + # What: raise HTTPException for the caller; why: operation stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=404, detail=str(exc)) + # What: handle conflict by st manager status; why: operation converts that failure into this concrete recovery, response, or cleanup behavior. + except Conflict as exc: + # What: compute st from status and manager; why: current model st get model later reads st, so operation must retain the computed value under that name. + st = manager.status() + # What: return jsonresponse and str and exc and get from operation; why: operation exposes jsonresponse and str and exc and get so its caller can continue with the function\'s computed outcome. + return JSONResponse( + # What: supply status code to JSONResponse; why: operation binds this 409 value to JSONResponse's status code input. + status_code=409, + # What: supply content to JSONResponse; why: operation binds this str and exc and get and st and error value to JSONResponse's content input. + content={ + # What: map the error field as str and exc; why: operation carries error into "error": str(exc). + "error": str(exc), + # What: map the code field as serve conflict; why: operation carries code into "code": "serve_conflict". + "code": "serve_conflict", + # What: map the current model field as get and st and model; why: operation carries current model into "currentModel": st.get("model"). + "currentModel": st.get("model"), + # What: map the current port field as get and st and port; why: operation carries current port into "currentPort": st.get("port"). + "currentPort": st.get("port"), + # What: execute the grouped source fragment; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + }, + # What: complete the JSONResponse call with status code and content; why: operation groups the supplied clauses as one JSONResponse call before its value is consumed. + ) + # What: handle exception by raise httpexception status code 500 detail f profile; why: operation converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: # noqa: BLE001 + # What: raise HTTPException for the caller; why: operation stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=500, detail=f"profile start failed: {exc}") + + # What: return run manual transaction and operation from engine_start_profile; why: engine_start_profile exposes run manual transaction and operation so its caller can continue with the function\'s computed outcome. + return await run_manual_transaction(operation) + + # What: register POST /engine/switch-profile on the application router; why: clients reach engine_switch_profile's handler only through this method-and-path binding. + @app.post("/engine/switch-profile", dependencies=auth) + # What: define engine_switch_profile around body; why: the registered API client call engine_switch_profile for engine switch profile and rely on this exact input and result contract. + async def engine_switch_profile(body: ProfileBody): + # What: define operation around the current object state; why: its direct callers call operation for operation and rely on this exact input and result contract. + async def operation(): + # What: establish the handler boundary for the protected operation; why: operation routes failures to catalog error and accounting prepare error and accounting outbox error and exception while preserving cleanup and success flow. + try: + # What: compute model and port and args from profile request and name and body; why: lifecycle pool manager switch for readiness model port args body force later reads model and port and args, so operation must retain the computed value under that name. + model, port, args = profile_request(body.name) + # What: compute result and ticket from run and lifecycle pool and switch for readiness and model; why: response await run proxy pool profile result body name later reads result and ticket, so operation must retain the computed value under that name. + result, ticket = await run( + # What: apply the lifecycle pool manager switch for readiness model port args body force portion of result and ticket; why: operation uses this clause to evaluate result and ticket as one grouped value. + lifecycle_pool, manager.switch_for_readiness, model, port, args, body.force + # What: complete the run call with lifecycle pool and switch for readiness and model and port and args; why: operation groups the supplied clauses as one run call before its value is consumed. + ) + # What: compute response from run and proxy pool and profile result and name; why: if not isinstance response jsonresponse later reads response, so operation must retain the computed value under that name. + response = await run(proxy_pool, profile_result, body.name, result, port) + # What: gate on isinstance and response and jsonresponse before response; why: operation admits response only for this predicate and excludes the opposite state. + if not isinstance(response, JSONResponse): + # What: return response from operation; why: operation exposes response so its caller can continue with the function\'s computed outcome. + return response + # What: compute content from loads and body and json and response; why: content rollback rollback later reads content, so operation must retain the computed value under that name. + content = json.loads(response.body) + # What: compute rollback from run and lifecycle pool and recover switch and ticket; why: if rollback get launched later reads rollback, so operation must retain the computed value under that name. + rollback = await run(lifecycle_pool, manager.recover_switch, ticket, body.force) + # What: gate on get and rollback before profile and get and name and catalog and body; why: operation admits profile and get and name and catalog and body only for this predicate and excludes the opposite state. + if rollback.get("launched"): + # What: compute profile from get and name and catalog and body; why: port rollback port timeout s profile ready timeout s later reads profile, so operation must retain the computed value under that name. + profile = router.catalog.get(body.name) + # What: compute rollback entry from run and proxy pool and partial and wait for ready; why: wait for ready manager probe pid rollback pid later reads rollback entry, so operation must retain the computed value under that name. + rollback["readiness"] = await run(proxy_pool, functools.partial( + # What: supply pid to run; why: operation binds this rollback and pid value to run's pid input. + wait_for_ready, manager, probe, pid=rollback["pid"], + # What: supply port to run; why: operation binds this rollback and port value to run's port input. + port=rollback["port"], timeout_s=profile.ready_timeout_s, + # What: complete the run call with proxy pool and partial; why: operation groups the supplied clauses as one run call before its value is consumed. + )) + # What: compute content entry from rollback; why: return jsonresponse status code content content later reads content entry, so operation must retain the computed value under that name. + content["rollback"] = rollback + # What: return jsonresponse and content and 503 from operation; why: operation exposes jsonresponse and content and 503 so its caller can continue with the function\'s computed outcome. + return JSONResponse(status_code=503, content=content) + # What: handle catalog error by raise httpexception status code 404 detail str exc; why: operation converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError as exc: + # What: raise HTTPException for the caller; why: operation stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=404, detail=str(exc)) + # What: execute except AccountingPrepareError AccountingOutboxError as exc; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + except (AccountingPrepareError, AccountingOutboxError) as exc: + # What: return accounting error and exc from operation; why: operation exposes accounting error and exc so its caller can continue with the function\'s computed outcome. + return accounting_error(exc) + # What: handle exception by if isinstance exc switch launch error; why: operation converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: # noqa: BLE001 + # What: gate on isinstance and exc and switch launch error before the computed value; why: operation admits the computed value only for this predicate and excludes the opposite state. + if isinstance(exc, SwitchLaunchError): + # What: re-propagate the active failure to the caller; why: operation stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: raise HTTPException for the caller; why: operation stops this rejected path before it can mutate state, dispatch work, or report success. + raise HTTPException(status_code=500, detail=f"profile switch failed: {exc}") + + # What: return run manual transaction and operation from engine_switch_profile; why: engine_switch_profile exposes run manual transaction and operation so its caller can continue with the function\'s computed outcome. + return await run_manual_transaction(operation) # ---- durable accounting outbox ---- diff --git a/python/freetoken/daemon/catalog.py b/python/freetoken/daemon/catalog.py new file mode 100644 index 0000000000..4253dbb521 --- /dev/null +++ b/python/freetoken/daemon/catalog.py @@ -0,0 +1,1955 @@ +"""Named, validated FreeToken engine profiles for ``ft daemon``. + +This intentionally borrows the useful *catalog* idea from llama-swap without +accepting its shell-command model. A profile describes only FreeToken's native +``--model``, ``--port`` and argument-vector contract, so loading a catalog never +creates a shell injection path and the daemon remains torch-free. +""" +# What: document named validated free token engine profiles for in the catalog docstring; why: introspection and maintainers read this exact docstring fragment to understand catalog behavior without executing it. +# What: document this intentionally borrows the useful catalog in the catalog docstring; why: introspection and maintainers read this exact docstring fragment to understand catalog behavior without executing it. +# What: document accepting its shell command model a profile in the catalog docstring; why: introspection and maintainers read this exact docstring fragment to understand catalog behavior without executing it. +# What: document model port and argument vector contract so in the catalog docstring; why: introspection and maintainers read this exact docstring fragment to understand catalog behavior without executing it. +# What: document creates a shell injection path and in the catalog docstring; why: introspection and maintainers read this exact docstring fragment to understand catalog behavior without executing it. +# What: preserve the paragraph boundary in the the catalog docstring; why: introspection and maintainers read this paragraph break to understand catalog behavior without executing it. + +# What: enable postponed evaluation of annotations; why: type hints in catalog can reference runtime types without eager imports or forward-reference failures. +from __future__ import annotations + +# What: import dataclass and replace for module initialization and init using dataclasses and dataclass and replace; why: module initialization and __init__ uses dataclass and replace, making that imported dependency available to its named operation. +from dataclasses import dataclass, replace +# What: import json for value using json; why: value uses json loads, making that imported dependency available to its named operation. +import json +# What: import re for module initialization using re; why: module initialization uses re compile, making that imported dependency available to its named operation. +import re +# What: import any for value using typing and any; why: value uses the any annotation in value, making that imported dependency available to its named operation. +from typing import Any + +# What: establish the handler boundary for the protected operation; why: catalog routes failures to module not found error while preserving cleanup and success flow. +try: # tomllib joined the stdlib in Python 3.11; FreeToken supports 3.10 too. + # What: import tomllib for load using tomllib; why: load uses tomllib tomldecode error, making that imported dependency available to its named operation. + import tomllib +# What: handle module not found error by import tomli as tomllib; why: catalog converts that failure into this concrete recovery, response, or cleanup behavior. +except ModuleNotFoundError: # pragma: no cover - exercised in the Python 3.10 package build + # What: import tomli for load using tomli and tomllib; why: load uses tomllib tomldecode error, making that imported dependency available to its named operation. + import tomli as tomllib + + +# What: compute simple name from compile and re and a za z0 9 and a za z0 9 and value; why: if not isinstance name str or later reads simple name, so catalog must retain the computed value under that name. +_SIMPLE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") +# What: compute model segment from compile and re and a za z0 9 and a za z0 9 and value; why: and all model segment fullmatch segment for segment later reads model segment, so catalog must retain the computed value under that name. +_MODEL_SEGMENT = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") +# What: compute safe http path from compile and re and a za z0 9 and value and a za z0 9; why: or not safe http path fullmatch value later reads safe http path, so catalog must retain the computed value under that name. +_SAFE_HTTP_PATH = re.compile(r"^/(?:[A-Za-z0-9._~-]+(?:/[A-Za-z0-9._~-]+)*)?$") +# What: compute upstream suffix from compile and re and a za z0 9 and a za z0 9 and value; why: isinstance suffix str and upstream suffix fullmatch suffix later reads upstream suffix, so catalog must retain the computed value under that name. +_UPSTREAM_SUFFIX = re.compile(r"^\.[A-Za-z0-9][A-Za-z0-9._-]{0,31}$") +# What: compute http header name from compile and re and value and a za z; why: isinstance header str and http header name fullmatch header later reads http header name, so catalog must retain the computed value under that name. +_HTTP_HEADER_NAME = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]{1,64}$") +# What: compute proxy template from compile and re and http and port and p; why: match proxy template fullmatch value later reads proxy template, so catalog must retain the computed value under that name. +_PROXY_TEMPLATE = re.compile( + # What: apply the r http port p prefix a za z0 9 portion of proxy template; why: catalog uses this clause to evaluate proxy template as one grouped value. + r"^http://127\.0\.0\.1:\$\{PORT\}(?P/(?:[A-Za-z0-9._~-]+(?:/[A-Za-z0-9._~-]+)*)?)?$" +# What: complete the re.compile call with ordered positional inputs; why: catalog groups the supplied clauses as one re.compile call before its value is consumed. +) + +# What: compute default check endpoint from health; why: check endpoint str default check endpoint later reads default check endpoint, so catalog must retain the computed value under that name. +DEFAULT_CHECK_ENDPOINT = "/health" +# What: compute default proxy from http and port; why: proxy str default proxy later reads default proxy, so catalog must retain the computed value under that name. +DEFAULT_PROXY = "http://127.0.0.1:${PORT}" +# What: compute default upstream no activation suffixes from js and json and css and png and gif; why: catalog consumes default upstream no activation suffixes during upstream no activation suffixes tuple str default upstream no activation suffixes, so default upstream no activation suffixes value receives the compute. +DEFAULT_UPSTREAM_NO_ACTIVATION_SUFFIXES = ( + # What: apply the js json css png gif jpg portion of default upstream no activation suffixes; why: catalog uses this clause to evaluate default upstream no activation suffixes as one grouped value. + ".js", ".json", ".css", ".png", ".gif", ".jpg", ".jpeg", ".ico", ".txt", +# What: complete the DEFAULT_UPSTREAM_NO_ACTIVATION_SUFFIXES collection with js and json and css and png; why: catalog groups the supplied clauses as one DEFAULT_UPSTREAM_NO_ACTIVATION_SUFFIXES collection before its value is consumed. +) +# What: compute default activity session headers from x session id and x litellm session id; why: activity session headers tuple str default activity session headers later reads default activity session headers, so catalog must retain the computed value under that name. +DEFAULT_ACTIVITY_SESSION_HEADERS = ("x-session-id", "x-litellm-session-id") + + +# What: define CatalogError as the owner of its declared state; why: daemon callers use this class boundary so those methods share one catalog error state invariant. +class CatalogError(ValueError): + """A catalog is malformed or requests an unsafe/ambiguous profile.""" +# What: document a catalog is malformed or requests in the CatalogError docstring; why: introspection and maintainers read this exact docstring fragment to understand catalog error behavior without executing it. + + +# What: generate dataclass initialization and value semantics for RoutingGroup; why: RoutingGroup acts as a typed state record with consistent construction, comparison, and representation. +@dataclass(frozen=True) +# What: define RoutingGroup as the owner of its declared state; why: daemon callers use this class boundary so those methods share one routing group state invariant. +class RoutingGroup: + """An atomically validated native equivalent of a llama-swap group.""" +# What: document an atomically validated native equivalent of in the RoutingGroup docstring; why: introspection and maintainers read this exact docstring fragment to understand routing group behavior without executing it. + + # What: compute name from the named fixture input; why: name str later reads name, so catalog must retain the computed value under that name. + name: str + # What: compute members from the named fixture input; why: if name in group members later reads members, so catalog must retain the computed value under that name. + members: tuple[str, ...] + # What: compute swap from true; why: render the applicable pinned llama swap model list later reads swap, so catalog must retain the computed value under that name. + swap: bool = True + # What: compute exclusive from true; why: unknown sorted set raw group members swap later reads exclusive, so catalog must retain the computed value under that name. + exclusive: bool = True + # What: compute persistent from false; why: unknown sorted set raw group members swap later reads persistent, so catalog must retain the computed value under that name. + persistent: bool = False + + +# What: generate dataclass initialization and value semantics for RouterSettings; why: RouterSettings acts as a typed state record with consistent construction, comparison, and representation. +@dataclass(frozen=True) +# What: define RouterSettings as the owner of its declared state; why: daemon callers use this class boundary so those methods share one router settings state invariant. +class RouterSettings: + """Global router policy, deliberately free of command execution fields.""" +# What: document global router policy deliberately free of in the RouterSettings docstring; why: introspection and maintainers read this exact docstring fragment to understand router settings behavior without executing it. + + # What: compute api keys from the named fixture input; why: api keys default ttl s unload timeout s upstream timeout s later reads api keys, so catalog must retain the computed value under that name. + api_keys: tuple[str, ...] = () + # What: compute default ttl s from 0 0; why: api keys default ttl s unload timeout s upstream timeout s later reads default ttl s, so catalog must retain the computed value under that name. + default_ttl_s: float = 0.0 + # What: compute unload timeout s from 30 0; why: unload timeout s float later reads unload timeout s, so catalog must retain the computed value under that name. + unload_timeout_s: float = 30.0 + # What: compute upstream timeout s from 900 0; why: upstream timeout s float later reads upstream timeout s, so catalog must retain the computed value under that name. + upstream_timeout_s: float = 900.0 + # What: compute scheduler from fifo; why: scheduler groups include aliases in list global concurrency limit later reads scheduler, so catalog must retain the computed value under that name. + scheduler: str = "fifo" + # What: compute groups from the named fixture input; why: for group in self settings groups later reads groups, so catalog must retain the computed value under that name. + groups: tuple[RoutingGroup, ...] = () + # What: compute include aliases in list from false; why: if self settings include aliases in list later reads include aliases in list, so catalog must retain the computed value under that name. + include_aliases_in_list: bool = False + # What: compute global concurrency limit from 0; why: scheduler groups include aliases in list global concurrency limit later reads global concurrency limit, so catalog must retain the computed value under that name. + global_concurrency_limit: int = 0 + # What: compute send loading state from false; why: send loading state bool later reads send loading state, so catalog must retain the computed value under that name. + send_loading_state: bool = False + # What: compute preload model from the named fixture input; why: if settings preload model is not later reads preload model, so catalog must retain the computed value under that name. + preload_model: str | None = None + # What: compute startup routing profile from the named fixture input; why: settings startup routing profile is not later reads startup routing profile, so catalog must retain the computed value under that name. + startup_routing_profile: str | None = None + # What: compute upstream no activation suffixes from default upstream no activation suffixes; why: upstream no activation suffixes later reads upstream no activation suffixes, so catalog must retain the computed value under that name. + upstream_no_activation_suffixes: tuple[str, ...] = DEFAULT_UPSTREAM_NO_ACTIVATION_SUFFIXES + # What: compute activity max entries from 1000; why: activity max entries capture buffer mb later reads activity max entries, so catalog must retain the computed value under that name. + activity_max_entries: int = 1000 + # What: compute capture buffer mb from 0; why: activity max entries capture buffer mb later reads capture buffer mb, so catalog must retain the computed value under that name. + capture_buffer_mb: int = 0 + # What: compute activity session headers from default activity session headers; why: activity session headers later reads activity session headers, so catalog must retain the computed value under that name. + activity_session_headers: tuple[str, ...] = DEFAULT_ACTIVITY_SESSION_HEADERS + # What: compute performance disabled from false; why: performance disabled performance every s later reads performance disabled, so catalog must retain the computed value under that name. + performance_disabled: bool = False + # What: compute performance every s from 5 0; why: performance disabled performance every s later reads performance every s, so catalog must retain the computed value under that name. + performance_every_s: float = 5.0 + + +# What: generate dataclass initialization and value semantics for ModelCapabilities; why: ModelCapabilities acts as a typed state record with consistent construction, comparison, and representation. +@dataclass(frozen=True) +# What: define ModelCapabilities as the owner of empty and public and model_listing_fields; why: daemon callers use this class boundary so those methods share one model capabilities state invariant. +class ModelCapabilities: + """Validated model-list metadata; it never enables inference behavior.""" +# What: document validated model list metadata it never enables in the ModelCapabilities docstring; why: introspection and maintainers read this exact docstring fragment to understand model capabilities behavior without executing it. + + # What: compute input modalities from the named fixture input; why: self input modalities or self output modalities or self tools or later reads input modalities, so catalog must retain the computed value under that name. + input_modalities: tuple[str, ...] = () + # What: compute output modalities from the named fixture input; why: self input modalities or self output modalities or self tools or later reads output modalities, so catalog must retain the computed value under that name. + output_modalities: tuple[str, ...] = () + # What: compute tools from false; why: self input modalities or self output modalities or self tools or later reads tools, so catalog must retain the computed value under that name. + tools: bool = False + # What: compute context from 0; why: self input modalities or self output modalities or self tools or later reads context, so catalog must retain the computed value under that name. + context: int = 0 + + # What: define empty around the current object state; why: its direct callers call empty for empty and rely on this exact input and result contract. + def empty(self) -> bool: + # What: return input modalities and output modalities and tools and context from empty; why: empty exposes input modalities and output modalities and tools and context so its caller can continue with the function\'s computed outcome. + return not ( + # What: apply the self input modalities or self output modalities or self tools or portion of the enclosing predicate; why: this clause remains in empty\'s enclosing expression so its grouping and evaluation order stay intact. + self.input_modalities or self.output_modalities or self.tools or self.context + # What: complete the empty signature with self; why: ModelCapabilities.empty groups the supplied clauses as one empty signature before its value is consumed. + ) + + # What: define public around the current object state; why: its direct callers call public for public and rely on this exact input and result contract. + def public(self) -> dict[str, Any]: + # What: initialize doc as an empty runtime accumulator; why: ModelCapabilities.public appends or maps entries into it during doc in list self input modalities before consuming the aggregate. + doc: dict[str, Any] = {} + # What: gate on input modalities before doc and list and input modalities; why: public admits doc and list and input modalities only for this predicate and excludes the opposite state. + if self.input_modalities: + # What: compute doc entry from list and input modalities; why: doc out list self output modalities later reads doc entry, so public must retain the computed value under that name. + doc["in"] = list(self.input_modalities) + # What: gate on output modalities before doc and list and output modalities; why: public admits doc and list and output modalities only for this predicate and excludes the opposite state. + if self.output_modalities: + # What: compute doc entry from list and output modalities; why: doc tools later reads doc entry, so public must retain the computed value under that name. + doc["out"] = list(self.output_modalities) + # What: gate on tools before doc; why: public admits doc only for this predicate and excludes the opposite state. + if self.tools: + # What: compute doc entry from true; why: doc context self context later reads doc entry, so public must retain the computed value under that name. + doc["tools"] = True + # What: gate on context before context and doc; why: public admits context and doc only for this predicate and excludes the opposite state. + if self.context: + # What: compute doc entry from context; why: return doc later reads doc entry, so public must retain the computed value under that name. + doc["context"] = self.context + # What: return doc from public; why: public exposes doc so its caller can continue with the function\'s computed outcome. + return doc + + # What: define model_listing_fields around the current object state; why: its direct callers call model_listing_fields for model listing fields and rely on this exact input and result contract. + def model_listing_fields(self) -> dict[str, Any]: + """Render the applicable pinned llama-swap model-list contract.""" + # What: document render the applicable pinned llama swap model list in the model_listing_fields docstring; why: introspection and maintainers read this exact docstring fragment to understand model listing fields behavior without executing it. + # What: initialize doc as an empty runtime accumulator; why: ModelCapabilities.model_listing_fields appends or maps entries into it during doc architecture architecture before consuming the aggregate. + doc: dict[str, Any] = {} + # What: gate on input modalities and output modalities before architecture and dict and str and any; why: model_listing_fields admits architecture and dict and str and any only for this predicate and excludes the opposite state. + if self.input_modalities or self.output_modalities: + # What: initialize architecture as an empty runtime accumulator; why: ModelCapabilities.model_listing_fields appends or maps entries into it during architecture input modalities list self input modalities before consuming the aggregate. + architecture: dict[str, Any] = {} + # What: gate on input modalities before architecture and list and input modalities; why: model_listing_fields admits architecture and list and input modalities only for this predicate and excludes the opposite state. + if self.input_modalities: + # What: compute architecture entry from list and input modalities; why: architecture output modalities list self output modalities later reads architecture entry, so model_listing_fields must retain the computed value under that name. + architecture["input_modalities"] = list(self.input_modalities) + # What: gate on output modalities before architecture and list and output modalities; why: model_listing_fields admits architecture and list and output modalities only for this predicate and excludes the opposite state. + if self.output_modalities: + # What: compute architecture entry from list and output modalities; why: architecture modality later reads architecture entry, so model_listing_fields must retain the computed value under that name. + architecture["output_modalities"] = list(self.output_modalities) + # What: gate on input modalities and output modalities before architecture and join and input modalities and output modalities; why: model_listing_fields admits architecture and join and input modalities and output modalities only for this predicate and excludes the opposite state. + if self.input_modalities and self.output_modalities: + # What: compute architecture entry from join and input modalities and output modalities and value and value; why: doc architecture architecture later reads architecture entry, so model_listing_fields must retain the computed value under that name. + architecture["modality"] = ( + # What: call operation.join with input modalities; why: model_listing_fields consumes the operation.join return value while evaluating f"{'+'.join(self.input_modalities)}->{'+'.join(self.output_modalities)}". + f"{'+'.join(self.input_modalities)}->{'+'.join(self.output_modalities)}" + # What: complete the architecture entry expression with architecture modality f join self input modalities join self output modalities; why: ModelCapabilities.model_listing_fields groups the supplied clauses as one architecture entry expression before its value is consumed. + ) + # What: compute doc entry from architecture; why: doc capabilities function calling later reads doc entry, so model_listing_fields must retain the computed value under that name. + doc["architecture"] = architecture + # What: gate on tools before doc; why: model_listing_fields admits doc only for this predicate and excludes the opposite state. + if self.tools: + # What: map the function calling field as true; why: ModelCapabilities.model_listing_fields carries function calling through doc entry into doc supported parameters tools tool choice. + doc["capabilities"] = {"function_calling": True} + # What: compute doc entry from tools and tool choice; why: doc context length self context later reads doc entry, so model_listing_fields must retain the computed value under that name. + doc["supported_parameters"] = ["tools", "tool_choice"] + # What: gate on context before context and doc; why: model_listing_fields admits context and doc only for this predicate and excludes the opposite state. + if self.context: + # What: compute doc entry from context; why: doc context window self context later reads doc entry, so model_listing_fields must retain the computed value under that name. + doc["context_length"] = self.context + # What: compute doc entry from context; why: doc meta n ctx self context later reads doc entry, so model_listing_fields must retain the computed value under that name. + doc["context_window"] = self.context + # What: map the n ctx field as context; why: ModelCapabilities.model_listing_fields carries n ctx through doc entry into return doc. + doc["meta"] = {"n_ctx": self.context} + # What: return doc from model_listing_fields; why: model_listing_fields exposes doc so its caller can continue with the function\'s computed outcome. + return doc + + +# What: generate dataclass initialization and value semantics for RequestField; why: RequestField acts as a typed state record with consistent construction, comparison, and representation. +@dataclass(frozen=True) +# What: define RequestField as the owner of key and value; why: daemon callers use this class boundary so those methods share one request field state invariant. +class RequestField: + """One immutable, validated JSON field assignment.""" +# What: document one immutable validated json field assignment in the RequestField docstring; why: introspection and maintainers read this exact docstring fragment to understand request field behavior without executing it. + + # What: compute path from the named fixture input; why: return join self path later reads path, so catalog must retain the computed value under that name. + path: tuple[str, ...] + # What: compute value json from the named fixture input; why: return json loads self value json later reads value json, so catalog must retain the computed value under that name. + value_json: str + # What: compute soft from false; why: field key if field soft else field value later reads soft, so catalog must retain the computed value under that name. + soft: bool = False + + # What: expose key as a read-only computed property; why: callers read key through attribute access while its getter retains control of the derived value. + @property + # What: define key around the current object state; why: the registered API client call key for key and rely on this exact input and result contract. + def key(self) -> str: + # What: return join and path and value from key; why: key exposes join and path and value so its caller can continue with the function\'s computed outcome. + return ".".join(self.path) + + # What: define value around the current object state; why: its direct callers call value for value and rely on this exact input and result contract. + def value(self) -> Any: + # What: return loads and value json and json from value; why: value exposes loads and value json and json so its caller can continue with the function\'s computed outcome. + return json.loads(self.value_json) + + +# What: define _request_fields_public around fields; why: its direct callers call _request_fields_public for request fields public and rely on this exact input and result contract. +def _request_fields_public(fields: tuple[RequestField, ...]) -> dict[str, Any]: + # What: return key and value and field and fields from _request_fields_public; why: _request_fields_public exposes key and value and field and fields so its caller can continue with the function\'s computed outcome. + return { + # What: call field.value with the declared inputs; why: _request_fields_public invokes field.value while performing for field in fields; the call advances that operation through its result or side effect. + field.key + ("?" if field.soft else ""): field.value() + # What: apply the for field in fields portion of the enclosing predicate; why: this clause remains in _request_fields_public\'s enclosing expression so its grouping and evaluation order stay intact. + for field in fields + # What: complete the _request_fields_public signature with fields; why: _request_fields_public groups the supplied clauses as one _request_fields_public signature before its value is consumed. + } + + +# What: generate dataclass initialization and value semantics for ModelSelector; why: ModelSelector acts as a typed state record with consistent construction, comparison, and representation. +@dataclass(frozen=True) +# What: define ModelSelector as the owner of metadata and public; why: daemon callers use this class boundary so those methods share one model selector state invariant. +class ModelSelector: + """A per-request virtual model resolved to one concrete local profile.""" +# What: document a per request virtual model resolved to in the ModelSelector docstring; why: introspection and maintainers read this exact docstring fragment to understand model selector behavior without executing it. + + # What: compute name from the named fixture input; why: name self name later reads name, so catalog must retain the computed value under that name. + name: str + # What: compute strategy from the named fixture input; why: strategy self strategy later reads strategy, so catalog must retain the computed value under that name. + strategy: str + # What: compute targets from the named fixture input; why: targets list self targets later reads targets, so catalog must retain the computed value under that name. + targets: tuple[str, ...] + # What: compute display name from the named fixture input; why: if self display name later reads display name, so catalog must retain the computed value under that name. + display_name: str | None = None + # What: compute description from the named fixture input; why: if self description later reads description, so catalog must retain the computed value under that name. + description: str | None = None + # What: compute unlisted from false; why: if self unlisted later reads unlisted, so catalog must retain the computed value under that name. + unlisted: bool = False + # What: compute metadata json from value; why: return json loads self metadata json later reads metadata json, so catalog must retain the computed value under that name. + metadata_json: str = "{}" + + # What: define metadata around the current object state; why: its direct callers call metadata for metadata and rely on this exact input and result contract. + def metadata(self) -> dict[str, Any]: + # What: return loads and metadata json and json from metadata; why: metadata exposes loads and metadata json and json so its caller can continue with the function\'s computed outcome. + return json.loads(self.metadata_json) + + # What: define public around the current object state; why: its direct callers call public for public and rely on this exact input and result contract. + def public(self) -> dict[str, Any]: + # What: compute doc from name and strategy and list and targets and name; why: doc display name self display name later reads doc, so public must retain the computed value under that name. + doc: dict[str, Any] = { + # What: map the name field as name; why: ModelSelector.public carries name through doc into doc display name self display name. + "name": self.name, + # What: map the strategy field as strategy; why: ModelSelector.public carries strategy through doc into doc display name self display name. + "strategy": self.strategy, + # What: map the targets field as list and targets; why: ModelSelector.public carries targets through doc into doc display name self display name. + "targets": list(self.targets), + # What: complete the doc mapping with name and strategy and targets; why: ModelSelector.public groups the supplied clauses as one doc mapping before its value is consumed. + } + # What: gate on display name before display name and doc; why: public admits display name and doc only for this predicate and excludes the opposite state. + if self.display_name: + # What: compute doc entry from display name; why: doc description self description later reads doc entry, so public must retain the computed value under that name. + doc["displayName"] = self.display_name + # What: gate on description before description and doc; why: public admits description and doc only for this predicate and excludes the opposite state. + if self.description: + # What: compute doc entry from description; why: doc unlisted later reads doc entry, so public must retain the computed value under that name. + doc["description"] = self.description + # What: gate on unlisted before doc; why: public admits doc only for this predicate and excludes the opposite state. + if self.unlisted: + # What: compute doc entry from true; why: doc metadata metadata later reads doc entry, so public must retain the computed value under that name. + doc["unlisted"] = True + # What: compute metadata from metadata; why: if metadata later reads metadata, so public must retain the computed value under that name. + metadata = self.metadata() + # What: gate on metadata before metadata and doc; why: public admits metadata and doc only for this predicate and excludes the opposite state. + if metadata: + # What: compute doc entry from metadata; why: return doc later reads doc entry, so public must retain the computed value under that name. + doc["metadata"] = metadata + # What: return doc from public; why: public exposes doc so its caller can continue with the function\'s computed outcome. + return doc + + +# What: generate dataclass initialization and value semantics for RoutingProfile; why: RoutingProfile acts as a typed state record with consistent construction, comparison, and representation. +@dataclass(frozen=True) +# What: define RoutingProfile as the owner of replacement and public; why: daemon callers use this class boundary so those methods share one routing profile state invariant. +class RoutingProfile: + """A runtime-selectable set of client model-ID replacements.""" +# What: document a runtime selectable set of client model id in the RoutingProfile docstring; why: introspection and maintainers read this exact docstring fragment to understand routing profile behavior without executing it. + + # What: compute name from the named fixture input; why: name self name later reads name, so catalog must retain the computed value under that name. + name: str + # What: compute pins from the named fixture input; why: for pin target in self pins later reads pins, so catalog must retain the computed value under that name. + pins: tuple[tuple[str, str | None], ...] + # What: compute description from the named fixture input; why: if self description later reads description, so catalog must retain the computed value under that name. + description: str | None = None + + # What: define replacement around model id; why: its direct callers call replacement for replacement and rely on this exact input and result contract. + def replacement(self, model_id: str) -> tuple[bool, str | None]: + # What: iterate across pins to perform pin and model id and target; why: replacement repeats the body only while or for the loop header admits an iteration. + for pin, target in self.pins: + # What: gate on pin and model id before target; why: replacement admits target only for this predicate and excludes the opposite state. + if pin == model_id: + # What: return target and true from replacement; why: replacement exposes target and true so its caller can continue with the function\'s computed outcome. + return True, target + # What: return false from replacement; why: replacement exposes false so its caller can continue with the function\'s computed outcome. + return False, None + + # What: define public around the current object state; why: its direct callers call public for public and rely on this exact input and result contract. + def public(self) -> dict[str, Any]: + # What: compute doc from name and pin and target and pins and name; why: doc description self description later reads doc, so public must retain the computed value under that name. + doc: dict[str, Any] = { + # What: map the name field as name; why: RoutingProfile.public carries name through doc into doc description self description. + "name": self.name, + # What: map the pins field as pin and target and pins; why: RoutingProfile.public carries pins through doc into doc description self description. + "pins": {pin: target for pin, target in self.pins}, + # What: complete the doc mapping with name and pins; why: RoutingProfile.public groups the supplied clauses as one doc mapping before its value is consumed. + } + # What: gate on description before description and doc; why: public admits description and doc only for this predicate and excludes the opposite state. + if self.description: + # What: compute doc entry from description; why: return doc later reads doc entry, so public must retain the computed value under that name. + doc["description"] = self.description + # What: return doc from public; why: public exposes doc so its caller can continue with the function\'s computed outcome. + return doc + + +# What: generate dataclass initialization and value semantics for ModelProfile; why: ModelProfile acts as a typed state record with consistent construction, comparison, and representation. +@dataclass(frozen=True) +# What: define ModelProfile as the owner of metadata and proxy_base_url and request and public; why: daemon callers use this class boundary so those methods share one model profile state invariant. +class ModelProfile: + # What: compute name from the named fixture input; why: doc name self name later reads name, so catalog must retain the computed value under that name. + name: str + # What: compute model from the named fixture input; why: body dict str any model self model later reads model, so catalog must retain the computed value under that name. + model: str + # What: compute args from the named fixture input; why: body dict str any model self model later reads args, so catalog must retain the computed value under that name. + args: tuple[str, ...] + # What: compute port from the named fixture input; why: def proxy base url port int str later reads port, so catalog must retain the computed value under that name. + port: int | None = None + # What: compute description from the named fixture input; why: if self description later reads description, so catalog must retain the computed value under that name. + description: str | None = None + # What: compute ready timeout s from 120 0; why: doc ready timeout s self ready timeout s later reads ready timeout s, so catalog must retain the computed value under that name. + ready_timeout_s: float = 120.0 + # What: compute ttl s from the named fixture input; why: if self ttl s is not later reads ttl s, so catalog must retain the computed value under that name. + ttl_s: float | None = None + # What: compute unload timeout s from the named fixture input; why: if self unload timeout s is not later reads unload timeout s, so catalog must retain the computed value under that name. + unload_timeout_s: float | None = None + # What: compute priority from 0; why: if self priority later reads priority, so catalog must retain the computed value under that name. + priority: int = 0 + # What: compute group from the named fixture input; why: if self group is not later reads group, so catalog must retain the computed value under that name. + group: str | None = None + # What: compute drop fields from the named fixture input; why: if self drop fields later reads drop fields, so catalog must retain the computed value under that name. + drop_fields: tuple[str, ...] = () + # What: compute aliases from the named fixture input; why: if self aliases later reads aliases, so catalog must retain the computed value under that name. + aliases: tuple[str, ...] = () + # What: compute unlisted from false; why: if self unlisted later reads unlisted, so catalog must retain the computed value under that name. + unlisted: bool = False + # What: compute concurrency limit from 0; why: if self concurrency limit later reads concurrency limit, so catalog must retain the computed value under that name. + concurrency_limit: int = 0 + # What: compute send loading state from the named fixture input; why: if self send loading state is not later reads send loading state, so catalog must retain the computed value under that name. + send_loading_state: bool | None = None + # What: compute capabilities from model capabilities; why: if not self capabilities empty later reads capabilities, so catalog must retain the computed value under that name. + capabilities: ModelCapabilities = ModelCapabilities() + # What: compute set fields from the named fixture input; why: if self set fields later reads set fields, so catalog must retain the computed value under that name. + set_fields: tuple[RequestField, ...] = () + # What: compute set fields by id from the named fixture input; why: if self set fields by id later reads set fields by id, so catalog must retain the computed value under that name. + set_fields_by_id: tuple[tuple[str, tuple[RequestField, ...]], ...] = () + # What: compute check endpoint from default check endpoint; why: if self check endpoint default check endpoint later reads check endpoint, so catalog must retain the computed value under that name. + check_endpoint: str = DEFAULT_CHECK_ENDPOINT + # What: compute proxy from default proxy; why: return self proxy replace port str port later reads proxy, so catalog must retain the computed value under that name. + proxy: str = DEFAULT_PROXY + # What: compute use model name from the named fixture input; why: if self use model name is not later reads use model name, so catalog must retain the computed value under that name. + use_model_name: str | None = None + # What: compute display name from the named fixture input; why: if self display name later reads display name, so catalog must retain the computed value under that name. + display_name: str | None = None + # What: compute metadata json from value; why: return json loads self metadata json later reads metadata json, so catalog must retain the computed value under that name. + metadata_json: str = "{}" + # What: compute upstream timeout s from the named fixture input; why: if self upstream timeout s is not later reads upstream timeout s, so catalog must retain the computed value under that name. + upstream_timeout_s: float | None = None + + # What: define metadata around the current object state; why: its direct callers call metadata for metadata and rely on this exact input and result contract. + def metadata(self) -> dict[str, Any]: + # What: return loads and metadata json and json from metadata; why: metadata exposes loads and metadata json and json so its caller can continue with the function\'s computed outcome. + return json.loads(self.metadata_json) + + # What: define proxy_base_url around port; why: its direct callers call proxy_base_url for proxy base url and rely on this exact input and result contract. + def proxy_base_url(self, port: int) -> str: + """Resolve the validated loopback template to this owned child port.""" + # What: document resolve the validated loopback template to in the proxy_base_url docstring; why: introspection and maintainers read this exact docstring fragment to understand proxy base url behavior without executing it. + # What: return replace and proxy and str and port and port from proxy_base_url; why: proxy_base_url exposes replace and proxy and str and port and port so its caller can continue with the function\'s computed outcome. + return self.proxy.replace("${PORT}", str(port)) + + # What: define request around the current object state; why: its direct callers call request for request and rely on this exact input and result contract. + def request(self) -> dict[str, Any]: + # What: map the model field as model; why: ModelProfile.request sends this field through body so the router selects the canonical model or alias for upstream dispatch. + body: dict[str, Any] = {"model": self.model, "args": list(self.args)} + # What: gate on port before port and body; why: request admits port and body only for this predicate and excludes the opposite state. + if self.port is not None: + # What: compute body entry from port; why: body dynamic port later reads body entry, so request must retain the computed value under that name. + body["port"] = self.port + # What: gate on port before body; why: request admits body only for this predicate and excludes the opposite state. + if self.port == 0: + # What: compute body entry from true; why: return body later reads body entry, so request must retain the computed value under that name. + body["dynamicPort"] = True + # What: return body from request; why: request exposes body so its caller can continue with the function\'s computed outcome. + return body + + # What: define public around the current object state; why: its direct callers call public for public and rely on this exact input and result contract. + def public(self) -> dict[str, Any]: + # What: compute doc from request; why: doc name self name later reads doc, so public must retain the computed value under that name. + doc = self.request() + # What: compute doc entry from name; why: doc display name self display name later reads doc entry, so public must retain the computed value under that name. + doc["name"] = self.name + # What: gate on display name before display name and doc; why: public admits display name and doc only for this predicate and excludes the opposite state. + if self.display_name: + # What: compute doc entry from display name; why: doc description self description later reads doc entry, so public must retain the computed value under that name. + doc["displayName"] = self.display_name + # What: gate on description before description and doc; why: public admits description and doc only for this predicate and excludes the opposite state. + if self.description: + # What: compute doc entry from description; why: doc ready timeout s self ready timeout s later reads doc entry, so public must retain the computed value under that name. + doc["description"] = self.description + # What: compute doc entry from ready timeout s; why: doc ttl s self ttl s later reads doc entry, so public must retain the computed value under that name. + doc["readyTimeoutS"] = self.ready_timeout_s + # What: gate on ttl s before ttl s and doc; why: public admits ttl s and doc only for this predicate and excludes the opposite state. + if self.ttl_s is not None: + # What: compute doc entry from ttl s; why: doc unload timeout s self unload timeout s later reads doc entry, so public must retain the computed value under that name. + doc["ttlS"] = self.ttl_s + # What: gate on unload timeout s before unload timeout s and doc; why: public admits unload timeout s and doc only for this predicate and excludes the opposite state. + if self.unload_timeout_s is not None: + # What: compute doc entry from unload timeout s; why: doc priority self priority later reads doc entry, so public must retain the computed value under that name. + doc["unloadTimeoutS"] = self.unload_timeout_s + # What: gate on priority before priority and doc; why: public admits priority and doc only for this predicate and excludes the opposite state. + if self.priority: + # What: compute doc entry from priority; why: doc group self group later reads doc entry, so public must retain the computed value under that name. + doc["priority"] = self.priority + # What: gate on group before group and doc; why: public admits group and doc only for this predicate and excludes the opposite state. + if self.group is not None: + # What: compute doc entry from group; why: doc drop fields list self drop fields later reads doc entry, so public must retain the computed value under that name. + doc["group"] = self.group + # What: gate on drop fields before doc and list and drop fields; why: public admits doc and list and drop fields only for this predicate and excludes the opposite state. + if self.drop_fields: + # What: compute doc entry from list and drop fields; why: doc aliases list self aliases later reads doc entry, so public must retain the computed value under that name. + doc["dropFields"] = list(self.drop_fields) + # What: gate on aliases before doc and list and aliases; why: public admits doc and list and aliases only for this predicate and excludes the opposite state. + if self.aliases: + # What: compute doc entry from list and aliases; why: doc unlisted later reads doc entry, so public must retain the computed value under that name. + doc["aliases"] = list(self.aliases) + # What: gate on unlisted before doc; why: public admits doc only for this predicate and excludes the opposite state. + if self.unlisted: + # What: compute doc entry from true; why: doc concurrency limit self concurrency limit later reads doc entry, so public must retain the computed value under that name. + doc["unlisted"] = True + # What: gate on concurrency limit before concurrency limit and doc; why: public admits concurrency limit and doc only for this predicate and excludes the opposite state. + if self.concurrency_limit: + # What: compute doc entry from concurrency limit; why: doc send loading state self send loading state later reads doc entry, so public must retain the computed value under that name. + doc["concurrencyLimit"] = self.concurrency_limit + # What: gate on send loading state before send loading state and doc; why: public admits send loading state and doc only for this predicate and excludes the opposite state. + if self.send_loading_state is not None: + # What: compute doc entry from send loading state; why: doc capabilities self capabilities public later reads doc entry, so public must retain the computed value under that name. + doc["sendLoadingState"] = self.send_loading_state + # What: gate on empty and capabilities before doc and public and capabilities; why: public admits doc and public and capabilities only for this predicate and excludes the opposite state. + if not self.capabilities.empty(): + # What: compute doc entry from public and capabilities; why: doc set fields request fields public self set fields later reads doc entry, so public must retain the computed value under that name. + doc["capabilities"] = self.capabilities.public() + # What: gate on set fields before doc and request fields public and set fields; why: public admits doc and request fields public and set fields only for this predicate and excludes the opposite state. + if self.set_fields: + # What: compute doc entry from request fields public and set fields; why: doc set fields by id later reads doc entry, so public must retain the computed value under that name. + doc["setFields"] = _request_fields_public(self.set_fields) + # What: gate on set fields by id before doc and model id and request fields public and fields and set fields by id; why: public admits doc and model id and request fields public and fields and set fields by id only for this predicate and excludes the opposite state. + if self.set_fields_by_id: + # What: compute doc entry from model id and request fields public and fields and set fields by id; why: doc check endpoint self check endpoint later reads doc entry, so public must retain the computed value under that name. + doc["setFieldsById"] = { + # What: call _request_fields_public with fields; why: public invokes _request_fields_public while performing for model id fields in self set fields by id; the call advances that operation through its result or side effect. + model_id: _request_fields_public(fields) + # What: apply the for model id fields in self set fields by id portion of doc entry; why: public uses this clause to evaluate doc entry as one grouped value. + for model_id, fields in self.set_fields_by_id + # What: complete the doc entry expression with doc set fields by id model id request fields public fields for model id fields in; why: ModelProfile.public groups the supplied clauses as one doc entry expression before its value is consumed. + } + # What: gate on check endpoint and default check endpoint before check endpoint and doc; why: public admits check endpoint and doc only for this predicate and excludes the opposite state. + if self.check_endpoint != DEFAULT_CHECK_ENDPOINT: + # What: compute doc entry from check endpoint; why: doc proxy self proxy later reads doc entry, so public must retain the computed value under that name. + doc["checkEndpoint"] = self.check_endpoint + # What: gate on proxy and default proxy before proxy and doc; why: public admits proxy and doc only for this predicate and excludes the opposite state. + if self.proxy != DEFAULT_PROXY: + # What: compute doc entry from proxy; why: doc use model name self use model name later reads doc entry, so public must retain the computed value under that name. + doc["proxy"] = self.proxy + # What: gate on use model name before use model name and doc; why: public admits use model name and doc only for this predicate and excludes the opposite state. + if self.use_model_name is not None: + # What: compute doc entry from use model name; why: doc metadata metadata later reads doc entry, so public must retain the computed value under that name. + doc["useModelName"] = self.use_model_name + # What: compute metadata from metadata; why: if metadata later reads metadata, so public must retain the computed value under that name. + metadata = self.metadata() + # What: gate on metadata before metadata and doc; why: public admits metadata and doc only for this predicate and excludes the opposite state. + if metadata: + # What: compute doc entry from metadata; why: doc upstream timeout s self upstream timeout s later reads doc entry, so public must retain the computed value under that name. + doc["metadata"] = metadata + # What: gate on upstream timeout s before upstream timeout s and doc; why: public admits upstream timeout s and doc only for this predicate and excludes the opposite state. + if self.upstream_timeout_s is not None: + # What: compute doc entry from upstream timeout s; why: return doc later reads doc entry, so public must retain the computed value under that name. + doc["upstreamTimeoutS"] = self.upstream_timeout_s + # What: return doc from public; why: public exposes doc so its caller can continue with the function\'s computed outcome. + return doc + + +# What: define ModelCatalog as the owner of __init__ and empty and load and get and public; why: daemon callers use this class boundary so those methods share one model catalog state invariant. +class ModelCatalog: + # What: define __init__ around profiles and settings and selectors and routing profiles and path; why: its direct callers call __init__ for init and rely on this exact input and result contract. + def __init__( + # What: declare the self input for __init__; why: __init__ consumes self during self profiles dict profiles, so callers must bind it with the other signature inputs. + self, + # What: declare the profiles input for __init__; why: __init__ consumes profiles during self profiles dict profiles, so callers must bind it with the other signature inputs. + profiles: dict[str, ModelProfile], + # What: declare the settings input for __init__; why: __init__ consumes settings during settings settings or router settings, so callers must bind it with the other signature inputs. + settings: RouterSettings | None = None, + # What: mark the remaining parameters as keyword-only; why: __init__ prevents callers from confusing adjacent lifecycle and timing arguments. + *, + # What: declare the selectors input for __init__; why: __init__ consumes selectors during self selectors dict selectors or, so callers must bind it with the other signature inputs. + selectors: dict[str, ModelSelector] | None = None, + # What: declare the routing profiles input for __init__; why: __init__ consumes routing profiles during self routing profiles dict routing profiles or, so callers must bind it with the other signature inputs. + routing_profiles: dict[str, RoutingProfile] | None = None, + # What: declare the path input for __init__; why: __init__ consumes path during self path path, so callers must bind it with the other signature inputs. + path: str | None = None, + # What: complete the enclosing predicate with def init profiles dict str model profile settings router settings selectors; why: ModelCatalog.__init__ groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: compute profiles from dict and profiles; why: canonical set self profiles later reads profiles, so __init__ must retain the computed value under that name. + self._profiles = dict(profiles) + # What: initialize aliases as an empty runtime accumulator; why: ModelCatalog.__init__ appends or maps entries into it during for alias in profile aliases before consuming the aggregate. + aliases: dict[str, str] = {} + # What: compute canonical from set and profiles; why: if alias in canonical later reads canonical, so __init__ must retain the computed value under that name. + canonical = set(self._profiles) + # What: iterate across items and profiles to perform model id and name; why: __init__ repeats the body only while or for the loop header admits an iteration. + for name, profile in self._profiles.items(): + # What: call _model_id with name; why: __init__ invokes _model_id while performing model id profile name; the call advances that operation through its result or side effect. + _model_id(name) + # What: call _model_id with name and profile; why: __init__ invokes _model_id while performing if name profile name; the call advances that operation through its result or side effect. + _model_id(profile.name) + # What: gate on name and profile before catalog error and name and profile; why: __init__ admits catalog error and name and profile only for this predicate and excludes the opposite state. + if name != profile.name: + # What: raise CatalogError for the caller; why: ModelCatalog.__init__ stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"profile key {name!r} must match profile name {profile.name!r}") + # What: iterate across aliases and profile to perform alias and model id; why: __init__ repeats the body only while or for the loop header admits an iteration. + for alias in profile.aliases: + # What: compute alias from model id and alias; why: if alias in canonical later reads alias, so __init__ must retain the computed value under that name. + alias = _model_id(alias) + # What: gate on alias and canonical before catalog error and alias; why: __init__ admits catalog error and alias only for this predicate and excludes the opposite state. + if alias in canonical: + # What: raise CatalogError for the caller; why: ModelCatalog.__init__ stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"model alias {alias!r} conflicts with a configured profile") + # What: gate on alias and aliases before catalog error and alias and name and aliases; why: __init__ admits catalog error and alias and name and aliases only for this predicate and excludes the opposite state. + if alias in aliases: + # What: raise CatalogError for the caller; why: ModelCatalog.__init__ stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f model alias alias r is portion of the enclosing predicate; why: this clause remains in __init__\'s enclosing expression so its grouping and evaluation order stay intact. + f"model alias {alias!r} is assigned to both {aliases[alias]!r} and {name!r}" + # What: complete the CatalogError call with alias; why: ModelCatalog.__init__ groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: compute aliases entry from name; why: self aliases aliases later reads aliases entry, so __init__ must retain the computed value under that name. + aliases[alias] = name + # What: compute aliases from aliases; why: the enclosing return or state update later reads aliases, so __init__ must retain the computed value under that name. + self._aliases = aliases + # What: compute selectors from dict and selectors; why: for name selector in self selectors items later reads selectors, so __init__ must retain the computed value under that name. + self._selectors = dict(selectors or {}) + # What: compute occupied from canonical and set and aliases; why: if name in occupied later reads occupied, so __init__ must retain the computed value under that name. + occupied = canonical | set(aliases) + # What: iterate across items and selectors to perform model id and name; why: __init__ repeats the body only while or for the loop header admits an iteration. + for name, selector in self._selectors.items(): + # What: call _model_id with name; why: __init__ invokes _model_id while performing if name selector name; the call advances that operation through its result or side effect. + _model_id(name) + # What: gate on name and selector before catalog error and name and selector; why: __init__ admits catalog error and name and selector only for this predicate and excludes the opposite state. + if name != selector.name: + # What: raise CatalogError for the caller; why: ModelCatalog.__init__ stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f selector key name r must portion of the enclosing predicate; why: this clause remains in __init__\'s enclosing expression so its grouping and evaluation order stay intact. + f"selector key {name!r} must match selector name {selector.name!r}" + # What: complete the CatalogError call with name; why: ModelCatalog.__init__ groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: gate on name and occupied before catalog error and name; why: __init__ admits catalog error and name only for this predicate and excludes the opposite state. + if name in occupied: + # What: raise CatalogError for the caller; why: ModelCatalog.__init__ stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"selector {name!r} conflicts with a model ID or alias") + # What: iterate across targets and selector to perform target and selectors and catalog error and name; why: __init__ repeats the body only while or for the loop header admits an iteration. + for target in selector.targets: + # What: gate on target and selectors before catalog error and name and target; why: __init__ admits catalog error and name and target only for this predicate and excludes the opposite state. + if target in self._selectors: + # What: raise CatalogError for the caller; why: ModelCatalog.__init__ stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f selector name r target target portion of the enclosing predicate; why: this clause remains in __init__\'s enclosing expression so its grouping and evaluation order stay intact. + f"selector {name!r} target {target!r} cannot reference another selector" + # What: complete the CatalogError call with name; why: ModelCatalog.__init__ groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: establish the handler boundary for the protected operation; why: ModelCatalog.__init__ routes failures to catalog error while preserving cleanup and success flow. + try: + # What: call self.get with target; why: __init__ invokes self.get while performing except catalog error as exc; the call advances that operation through its result or side effect. + self.get(target) + # What: handle catalog error by raise catalog error; why: ModelCatalog.__init__ converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError as exc: + # What: raise CatalogError for the caller; why: ModelCatalog.__init__ stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f selector name r target target portion of the enclosing predicate; why: this clause remains in __init__\'s enclosing expression so its grouping and evaluation order stay intact. + f"selector {name!r} target {target!r} is not a configured model or alias" + # What: apply the from exc portion of the enclosing predicate; why: this clause remains in __init__\'s enclosing expression so its grouping and evaluation order stay intact. + ) from exc + # What: compute routing profiles from dict and routing profiles; why: for name routing profile in self routing profiles items later reads routing profiles, so __init__ must retain the computed value under that name. + self._routing_profiles = dict(routing_profiles or {}) + # What: iterate across items and routing profiles to perform simple name and name; why: __init__ repeats the body only while or for the loop header admits an iteration. + for name, routing_profile in self._routing_profiles.items(): + # What: preserve the exact simple name name profile name literal fragment; why: __init__ passes this fragment verbatim through _simple_name(name, "profile name"), because changing it would alter a protocol payload, serialized fixture, or public message. + _simple_name(name, "profile name") + # What: gate on name and routing profile before catalog error and name and routing profile; why: __init__ admits catalog error and name and routing profile only for this predicate and excludes the opposite state. + if name != routing_profile.name: + # What: raise CatalogError for the caller; why: ModelCatalog.__init__ stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f routing profile key name r portion of the enclosing predicate; why: this clause remains in __init__\'s enclosing expression so its grouping and evaluation order stay intact. + f"routing profile key {name!r} must match profile name {routing_profile.name!r}" + # What: complete the CatalogError call with name; why: ModelCatalog.__init__ groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: gate on pins and routing profile before catalog error and name; why: __init__ admits catalog error and name only for this predicate and excludes the opposite state. + if not routing_profile.pins: + # What: raise CatalogError for the caller; why: ModelCatalog.__init__ stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"profiles.{name}.pins must contain at least one entry") + # What: iterate across pins and routing profile to perform model id and pin; why: __init__ repeats the body only while or for the loop header admits an iteration. + for pin, target in routing_profile.pins: + # What: call _model_id with pin; why: __init__ invokes _model_id while performing if target is not and not; the call advances that operation through its result or side effect. + _model_id(pin) + # What: gate on target and has routable id before catalog error and name and pin and target; why: __init__ admits catalog error and name and pin and target only for this predicate and excludes the opposite state. + if target is not None and not self.has_routable_id(target): + # What: raise CatalogError for the caller; why: ModelCatalog.__init__ stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f profiles name pins pin references portion of the enclosing predicate; why: this clause remains in __init__\'s enclosing expression so its grouping and evaluation order stay intact. + f"profiles.{name}.pins.{pin} references unknown model {target!r}" + # What: complete the CatalogError call with name; why: ModelCatalog.__init__ groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: compute settings from settings and router settings; why: if settings preload model is not later reads settings, so __init__ must retain the computed value under that name. + settings = settings or RouterSettings() + # What: gate on preload model and settings before selector and preload model and catalog error and settings; why: __init__ admits selector and preload model and catalog error and settings only for this predicate and excludes the opposite state. + if settings.preload_model is not None: + # What: gate on selector and preload model and settings before catalog error; why: __init__ admits catalog error only for this predicate and excludes the opposite state. + if self.selector(settings.preload_model) is not None: + # What: raise CatalogError for the caller; why: ModelCatalog.__init__ stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.preload_model must name a concrete model or alias") + # What: compute settings from replace and settings and name and get; why: settings startup routing profile is not later reads settings, so __init__ must retain the computed value under that name. + settings = replace(settings, preload_model=self.get(settings.preload_model).name) + # What: gate on startup routing profile and routing profiles and settings before catalog error; why: __init__ admits catalog error only for this predicate and excludes the opposite state. + if ( + # What: apply the settings startup routing profile is not portion of the enclosing predicate; why: this clause remains in __init__\'s enclosing expression so its grouping and evaluation order stay intact. + settings.startup_routing_profile is not None + # What: apply the and settings startup routing profile not in self routing profiles portion of the enclosing predicate; why: this clause remains in __init__\'s enclosing expression so its grouping and evaluation order stay intact. + and settings.startup_routing_profile not in self._routing_profiles + # What: complete the enclosing predicate with if settings startup routing profile is not and settings startup routing profile not in self routing profiles; why: ModelCatalog.__init__ groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: ModelCatalog.__init__ stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.startup_routing_profile references an unknown profile") + # What: compute settings from settings; why: the enclosing return or state update later reads settings, so __init__ must retain the computed value under that name. + self.settings = settings + # What: compute path from path; why: the enclosing return or state update later reads path, so __init__ must retain the computed value under that name. + self.path = path + + # What: bind empty to the class rather than an instance; why: factory and parser callers construct empty from class-level state without requiring an existing object. + @classmethod + # What: define empty around the current object state; why: the registered API client call empty for empty and rely on this exact input and result contract. + def empty(cls) -> "ModelCatalog": + # What: return no value from empty; why: empty returns no value to callers that depend on its completed result. + return cls({}) + + # What: bind load to the class rather than an instance; why: factory and parser callers construct load from class-level state without requiring an existing object. + @classmethod + # What: define load around path; why: the registered API client call load for load and rely on this exact input and result contract. + def load(cls, path: str) -> "ModelCatalog": + # What: establish the handler boundary for the protected operation; why: ModelCatalog.load routes failures to oserror and tomldecode error and tomllib while preserving cleanup and success flow. + try: + # What: enter the open managed context before raw tomllib load source; why: load releases this resource or lock after raw tomllib load source on both success and failure paths. + with open(path, "rb") as source: + # What: compute raw from load and source and tomllib; why: models raw get models later reads raw, so load must retain the computed value under that name. + raw = tomllib.load(source) + # What: handle oserror and tomldecode error and tomllib by raise catalog error f cannot read catalog path; why: ModelCatalog.load converts that failure into this concrete recovery, response, or cleanup behavior. + except (OSError, tomllib.TOMLDecodeError) as exc: + # What: raise CatalogError for the caller; why: ModelCatalog.load stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"cannot read catalog {path!r}: {exc}") from exc + # What: compute models from get and raw and models; why: if not isinstance models dict later reads models, so load must retain the computed value under that name. + models = raw.get("models") + # What: gate on isinstance and models and dict before catalog error; why: load admits catalog error only for this predicate and excludes the opposite state. + if not isinstance(models, dict): + # What: raise CatalogError for the caller; why: ModelCatalog.load stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("catalog requires a [models] table") + # What: initialize profiles as an empty runtime accumulator; why: ModelCatalog.load appends or maps entries into it during profiles model id name profile model id name value before consuming the aggregate. + profiles: dict[str, ModelProfile] = {} + # What: iterate across items and models to perform profiles and profile and value and model id and name; why: load repeats the body only while or for the loop header admits an iteration. + for name, value in models.items(): + # What: compute profiles entry from profile and value and model id and name; why: raw profiles raw get profiles later reads profiles entry, so load must retain the computed value under that name. + profiles[_model_id(name)] = _profile(_model_id(name), value) + # What: compute raw selectors from get and raw and selectors; why: if not isinstance raw selectors dict later reads raw selectors, so load must retain the computed value under that name. + raw_selectors = raw.get("selectors", {}) + # What: gate on isinstance and raw selectors and dict before catalog error; why: load admits catalog error only for this predicate and excludes the opposite state. + if not isinstance(raw_selectors, dict): + # What: raise CatalogError for the caller; why: ModelCatalog.load stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("selectors must be a table") + # What: compute selectors from model id and name and selector and value; why: selectors selectors later reads selectors, so load must retain the computed value under that name. + selectors = { + # What: call _model_id with name; why: load invokes _model_id while performing for name value in raw selectors items; the call advances that operation through its result or side effect. + _model_id(name): _selector(_model_id(name), value) + # What: call raw_selectors.items with the declared inputs; why: load consumes the raw_selectors.items return value while evaluating for name, value in raw_selectors.items(). + for name, value in raw_selectors.items() + # What: complete the selectors expression with selectors model id name selector model id name value for name; why: ModelCatalog.load groups the supplied clauses as one selectors expression before its value is consumed. + } + # What: compute raw profiles from get and raw and profiles; why: if not isinstance raw profiles dict later reads raw profiles, so load must retain the computed value under that name. + raw_profiles = raw.get("profiles", {}) + # What: gate on isinstance and raw profiles and dict before catalog error; why: load admits catalog error only for this predicate and excludes the opposite state. + if not isinstance(raw_profiles, dict): + # What: raise CatalogError for the caller; why: ModelCatalog.load stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("profiles must be a table") + # What: compute routing profiles from simple name and name and routing profile and value; why: routing profiles routing profiles later reads routing profiles, so load must retain the computed value under that name. + routing_profiles = { + # What: call _simple_name with name and profile and name; why: load invokes _simple_name while performing simple name name profile name value; the call advances that operation through its result or side effect. + _simple_name(name, "profile name"): _routing_profile( + # What: call _simple_name with name and profile and name; why: load consumes the _simple_name return value while evaluating _simple_name(name, "profile name"), value. + _simple_name(name, "profile name"), value + # What: complete the _routing_profile call with simple name and value; why: ModelCatalog.load groups the supplied clauses as one _routing_profile call before its value is consumed. + ) + # What: call raw_profiles.items with the declared inputs; why: load consumes the raw_profiles.items return value while evaluating for name, value in raw_profiles.items(). + for name, value in raw_profiles.items() + # What: complete the routing_profiles expression with routing profiles simple name name profile name routing profile simple name name profile; why: ModelCatalog.load groups the supplied clauses as one routing_profiles expression before its value is consumed. + } + # What: return profiles and router settings and selectors and routing profiles from load; why: load exposes profiles and router settings and selectors and routing profiles so its caller can continue with the function\'s computed outcome. + return cls( + # What: apply the profiles portion of the enclosing predicate; why: this clause remains in load\'s enclosing expression so its grouping and evaluation order stay intact. + profiles, + # What: call _router_settings with get and raw and router and profiles; why: load invokes _router_settings while performing selectors selectors; the call advances that operation through its result or side effect. + _router_settings(raw.get("router", {}), profiles), + # What: supply selectors to cls; why: load binds this selectors value to cls's selectors input. + selectors=selectors, + # What: supply routing profiles to cls; why: load binds this routing profiles value to cls's routing profiles input. + routing_profiles=routing_profiles, + # What: supply path to cls; why: load binds this path value to cls's path input. + path=path, + # What: complete the cls call with selectors and routing profiles and path; why: ModelCatalog.load groups the supplied clauses as one cls call before its value is consumed. + ) + + # What: define get around name; why: its direct callers call get for get and rely on this exact input and result contract. + def get(self, name: str) -> ModelProfile: + # What: establish the handler boundary for the protected operation; why: ModelCatalog.get routes failures to key error while preserving cleanup and success flow. + try: + # What: return profiles and get and name and aliases from get; why: get exposes profiles and get and name and aliases so its caller can continue with the function\'s computed outcome. + return self._profiles[self._aliases.get(name, name)] + # What: handle key error by raise catalog error f unknown model profile name; why: ModelCatalog.get converts that failure into this concrete recovery, response, or cleanup behavior. + except KeyError as exc: + # What: raise CatalogError for the caller; why: ModelCatalog.get stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"unknown model profile {name!r}") from exc + + # What: define public around the current object state; why: its direct callers call public for public and rely on this exact input and result contract. + def public(self) -> list[dict[str, Any]]: + # What: return public and name and sorted and profiles from public; why: public exposes public and name and sorted and profiles so its caller can continue with the function\'s computed outcome. + return [self._profiles[name].public() for name in sorted(self._profiles)] + + # What: define public_selectors around the current object state; why: its direct callers call public_selectors for public selectors and rely on this exact input and result contract. + def public_selectors(self) -> list[dict[str, Any]]: + # What: return public and name and sorted and selectors from public_selectors; why: public_selectors exposes public and name and sorted and selectors so its caller can continue with the function\'s computed outcome. + return [self._selectors[name].public() for name in sorted(self._selectors)] + + # What: define public_routing_profiles around the current object state; why: its direct callers call public_routing_profiles for public routing profiles and rely on this exact input and result contract. + def public_routing_profiles(self) -> list[dict[str, Any]]: + # What: return public and name and sorted and routing profiles from public_routing_profiles; why: public_routing_profiles exposes public and name and sorted and routing profiles so its caller can continue with the function\'s computed outcome. + return [ + # What: call self._routing_profiles.public with the declared inputs; why: public_routing_profiles consumes the self._routing_profiles.public return value while evaluating self._routing_profiles[name].public() for name in sorted(self._routing_p. + self._routing_profiles[name].public() for name in sorted(self._routing_profiles) + # What: complete the public_routing_profiles signature with self; why: ModelCatalog.public_routing_profiles groups the supplied clauses as one public_routing_profiles signature before its value is consumed. + ] + + # What: define profiles around the current object state; why: its direct callers call profiles for profiles and rely on this exact input and result contract. + def profiles(self) -> tuple[ModelProfile, ...]: + """Return immutable profile values for internal identity matching.""" + # What: document return immutable profile values for internal in the profiles docstring; why: introspection and maintainers read this exact docstring fragment to understand profiles behavior without executing it. + # What: return tuple and profiles and name and sorted from profiles; why: profiles exposes tuple and profiles and name and sorted so its caller can continue with the function\'s computed outcome. + return tuple(self._profiles[name] for name in sorted(self._profiles)) + + # What: define selector around name; why: its direct callers call selector for selector and rely on this exact input and result contract. + def selector(self, name: str) -> ModelSelector | None: + # What: return get and name and selectors from selector; why: selector exposes get and name and selectors so its caller can continue with the function\'s computed outcome. + return self._selectors.get(name) + + # What: define routing_profile around name; why: its direct callers call routing_profile for routing profile and rely on this exact input and result contract. + def routing_profile(self, name: str) -> RoutingProfile | None: + # What: return get and name and routing profiles from routing_profile; why: routing_profile exposes get and name and routing profiles so its caller can continue with the function\'s computed outcome. + return self._routing_profiles.get(name) + + # What: define has_routable_id around name; why: its direct callers call has_routable_id for has routable id and rely on this exact input and result contract. + def has_routable_id(self, name: str) -> bool: + # What: return name and selectors and profiles and aliases from has_routable_id; why: has_routable_id exposes name and selectors and profiles and aliases so its caller can continue with the function\'s computed outcome. + return name in self._selectors or name in self._profiles or name in self._aliases + + # What: define listed_model_ids around the current object state; why: its direct callers call listed_model_ids for listed model ids and rely on this exact input and result contract. + def listed_model_ids(self) -> tuple[str, ...]: + """Return the OpenAI-visible IDs without exposing hidden canonical profiles.""" + # What: document return the open ai visible ids without exposing in the listed_model_ids docstring; why: introspection and maintainers read this exact docstring fragment to understand listed model ids behavior without executing it. + # What: initialize result as an empty runtime accumulator; why: ModelCatalog.listed_model_ids appends or maps entries into it during result append name before consuming the aggregate. + result: list[str] = [] + # What: iterate across sorted and profiles to perform profile and profiles and name; why: listed_model_ids repeats the body only while or for the loop header admits an iteration. + for name in sorted(self._profiles): + # What: compute profile from profiles and name; why: if profile unlisted later reads profile, so listed_model_ids must retain the computed value under that name. + profile = self._profiles[name] + # What: gate on unlisted and profile before the computed value; why: listed_model_ids admits the computed value only for this predicate and excludes the opposite state. + if profile.unlisted: + # What: apply the continue portion of the enclosing predicate; why: this clause remains in listed_model_ids\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: call result.append with name; why: listed_model_ids invokes result.append while performing if self settings include aliases in list; the call advances that operation through its result or side effect. + result.append(name) + # What: gate on include aliases in list and settings before extend and aliases and result and profile; why: listed_model_ids admits extend and aliases and result and profile only for this predicate and excludes the opposite state. + if self.settings.include_aliases_in_list: + # What: call result.extend with aliases and profile; why: listed_model_ids invokes result.extend while performing result extend; the call advances that operation through its result or side effect. + result.extend(profile.aliases) + # What: call result.extend with name and sorted and selectors and unlisted; why: listed_model_ids invokes result.extend while performing name for name in sorted self selectors; the call advances that operation through its result or side effect. + result.extend( + # What: call sorted with selectors; why: listed_model_ids consumes the sorted return value while evaluating name for name in sorted(self._selectors) if not self._selectors[name].un. + name for name in sorted(self._selectors) if not self._selectors[name].unlisted + # What: complete the result.extend call with name; why: ModelCatalog.listed_model_ids groups the supplied clauses as one result.extend call before its value is consumed. + ) + # What: return tuple and result from listed_model_ids; why: listed_model_ids exposes tuple and result so its caller can continue with the function\'s computed outcome. + return tuple(result) + + # What: define resolve_upstream_path around path; why: its direct callers call resolve_upstream_path for resolve upstream path and rely on this exact input and result contract. + def resolve_upstream_path(self, path: str) -> tuple[str, ModelProfile, str]: + """Resolve the longest configured model-ID prefix from a decoded path.""" + # What: document resolve the longest configured model id prefix in the resolve_upstream_path docstring; why: introspection and maintainers read this exact docstring fragment to understand resolve upstream path behavior without executing it. + # What: compute parts from split and strip and path and value and value; why: for index in range len parts later reads parts, so resolve_upstream_path must retain the computed value under that name. + parts = path.strip("/").split("/") + # What: compute match from the named fixture input; why: match candidate profile join parts index later reads match, so resolve_upstream_path must retain the computed value under that name. + match: tuple[str, ModelProfile, str] | None = None + # What: iterate across range and len and parts to perform candidate and join and parts and index; why: resolve_upstream_path repeats the body only while or for the loop header admits an iteration. + for index in range(1, len(parts) + 1): + # What: compute candidate from join and parts and index and value; why: canonical self aliases get candidate candidate later reads candidate, so resolve_upstream_path must retain the computed value under that name. + candidate = "/".join(parts[:index]) + # What: compute canonical from get and candidate and aliases; why: profile self profiles get canonical later reads canonical, so resolve_upstream_path must retain the computed value under that name. + canonical = self._aliases.get(candidate, candidate) + # What: compute profile from get and canonical and profiles; why: if profile is not later reads profile, so resolve_upstream_path must retain the computed value under that name. + profile = self._profiles.get(canonical) + # What: gate on profile before match and candidate and profile and join and parts; why: resolve_upstream_path admits match and candidate and profile and join and parts only for this predicate and excludes the opposite state. + if profile is not None: + # What: compute match from candidate and profile and join and parts; why: if match is later reads match, so resolve_upstream_path must retain the computed value under that name. + match = candidate, profile, "/" + "/".join(parts[index:]) + # What: gate on match before catalog error; why: resolve_upstream_path admits catalog error only for this predicate and excludes the opposite state. + if match is None: + # What: raise CatalogError for the caller; why: ModelCatalog.resolve_upstream_path stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("upstream path does not begin with a configured model ID") + # What: return match from resolve_upstream_path; why: resolve_upstream_path exposes match so its caller can continue with the function\'s computed outcome. + return match + + # What: define group_for around name; why: its direct callers call group_for for group for and rely on this exact input and result contract. + def group_for(self, name: str) -> RoutingGroup | None: + # What: iterate across groups and settings to perform name and members and group; why: group_for repeats the body only while or for the loop header admits an iteration. + for group in self.settings.groups: + # What: gate on name and members and group before group; why: group_for admits group only for this predicate and excludes the opposite state. + if name in group.members: + # What: return group from group_for; why: group_for exposes group so its caller can continue with the function\'s computed outcome. + return group + # What: return no value from group_for; why: group_for returns no value to callers that depend on its completed result. + return None + + +# What: define _finite_seconds around value and field and minimum and maximum; why: its direct callers call _finite_seconds for finite seconds and rely on this exact input and result contract. +def _finite_seconds(value: object, field: str, *, minimum: float, maximum: float) -> float: + # What: gate on isinstance and value and bool and minimum and maximum before catalog error and field and minimum and maximum; why: _finite_seconds admits catalog error and field and minimum and maximum only for this predicate and excludes the opposite state. + if (not isinstance(value, (int, float)) or isinstance(value, bool) + # What: apply the or not minimum value maximum portion of the enclosing predicate; why: this clause remains in _finite_seconds\'s enclosing expression so its grouping and evaluation order stay intact. + or not minimum <= value <= maximum): + # What: raise CatalogError for the caller; why: _finite_seconds stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must be from {minimum:g} through {maximum:g} seconds") + # What: return float and value from _finite_seconds; why: _finite_seconds exposes float and value so its caller can continue with the function\'s computed outcome. + return float(value) + + +# What: define _router_settings around value and profiles; why: its direct callers call _router_settings for router settings and rely on this exact input and result contract. +def _router_settings(value: object, profiles: dict[str, ModelProfile]) -> RouterSettings: + # What: gate on value before value; why: _router_settings admits value only for this predicate and excludes the opposite state. + if value is None: + # What: initialize value as an empty runtime accumulator; why: _router_settings appends or maps entries into it during if not isinstance value dict before consuming the aggregate. + value = {} + # What: gate on isinstance and value and dict before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if not isinstance(value, dict): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router must be a table") + # What: compute allowed from api keys and default ttl s and unload timeout s and upstream timeout s and scheduler; why: unknown sorted set value allowed later reads allowed, so _router_settings must retain the computed value under that name. + allowed = { + # What: apply the api keys default ttl s unload timeout s upstream timeout s portion of allowed; why: _router_settings uses this clause to evaluate allowed as one grouped value. + "api_keys", "default_ttl_s", "unload_timeout_s", "upstream_timeout_s", + # What: apply the scheduler groups include aliases in list global concurrency limit portion of allowed; why: _router_settings uses this clause to evaluate allowed as one grouped value. + "scheduler", "groups", "include_aliases_in_list", "global_concurrency_limit", + # What: apply the send loading state preload model startup routing profile portion of allowed; why: _router_settings uses this clause to evaluate allowed as one grouped value. + "send_loading_state", "preload_model", "startup_routing_profile", + # What: apply the upstream no activation suffixes portion of allowed; why: _router_settings uses this clause to evaluate allowed as one grouped value. + "upstream_no_activation_suffixes", + # What: apply the activity max entries capture buffer mb portion of allowed; why: _router_settings uses this clause to evaluate allowed as one grouped value. + "activity_max_entries", "capture_buffer_mb", + # What: apply the activity session headers portion of allowed; why: _router_settings uses this clause to evaluate allowed as one grouped value. + "activity_session_headers", + # What: apply the performance disabled performance every s portion of allowed; why: _router_settings uses this clause to evaluate allowed as one grouped value. + "performance_disabled", "performance_every_s", + # What: complete the allowed collection with api keys and default ttl s and unload timeout s and upstream timeout s; why: _router_settings groups the supplied clauses as one allowed collection before its value is consumed. + } + # What: compute unknown from sorted and allowed and set and value; why: if unknown later reads unknown, so _router_settings must retain the computed value under that name. + unknown = sorted(set(value) - allowed) + # What: gate on unknown before catalog error and join and unknown; why: _router_settings admits catalog error and join and unknown only for this predicate and excludes the opposite state. + if unknown: + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"router: unsupported keys: {', '.join(unknown)}") + # What: compute raw keys from get and value and api keys; why: if not isinstance raw keys list or later reads raw keys, so _router_settings must retain the computed value under that name. + raw_keys = value.get("api_keys", []) + # What: gate on isinstance and raw keys and list and all and key before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if (not isinstance(raw_keys, list) or not all(isinstance(key, str) and key and "\x00" not in key + # What: apply the for key in raw keys portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + for key in raw_keys)): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.api_keys must be non-empty strings without NUL") + # What: gate on len and raw keys and set before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if len(set(raw_keys)) != len(raw_keys): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.api_keys must not contain duplicates") + # What: compute scheduler from get and value and scheduler and fifo; why: if scheduler fifo later reads scheduler, so _router_settings must retain the computed value under that name. + scheduler = value.get("scheduler", "fifo") + # What: gate on scheduler before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if scheduler != "fifo": + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.scheduler currently supports only fifo") + # What: compute include aliases in list from get and value and include aliases in list and false; why: if not isinstance include aliases in list bool later reads include aliases in list, so _router_settings must retain the computed value under that name. + include_aliases_in_list = value.get("include_aliases_in_list", False) + # What: gate on isinstance and include aliases in list and bool before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if not isinstance(include_aliases_in_list, bool): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.include_aliases_in_list must be a boolean") + # What: compute global concurrency limit from get and value and global concurrency limit and 0; why: not isinstance global concurrency limit int later reads global concurrency limit, so _router_settings must retain the computed value under that name. + global_concurrency_limit = value.get("global_concurrency_limit", 0) + # What: gate on isinstance and global concurrency limit and bool and int before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if ( + # What: call isinstance with global concurrency limit and int; why: _router_settings invokes isinstance while performing or isinstance global concurrency limit bool; the call advances that operation through its result or side effect. + not isinstance(global_concurrency_limit, int) + # What: call isinstance with global concurrency limit and bool; why: _router_settings invokes isinstance while performing or not global concurrency limit 000 000; the call advances that operation through its result or side effect. + or isinstance(global_concurrency_limit, bool) + # What: apply the or not global concurrency limit 000 000 portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + or not 0 <= global_concurrency_limit <= 1_000_000 + # What: complete the enclosing predicate with if not isinstance global concurrency limit int or isinstance global concurrency limit bool; why: _router_settings groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.global_concurrency_limit must be an integer from 0 through 1000000") + # What: compute send loading state from get and value and send loading state and false; why: if not isinstance send loading state bool later reads send loading state, so _router_settings must retain the computed value under that name. + send_loading_state = value.get("send_loading_state", False) + # What: gate on isinstance and send loading state and bool before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if not isinstance(send_loading_state, bool): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.send_loading_state must be a boolean") + # What: compute preload model from get and value and preload model; why: if preload model is not later reads preload model, so _router_settings must retain the computed value under that name. + preload_model = value.get("preload_model") + # What: gate on preload model before preload model and model id; why: _router_settings admits preload model and model id only for this predicate and excludes the opposite state. + if preload_model is not None: + # What: compute preload model from model id and preload model; why: preload model preload model later reads preload model, so _router_settings must retain the computed value under that name. + preload_model = _model_id(preload_model) + # What: compute startup routing profile from get and value and startup routing profile; why: if startup routing profile is not later reads startup routing profile, so _router_settings must retain the computed value under that name. + startup_routing_profile = value.get("startup_routing_profile") + # What: gate on startup routing profile before startup routing profile and simple name; why: _router_settings admits startup routing profile and simple name only for this predicate and excludes the opposite state. + if startup_routing_profile is not None: + # What: compute startup routing profile from simple name and startup routing profile and router and startup routing profile; why: startup routing profile router startup routing profile later reads startup routing profile, so _router_settings must retain the computed value under that name. + startup_routing_profile = _simple_name( + # What: apply the startup routing profile router startup routing profile portion of startup routing profile; why: _router_settings uses this clause to evaluate startup routing profile as one grouped value. + startup_routing_profile, "router.startup_routing_profile" + # What: complete the _simple_name call with startup routing profile; why: _router_settings groups the supplied clauses as one _simple_name call before its value is consumed. + ) + # What: evaluate and capture upstream no activation suffixes value get; why: the enclosing qualifier uses the captured result in its next validation or artifact step. + upstream_no_activation_suffixes = value.get( + # What: call list with default upstream no activation suffixes; why: _router_settings consumes the list return value while evaluating "upstream_no_activation_suffixes", list(DEFAULT_UPSTREAM_NO_ACTIVATION_S. + "upstream_no_activation_suffixes", list(DEFAULT_UPSTREAM_NO_ACTIVATION_SUFFIXES) + # What: complete the value.get call with list; why: _router_settings groups the supplied clauses as one value.get call before its value is consumed. + ) + # What: gate on isinstance and upstream no activation suffixes and list and len and all before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if ( + # What: call isinstance with upstream no activation suffixes and list; why: _router_settings invokes isinstance while performing or len upstream no activation suffixes; the call advances that operation through its result or side effect. + not isinstance(upstream_no_activation_suffixes, list) + # What: call len with upstream no activation suffixes; why: _router_settings invokes len while performing or not all; the call advances that operation through its result or side effect. + or len(upstream_no_activation_suffixes) > 64 + # What: call all with suffix and upstream no activation suffixes and isinstance and str; why: _router_settings invokes all while performing isinstance suffix str and upstream suffix fullmatch suffix; the call advances that operation through its result or side effect. + or not all( + # What: call isinstance with suffix and str; why: _router_settings invokes isinstance while performing for suffix in upstream no activation suffixes; the call advances that operation through its result or side effect. + isinstance(suffix, str) and _UPSTREAM_SUFFIX.fullmatch(suffix) + # What: apply the for suffix in upstream no activation suffixes portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + for suffix in upstream_no_activation_suffixes + # What: complete the all call with suffix; why: _router_settings groups the supplied clauses as one all call before its value is consumed. + ) + # What: complete the enclosing predicate with if not isinstance upstream no activation suffixes list or len upstream no activation suffixes 64; why: _router_settings groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the router upstream no activation suffixes must contain at most safe portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + "router.upstream_no_activation_suffixes must contain at most 64 safe dot suffixes" + # What: complete the CatalogError call with ordered positional inputs; why: _router_settings groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: gate on len and upstream no activation suffixes and set before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if len(set(upstream_no_activation_suffixes)) != len(upstream_no_activation_suffixes): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.upstream_no_activation_suffixes must not contain duplicates") + # What: compute activity max entries from get and value and activity max entries and 1000; why: if not isinstance activity max entries int or later reads activity max entries, so _router_settings must retain the computed value under that name. + activity_max_entries = value.get("activity_max_entries", 1000) + # What: gate on isinstance and activity max entries and bool and int before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if (not isinstance(activity_max_entries, int) or isinstance(activity_max_entries, bool) + # What: apply the or not activity max entries 000 portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + or not 1 <= activity_max_entries <= 100_000): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.activity_max_entries must be an integer from 1 through 100000") + # What: compute capture buffer mb from get and value and capture buffer mb and 0; why: if not isinstance capture buffer mb int or later reads capture buffer mb, so _router_settings must retain the computed value under that name. + capture_buffer_mb = value.get("capture_buffer_mb", 0) + # What: gate on isinstance and capture buffer mb and bool and int before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if (not isinstance(capture_buffer_mb, int) or isinstance(capture_buffer_mb, bool) + # What: apply the or not capture buffer mb portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + or not 0 <= capture_buffer_mb <= 256): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.capture_buffer_mb must be an integer from 0 through 256") + # What: evaluate and capture activity session headers value get; why: the enclosing qualifier uses the captured result in its next validation or artifact step. + activity_session_headers = value.get( + # What: call list with default activity session headers; why: _router_settings consumes the list return value while evaluating "activity_session_headers", list(DEFAULT_ACTIVITY_SESSION_HEADERS). + "activity_session_headers", list(DEFAULT_ACTIVITY_SESSION_HEADERS) + # What: complete the value.get call with list; why: _router_settings groups the supplied clauses as one value.get call before its value is consumed. + ) + # What: gate on isinstance and activity session headers and list and len and all before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if ( + # What: call isinstance with activity session headers and list; why: _router_settings invokes isinstance while performing or len activity session headers; the call advances that operation through its result or side effect. + not isinstance(activity_session_headers, list) + # What: call len with activity session headers; why: _router_settings invokes len while performing or not all; the call advances that operation through its result or side effect. + or len(activity_session_headers) > 16 + # What: call all with header and activity session headers and isinstance and str; why: _router_settings invokes all while performing isinstance header str and http header name fullmatch header; the call advances that operation through its result or side effect. + or not all( + # What: call isinstance with header and str; why: _router_settings invokes isinstance while performing for header in activity session headers; the call advances that operation through its result or side effect. + isinstance(header, str) and _HTTP_HEADER_NAME.fullmatch(header) + # What: apply the for header in activity session headers portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + for header in activity_session_headers + # What: complete the all call with header; why: _router_settings groups the supplied clauses as one all call before its value is consumed. + ) + # What: complete the enclosing predicate with if not isinstance activity session headers list or len activity session headers 16; why: _router_settings groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the router activity session headers must contain at most safe portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + "router.activity_session_headers must contain at most 16 safe HTTP header names" + # What: complete the CatalogError call with ordered positional inputs; why: _router_settings groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: compute normalized session headers from tuple and lower and header and activity session headers; why: if len set normalized session headers len normalized session headers later reads normalized session headers, so _router_settings must retain the computed value under that name. + normalized_session_headers = tuple(header.lower() for header in activity_session_headers) + # What: gate on len and normalized session headers and set before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if len(set(normalized_session_headers)) != len(normalized_session_headers): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.activity_session_headers must not contain duplicates") + # What: gate on any and header and normalized session headers and split before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if any( + # What: apply the authorization in header or token in portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + "authorization" in header or "token" in header or "secret" in header + # What: call header.split with value; why: _router_settings invokes header.split while performing for header in normalized session headers; the call advances that operation through its result or side effect. + or ("api" in header.split("-") and "key" in header.split("-")) + # What: apply the for header in normalized session headers portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + for header in normalized_session_headers + # What: complete the any call with header; why: _router_settings groups the supplied clauses as one any call before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.activity_session_headers must not name credential headers") + # What: compute performance disabled from get and value and performance disabled and false; why: if not isinstance performance disabled bool later reads performance disabled, so _router_settings must retain the computed value under that name. + performance_disabled = value.get("performance_disabled", False) + # What: gate on isinstance and performance disabled and bool before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if not isinstance(performance_disabled, bool): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.performance_disabled must be a boolean") + # What: compute performance every s from finite seconds and get and value and router and performance every s; why: value get performance every s later reads performance every s, so _router_settings must retain the computed value under that name. + performance_every_s = _finite_seconds( + # What: call value.get with performance every s and 5; why: _router_settings invokes value.get while performing router performance every s minimum maximum; the call advances that operation through its result or side effect. + value.get("performance_every_s", 5), + # What: supply minimum to _finite_seconds; why: _router_settings binds this 5 value to _finite_seconds's minimum input. + "router.performance_every_s", minimum=5, maximum=3600, + # What: complete the _finite_seconds call with minimum and maximum; why: _router_settings groups the supplied clauses as one _finite_seconds call before its value is consumed. + ) + # What: compute raw groups from get and value and groups; why: if not isinstance raw groups dict later reads raw groups, so _router_settings must retain the computed value under that name. + raw_groups = value.get("groups", {}) + # What: gate on isinstance and raw groups and dict before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if not isinstance(raw_groups, dict): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("router.groups must be a table") + # What: initialize groups as an empty runtime accumulator; why: _router_settings appends or maps entries into it during raise catalog error f router groups name must be before consuming the aggregate. + groups: list[RoutingGroup] = [] + # What: compute claimed from set; why: if len set members len members later reads claimed, so _router_settings must retain the computed value under that name. + claimed: set[str] = set() + # What: iterate across items and raw groups to perform name and simple name and raw name; why: _router_settings repeats the body only while or for the loop header admits an iteration. + for raw_name, raw_group in raw_groups.items(): + # What: compute name from simple name and raw name and router and group and names; why: raise catalog error f router groups name must later reads name, so _router_settings must retain the computed value under that name. + name = _simple_name(raw_name, "router group names") + # What: gate on isinstance and raw group and dict before catalog error and name; why: _router_settings admits catalog error and name only for this predicate and excludes the opposite state. + if not isinstance(raw_group, dict): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"router.groups.{name} must be a table") + # What: compute unknown from sorted and set and raw group and members and swap; why: if unknown later reads unknown, so _router_settings must retain the computed value under that name. + unknown = sorted(set(raw_group) - {"members", "swap", "exclusive", "persistent"}) + # What: gate on unknown before catalog error and name and join and unknown; why: _router_settings admits catalog error and name and join and unknown only for this predicate and excludes the opposite state. + if unknown: + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"router.groups.{name}: unsupported keys: {', '.join(unknown)}") + # What: compute members from get and raw group and members; why: if not isinstance members list or later reads members, so _router_settings must retain the computed value under that name. + members = raw_group.get("members") + # What: gate on members and isinstance and list and all and member before catalog error and name; why: _router_settings admits catalog error and name only for this predicate and excludes the opposite state. + if (not isinstance(members, list) or not members + # What: call all with member and members and isinstance and str; why: _router_settings invokes all while performing raise catalog error f router groups name members; the call advances that operation through its result or side effect. + or not all(isinstance(member, str) and member in profiles for member in members)): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"router.groups.{name}.members must name configured models") + # What: gate on intersection and members and len and claimed and set before catalog error; why: _router_settings admits catalog error only for this predicate and excludes the opposite state. + if len(set(members)) != len(members) or claimed.intersection(members): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError("a model can belong to only one router group") + # What: call claimed.update with members; why: _router_settings invokes claimed.update while performing flags key raw group get key default for; the call advances that operation through its result or side effect. + claimed.update(members) + # What: compute flags from key and get and default and raw group and swap; why: if not all isinstance flag bool later reads flags, so _router_settings must retain the computed value under that name. + flags = {key: raw_group.get(key, default) for key, default in + # What: apply the swap exclusive persistent portion of flags; why: _router_settings uses this clause to evaluate flags as one grouped value. + (("swap", True), ("exclusive", True), ("persistent", False))} + # What: gate on all and isinstance and flag and bool and values before catalog error and name; why: _router_settings admits catalog error and name only for this predicate and excludes the opposite state. + if not all(isinstance(flag, bool) for flag in flags.values()): + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"router.groups.{name} flags must be booleans") + # The native coordinator deliberately owns exactly one resident child. + # Accepting llama-swap's coexistence flags here would silently promise + # a scheduling policy we cannot implement. Fail atomically at reload + # time instead; an operator can express the supported policy as an + # exclusive swapping group, or a singleton persistent protected slot. + # What: gate on flags before catalog error and name; why: _router_settings admits catalog error and name only for this predicate and excludes the opposite state. + if not flags["exclusive"]: + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f router groups name single resident native routing portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + f"router.groups.{name}: single-resident native routing requires exclusive = true" + # What: complete the CatalogError call with name; why: _router_settings groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: gate on flags before catalog error and name; why: _router_settings admits catalog error and name only for this predicate and excludes the opposite state. + if flags["persistent"] and flags["swap"]: + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"router.groups.{name}: persistent groups must set swap = false") + # What: gate on flags before catalog error and name; why: _router_settings admits catalog error and name only for this predicate and excludes the opposite state. + if not flags["persistent"] and not flags["swap"]: + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f router groups name swap false requires portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + f"router.groups.{name}: swap = false requires multi-resident routing and is unsupported" + # What: complete the CatalogError call with name; why: _router_settings groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: gate on flags and len and members before catalog error and name; why: _router_settings admits catalog error and name only for this predicate and excludes the opposite state. + if flags["persistent"] and len(members) != 1: + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f router groups name a persistent group portion of the enclosing predicate; why: this clause remains in _router_settings\'s enclosing expression so its grouping and evaluation order stay intact. + f"router.groups.{name}: a persistent group needs exactly one member under single-resident routing" + # What: complete the CatalogError call with name; why: _router_settings groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: supply expanded arguments to groups.append; why: _router_settings binds this flags value to groups.append's expanded input. + groups.append(RoutingGroup(name, tuple(members), **flags)) + # What: compute membership from member and name and group and groups; why: if profile group is not and membership get later reads membership, so _router_settings must retain the computed value under that name. + membership = {member: group.name for group in groups for member in group.members} + # What: iterate across items and profiles to perform group and catalog error and profile and get and name; why: _router_settings repeats the body only while or for the loop header admits an iteration. + for name, profile in profiles.items(): + # What: gate on group and profile and get and name and membership before catalog error and name; why: _router_settings admits catalog error and name only for this predicate and excludes the opposite state. + if profile.group is not None and membership.get(name) != profile.group: + # What: raise CatalogError for the caller; why: _router_settings stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name}.group must match router group membership") + # What: return router settings and scheduler and include aliases in list and global concurrency limit from _router_settings; why: _router_settings exposes router settings and scheduler and include aliases in list and global concurrency limit so its caller can continue with the function\'s computed outcome. + return RouterSettings( + # What: supply api keys to tuple; why: _router_settings binds this tuple and raw keys value to tuple's api keys input. + api_keys=tuple(raw_keys), + # What: supply default ttl s to _finite_seconds; why: _router_settings binds this finite seconds and get and value and router and default ttl s value to _finite_seconds's default ttl s input. + default_ttl_s=_finite_seconds(value.get("default_ttl_s", 0), "router.default_ttl_s", minimum=0, maximum=86400), + # What: supply unload timeout s to _finite_seconds; why: _router_settings binds this finite seconds and get and value and router and unload timeout s value to _finite_seconds's unload timeout s input. + unload_timeout_s=_finite_seconds(value.get("unload_timeout_s", 30), "router.unload_timeout_s", minimum=1, maximum=900), + # What: supply upstream timeout s to _finite_seconds; why: _router_settings binds this finite seconds and get and value and router and upstream timeout s value to _finite_seconds's upstream timeout s input. + upstream_timeout_s=_finite_seconds(value.get("upstream_timeout_s", 900), "router.upstream_timeout_s", minimum=1, maximum=7200), + # What: supply scheduler to RouterSettings; why: _router_settings binds this scheduler value to RouterSettings's scheduler input. + scheduler=scheduler, + # What: supply groups to tuple; why: _router_settings binds this tuple and groups value to tuple's groups input. + groups=tuple(groups), + # What: supply include aliases in list to RouterSettings; why: _router_settings binds this include aliases in list value to RouterSettings's include aliases in list input. + include_aliases_in_list=include_aliases_in_list, + # What: supply global concurrency limit to RouterSettings; why: _router_settings binds this global concurrency limit value to RouterSettings's global concurrency limit input. + global_concurrency_limit=global_concurrency_limit, + # What: supply send loading state to RouterSettings; why: _router_settings binds this send loading state value to RouterSettings's send loading state input. + send_loading_state=send_loading_state, + # What: supply preload model to RouterSettings; why: _router_settings binds this preload model value to RouterSettings's preload model input. + preload_model=preload_model, + # What: supply startup routing profile to RouterSettings; why: _router_settings binds this startup routing profile value to RouterSettings's startup routing profile input. + startup_routing_profile=startup_routing_profile, + # What: supply upstream no activation suffixes to tuple; why: _router_settings binds this tuple and upstream no activation suffixes value to tuple's upstream no activation suffixes input. + upstream_no_activation_suffixes=tuple(upstream_no_activation_suffixes), + # What: supply activity max entries to RouterSettings; why: _router_settings binds this activity max entries value to RouterSettings's activity max entries input. + activity_max_entries=activity_max_entries, + # What: supply capture buffer mb to RouterSettings; why: _router_settings binds this capture buffer mb value to RouterSettings's capture buffer mb input. + capture_buffer_mb=capture_buffer_mb, + # What: supply activity session headers to RouterSettings; why: _router_settings binds this normalized session headers value to RouterSettings's activity session headers input. + activity_session_headers=normalized_session_headers, + # What: supply performance disabled to RouterSettings; why: _router_settings binds this performance disabled value to RouterSettings's performance disabled input. + performance_disabled=performance_disabled, + # What: supply performance every s to RouterSettings; why: _router_settings binds this performance every s value to RouterSettings's performance every s input. + performance_every_s=performance_every_s, + # What: complete the RouterSettings call with api keys and default ttl s and unload timeout s and upstream timeout s and scheduler; why: _router_settings groups the supplied clauses as one RouterSettings call before its value is consumed. + ) + + +# What: define _simple_name around name and label; why: its direct callers call _simple_name for simple name and rely on this exact input and result contract. +def _simple_name(name: object, label: str = "names") -> str: + # What: gate on isinstance and name and str and fullmatch and simple name before catalog error and label; why: _simple_name admits catalog error and label only for this predicate and excludes the opposite state. + if not isinstance(name, str) or not _SIMPLE_NAME.fullmatch(name): + # What: raise CatalogError for the caller; why: _simple_name stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{label} must match [A-Za-z0-9][A-Za-z0-9._-]{{0,127}}") + # What: return name from _simple_name; why: _simple_name exposes name so its caller can continue with the function\'s computed outcome. + return name + + +# What: define _valid_model_id around name; why: its direct callers call _valid_model_id for valid model id and rely on this exact input and result contract. +def _valid_model_id(name: object) -> bool: + # What: return bool and isinstance and name and str from _valid_model_id; why: _valid_model_id exposes bool and isinstance and name and str so its caller can continue with the function\'s computed outcome. + return bool( + # What: call isinstance with name and str; why: _valid_model_id invokes isinstance while performing and len name; the call advances that operation through its result or side effect. + isinstance(name, str) + # What: call len with name; why: _valid_model_id invokes len while performing and all model segment fullmatch segment for segment; the call advances that operation through its result or side effect. + and len(name) <= 128 + # What: call all with fullmatch and segment and model segment and split; why: _valid_model_id consumes the all return value while evaluating and all(_MODEL_SEGMENT.fullmatch(segment) for segment in name.split("/"). + and all(_MODEL_SEGMENT.fullmatch(segment) for segment in name.split("/")) + # What: complete the bool call with isinstance; why: _valid_model_id groups the supplied clauses as one bool call before its value is consumed. + ) + + +# What: define _model_id around name; why: its direct callers call _model_id for model id and rely on this exact input and result contract. +def _model_id(name: object) -> str: + # What: gate on valid model id and name before catalog error; why: _model_id admits catalog error only for this predicate and excludes the opposite state. + if not _valid_model_id(name): + # What: raise CatalogError for the caller; why: _model_id stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: preserve the exact model ids must be slash separated a za z0 9 literal fragment; why: _model_id passes this fragment verbatim through "model IDs must be slash-separated [A-Za-z0-9][A-Za-z0-9._:-] segments ", because changing it would alter a protocol payload, serialized fixture, or public message. + # What: preserve the exact with at most characters total literal fragment; why: _model_id passes this fragment verbatim through "model IDs must be slash-separated [A-Za-z0-9][A-Za-z0-9._:-] segments ", because changing it would alter a protocol payload, serialized fixture, or public message. + "model IDs must be slash-separated [A-Za-z0-9][A-Za-z0-9._:-] segments " + "with at most 128 characters total" + # What: complete the CatalogError call with ordered positional inputs; why: _model_id groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: return name from _model_id; why: _model_id exposes name so its caller can continue with the function\'s computed outcome. + return name + + +# What: define _profile around name and value; why: its direct callers call _profile for profile and rely on this exact input and result contract. +def _profile(name: str, value: object) -> ModelProfile: + # What: gate on isinstance and value and dict before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if not isinstance(value, dict): + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name} must be a table") + # What: compute allowed from model and args and port and description and ready timeout s; why: unknown sorted set value allowed later reads allowed, so _profile must retain the computed value under that name. + allowed = { + # What: apply the model args port description ready timeout s ttl s portion of allowed; why: _profile uses this clause to evaluate allowed as one grouped value. + "model", "args", "port", "description", "ready_timeout_s", "ttl_s", + # What: apply the unload timeout s priority group drop fields aliases unlisted portion of allowed; why: _profile uses this clause to evaluate allowed as one grouped value. + "unload_timeout_s", "priority", "group", "drop_fields", "aliases", "unlisted", + # What: apply the concurrency limit send loading state capabilities set fields portion of allowed; why: _profile uses this clause to evaluate allowed as one grouped value. + "concurrency_limit", "send_loading_state", "capabilities", "set_fields", + # What: apply the set fields by id check endpoint proxy use model name name metadata portion of allowed; why: _profile uses this clause to evaluate allowed as one grouped value. + "set_fields_by_id", "check_endpoint", "proxy", "use_model_name", "name", "metadata", + # What: apply the upstream timeout s portion of allowed; why: _profile uses this clause to evaluate allowed as one grouped value. + "upstream_timeout_s", + # What: complete the allowed collection with model and args and port and description; why: _profile groups the supplied clauses as one allowed collection before its value is consumed. + } + # What: compute unknown from sorted and allowed and set and value; why: if unknown later reads unknown, so _profile must retain the computed value under that name. + unknown = sorted(set(value) - allowed) + # What: gate on unknown before catalog error and name and join and unknown; why: _profile admits catalog error and name and join and unknown only for this predicate and excludes the opposite state. + if unknown: + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name}: unsupported keys: {', '.join(unknown)}") + # What: compute model from get and value and model; why: if not isinstance model str or later reads model, so _profile must retain the computed value under that name. + model = value.get("model") + # What: gate on model and isinstance and str and strip before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if not isinstance(model, str) or not model.strip() or "\x00" in model: + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name}.model must be a non-empty string without NUL") + # What: compute raw args from get and value and args; why: if not isinstance raw args list or later reads raw args, so _profile must retain the computed value under that name. + raw_args = value.get("args", []) + # What: gate on isinstance and raw args and list and all and arg before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if not isinstance(raw_args, list) or not all(isinstance(arg, str) and "\x00" not in arg for arg in raw_args): + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name}.args must be an array of strings without NUL") + # The daemon owns these two options. Letting a profile smuggle them through + # produces ambiguous process state and defeats the lifecycle conflict guard. + # What: iterate across raw args to perform option and split and arg; why: _profile repeats the body only while or for the loop header admits an iteration. + for arg in raw_args: + # What: compute option from split and arg and 0 and value and 1; why: if arg or option p or later reads option, so _profile must retain the computed value under that name. + option = arg.split("=", 1)[0] + # What: compute reserved from model and model path and port; why: option startswith and any flag startswith option for later reads reserved, so _profile must retain the computed value under that name. + reserved = ("--model", "--model-path", "--port") + # What: gate on arg and option and startswith and any and flag before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if arg == "--" or option == "-p" or ( + # What: call option.startswith with value; why: _profile consumes the option.startswith return value while evaluating option.startswith("--") and any(flag.startswith(option) for flag in rese. + option.startswith("--") and any(flag.startswith(option) for flag in reserved) + # What: complete the enclosing predicate with arg equals or option equals p or option startswith and; why: _profile groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name}.args must not set --model or --port") + # What: compute port from get and value and port; why: if port is not and not later reads port, so _profile must retain the computed value under that name. + port = value.get("port") + # Port zero is an explicit request for a fresh loopback port on each + # activation. It is not passed through to uvicorn: the native router + # reserves an OS-selected candidate and records that concrete target for + # readiness, proxying, accounting, and re-adoption. ``None`` keeps the + # daemon-wide fixed default for backwards-compatible catalogs. + # What: gate on port and isinstance and bool and int before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if port is not None and (not isinstance(port, int) or isinstance(port, bool) or not 0 <= port <= 65535): + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name}.port must be an integer from 0 through 65535") + # What: compute description from public text and get and value and name and description; why: value get description f models name description later reads description, so _profile must retain the computed value under that name. + description = _public_text( + # What: call value.get with description; why: _profile consumes the value.get return value while evaluating value.get("description"), f"models.{name}.description". + value.get("description"), f"models.{name}.description" + # What: complete the _public_text call with get and name; why: _profile groups the supplied clauses as one _public_text call before its value is consumed. + ) + # What: compute display name from public text and get and value and name and name; why: check endpoint proxy use model name display name metadata json later reads display name, so _profile must retain the computed value under that name. + display_name = _public_text(value.get("name"), f"models.{name}.name") + # What: compute metadata json from metadata json and get and value and name and metadata; why: check endpoint proxy use model name display name metadata json later reads metadata json, so _profile must retain the computed value under that name. + metadata_json = _metadata_json( + # What: call value.get with metadata and the named fixture input; why: _profile consumes the value.get return value while evaluating value.get("metadata", {}), f"models.{name}.metadata". + value.get("metadata", {}), f"models.{name}.metadata" + # What: complete the _metadata_json call with get and name; why: _profile groups the supplied clauses as one _metadata_json call before its value is consumed. + ) + # What: compute upstream timeout s from get and value and upstream timeout s; why: if upstream timeout s is not later reads upstream timeout s, so _profile must retain the computed value under that name. + upstream_timeout_s = value.get("upstream_timeout_s") + # What: gate on upstream timeout s before upstream timeout s and finite seconds and name; why: _profile admits upstream timeout s and finite seconds and name only for this predicate and excludes the opposite state. + if upstream_timeout_s is not None: + # What: compute upstream timeout s from finite seconds and upstream timeout s and name and models and upstream timeout s; why: upstream timeout s f models name upstream timeout s later reads upstream timeout s, so _profile must retain the computed value under that name. + upstream_timeout_s = _finite_seconds( + # What: apply the upstream timeout s f models name upstream timeout s portion of upstream timeout s; why: _profile uses this clause to evaluate upstream timeout s as one grouped value. + upstream_timeout_s, f"models.{name}.upstream_timeout_s", + # What: supply minimum to _finite_seconds; why: _profile binds this 1 value to _finite_seconds's minimum input. + minimum=1, maximum=7200, + # What: complete the _finite_seconds call with minimum and maximum; why: _profile groups the supplied clauses as one _finite_seconds call before its value is consumed. + ) + # What: compute ready timeout s from finite seconds and get and value and name and ready timeout s; why: name model tuple raw args port description later reads ready timeout s, so _profile must retain the computed value under that name. + ready_timeout_s = _finite_seconds(value.get("ready_timeout_s", 120), f"models.{name}.ready_timeout_s", minimum=1, maximum=900) + # What: compute ttl s from get and value and ttl s; why: if ttl s is not later reads ttl s, so _profile must retain the computed value under that name. + ttl_s = value.get("ttl_s") + # What: gate on ttl s before ttl s and finite seconds and name; why: _profile admits ttl s and finite seconds and name only for this predicate and excludes the opposite state. + if ttl_s is not None: + # What: compute ttl s from finite seconds and ttl s and name and models and ttl s; why: ttl s unload timeout s priority group later reads ttl s, so _profile must retain the computed value under that name. + ttl_s = _finite_seconds(ttl_s, f"models.{name}.ttl_s", minimum=0, maximum=86400) + # What: compute unload timeout s from get and value and unload timeout s; why: if unload timeout s is not later reads unload timeout s, so _profile must retain the computed value under that name. + unload_timeout_s = value.get("unload_timeout_s") + # What: gate on unload timeout s before unload timeout s and finite seconds and name; why: _profile admits unload timeout s and finite seconds and name only for this predicate and excludes the opposite state. + if unload_timeout_s is not None: + # What: compute unload timeout s from finite seconds and unload timeout s and name and models and unload timeout s; why: ttl s unload timeout s priority group later reads unload timeout s, so _profile must retain the computed value under that name. + unload_timeout_s = _finite_seconds(unload_timeout_s, f"models.{name}.unload_timeout_s", minimum=1, maximum=900) + # What: compute priority from get and value and priority and 0; why: if not isinstance priority int or later reads priority, so _profile must retain the computed value under that name. + priority = value.get("priority", 0) + # What: gate on isinstance and priority and bool and int before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if not isinstance(priority, int) or isinstance(priority, bool) or not -1000 <= priority <= 1000: + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name}.priority must be an integer from -1000 through 1000") + # What: compute group from get and value and group; why: if group is not later reads group, so _profile must retain the computed value under that name. + group = value.get("group") + # What: gate on group before group and simple name and name; why: _profile admits group and simple name and name only for this predicate and excludes the opposite state. + if group is not None: + # What: compute group from simple name and group and name and models and group; why: ttl s unload timeout s priority group later reads group, so _profile must retain the computed value under that name. + group = _simple_name(group, f"models.{name}.group") + # What: compute drop fields from get and value and drop fields; why: if not isinstance drop fields list or later reads drop fields, so _profile must retain the computed value under that name. + drop_fields = value.get("drop_fields", []) + # What: gate on isinstance and drop fields and list and len before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if not isinstance(drop_fields, list) or len(drop_fields) > 64: + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f models name drop fields must be portion of the enclosing predicate; why: this clause remains in _profile\'s enclosing expression so its grouping and evaluation order stay intact. + f"models.{name}.drop_fields must be at most 64 safe JSON field paths" + # What: complete the CatalogError call with name; why: _profile groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: compute normalized drop fields from tuple and request field path and field and drop fields; why: if len set normalized drop fields len normalized drop fields later reads normalized drop fields, so _profile must retain the computed value under that name. + normalized_drop_fields = tuple( + # What: call _request_field_path with field and name and models and drop fields; why: _profile consumes the _request_field_path return value while evaluating _request_field_path(field, f"models.{name}.drop_fields") for field in dr. + _request_field_path(field, f"models.{name}.drop_fields") for field in drop_fields + # What: complete the tuple call with request field path; why: _profile groups the supplied clauses as one tuple call before its value is consumed. + ) + # What: gate on len and normalized drop fields and set before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if len(set(normalized_drop_fields)) != len(normalized_drop_fields): + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name}.drop_fields must not contain duplicates") + # What: gate on normalized drop fields before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if ("model",) in normalized_drop_fields: + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name}.drop_fields must not remove model") + # What: compute aliases from get and value and aliases; why: not isinstance aliases list later reads aliases, so _profile must retain the computed value under that name. + aliases = value.get("aliases", []) + # What: gate on isinstance and aliases and list and all and len before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if ( + # What: call isinstance with aliases and list; why: _profile invokes isinstance while performing or not all valid model id alias for; the call advances that operation through its result or side effect. + not isinstance(aliases, list) + # What: call all with valid model id and alias and aliases; why: _profile invokes all while performing or len set aliases len aliases; the call advances that operation through its result or side effect. + or not all(_valid_model_id(alias) for alias in aliases) + # What: call len with set and aliases; why: _profile consumes the len return value while evaluating or len(set(aliases)) != len(aliases). + or len(set(aliases)) != len(aliases) + # What: complete the enclosing predicate with if not isinstance aliases list or not all valid model id; why: _profile groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name}.aliases must be distinct valid profile names") + # What: compute unlisted from get and value and unlisted and false; why: if not isinstance unlisted bool later reads unlisted, so _profile must retain the computed value under that name. + unlisted = value.get("unlisted", False) + # What: gate on isinstance and unlisted and bool before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if not isinstance(unlisted, bool): + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name}.unlisted must be a boolean") + # What: compute concurrency limit from get and value and concurrency limit and 0; why: not isinstance concurrency limit int later reads concurrency limit, so _profile must retain the computed value under that name. + concurrency_limit = value.get("concurrency_limit", 0) + # What: gate on isinstance and concurrency limit and bool and int before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if ( + # What: call isinstance with concurrency limit and int; why: _profile invokes isinstance while performing or isinstance concurrency limit bool; the call advances that operation through its result or side effect. + not isinstance(concurrency_limit, int) + # What: call isinstance with concurrency limit and bool; why: _profile invokes isinstance while performing or not concurrency limit 000 000; the call advances that operation through its result or side effect. + or isinstance(concurrency_limit, bool) + # What: apply the or not concurrency limit 000 000 portion of the enclosing predicate; why: this clause remains in _profile\'s enclosing expression so its grouping and evaluation order stay intact. + or not 0 <= concurrency_limit <= 1_000_000 + # What: complete the enclosing predicate with if not isinstance concurrency limit int or isinstance concurrency limit bool; why: _profile groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f models name concurrency limit must be portion of the enclosing predicate; why: this clause remains in _profile\'s enclosing expression so its grouping and evaluation order stay intact. + f"models.{name}.concurrency_limit must be an integer from 0 through 1000000" + # What: complete the CatalogError call with name; why: _profile groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: compute send loading state from get and value and send loading state; why: if send loading state is not and not later reads send loading state, so _profile must retain the computed value under that name. + send_loading_state = value.get("send_loading_state") + # What: gate on send loading state and isinstance and bool before catalog error and name; why: _profile admits catalog error and name only for this predicate and excludes the opposite state. + if send_loading_state is not None and not isinstance(send_loading_state, bool): + # What: raise CatalogError for the caller; why: _profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"models.{name}.send_loading_state must be a boolean") + # What: compute capabilities from capabilities and name and get and value and capabilities; why: concurrency limit send loading state capabilities set fields set fields by id later reads capabilities, so _profile must retain the computed value under that name. + capabilities = _capabilities(name, value.get("capabilities", {})) + # What: compute set fields from request fields and name and get and value and set fields; why: concurrency limit send loading state capabilities set fields set fields by id later reads set fields, so _profile must retain the computed value under that name. + set_fields = _request_fields(name, "set_fields", value.get("set_fields", {})) + # What: compute set fields by id from request fields by id and name and get and value and set fields by id; why: name value get set fields by id later reads set fields by id, so _profile must retain the computed value under that name. + set_fields_by_id = _request_fields_by_id( + # What: call value.get with set fields by id and the named fixture input; why: _profile consumes the value.get return value while evaluating name, value.get("set_fields_by_id", {}). + name, value.get("set_fields_by_id", {}) + # What: complete the _request_fields_by_id call with name and get; why: _profile groups the supplied clauses as one _request_fields_by_id call before its value is consumed. + ) + # What: compute check endpoint from check endpoint and get and default check endpoint and value; why: value get check endpoint default check endpoint later reads check endpoint, so _profile must retain the computed value under that name. + check_endpoint = _check_endpoint( + # What: call value.get with check endpoint and default check endpoint; why: _profile invokes value.get while performing f models name check endpoint; the call advances that operation through its result or side effect. + value.get("check_endpoint", DEFAULT_CHECK_ENDPOINT), + # What: apply the f models name check endpoint portion of check endpoint; why: _profile uses this clause to evaluate check endpoint as one grouped value. + f"models.{name}.check_endpoint", + # What: complete the _check_endpoint call with get and name; why: _profile groups the supplied clauses as one _check_endpoint call before its value is consumed. + ) + # What: compute proxy from proxy template and get and default proxy and value; why: value get proxy default proxy f models name later reads proxy, so _profile must retain the computed value under that name. + proxy = _proxy_template( + # What: call value.get with proxy and default proxy; why: _profile consumes the value.get return value while evaluating value.get("proxy", DEFAULT_PROXY), f"models.{name}.proxy". + value.get("proxy", DEFAULT_PROXY), f"models.{name}.proxy" + # What: complete the _proxy_template call with get and name; why: _profile groups the supplied clauses as one _proxy_template call before its value is consumed. + ) + # What: compute use model name from upstream model name and get and value and name and use model name; why: value get use model name f models name use model name later reads use model name, so _profile must retain the computed value under that name. + use_model_name = _upstream_model_name( + # What: call value.get with use model name; why: _profile consumes the value.get return value while evaluating value.get("use_model_name"), f"models.{name}.use_model_name". + value.get("use_model_name"), f"models.{name}.use_model_name" + # What: complete the _upstream_model_name call with get and name; why: _profile groups the supplied clauses as one _upstream_model_name call before its value is consumed. + ) + # What: compute aliases from list and fromkeys and dict and aliases; why: aliases later reads aliases, so _profile must retain the computed value under that name. + aliases = list(dict.fromkeys([ + # What: apply the aliases portion of aliases; why: _profile uses this clause to evaluate aliases as one grouped value. + *aliases, + # What: apply the model id for model id value in set fields by id portion of aliases; why: _profile uses this clause to evaluate aliases as one grouped value. + *(model_id for model_id, _ in set_fields_by_id if model_id != name), + # What: complete the list call with fromkeys; why: _profile groups the supplied clauses as one list call before its value is consumed. + ])) + # What: return model profile and name and model and port from _profile; why: _profile exposes model profile and name and model and port so its caller can continue with the function\'s computed outcome. + return ModelProfile( + # What: call tuple with raw args; why: _profile invokes tuple while performing ttl s unload timeout s priority group; the call advances that operation through its result or side effect. + name, model, tuple(raw_args), port, description, ready_timeout_s, + # What: apply the ttl s unload timeout s priority group portion of the enclosing predicate; why: this clause remains in _profile\'s enclosing expression so its grouping and evaluation order stay intact. + ttl_s, unload_timeout_s, priority, group, + # What: call tuple with join and path and normalized drop fields and value; why: _profile invokes tuple while performing concurrency limit send loading state capabilities set fields set fields by id; the call advances that operation through its result or side effect. + tuple(".".join(path) for path in normalized_drop_fields), tuple(aliases), unlisted, + # What: apply the concurrency limit send loading state capabilities set fields set fields by id portion of the enclosing predicate; why: this clause remains in _profile\'s enclosing expression so its grouping and evaluation order stay intact. + concurrency_limit, send_loading_state, capabilities, set_fields, set_fields_by_id, + # What: apply the check endpoint proxy use model name display name metadata json portion of the enclosing predicate; why: this clause remains in _profile\'s enclosing expression so its grouping and evaluation order stay intact. + check_endpoint, proxy, use_model_name, display_name, metadata_json, + # What: apply the upstream timeout s portion of the enclosing predicate; why: this clause remains in _profile\'s enclosing expression so its grouping and evaluation order stay intact. + upstream_timeout_s, + # What: complete the ModelProfile call with name and model and tuple and port and description; why: _profile groups the supplied clauses as one ModelProfile call before its value is consumed. + ) + + +# What: define _check_endpoint around value and field; why: its direct callers call _check_endpoint for check endpoint and rely on this exact input and result contract. +def _check_endpoint(value: object, field: str) -> str: + # What: gate on any and isinstance and value and str and len before catalog error and field; why: _check_endpoint admits catalog error and field only for this predicate and excludes the opposite state. + if ( + # What: call isinstance with value and str; why: _check_endpoint invokes isinstance while performing or len value; the call advances that operation through its result or side effect. + not isinstance(value, str) + # What: call len with value; why: _check_endpoint invokes len while performing or not safe http path fullmatch value; the call advances that operation through its result or side effect. + or len(value) > 256 + # What: call _SAFE_HTTP_PATH.fullmatch with value; why: _check_endpoint invokes _SAFE_HTTP_PATH.fullmatch while performing or any segment in for segment; the call advances that operation through its result or side effect. + or not _SAFE_HTTP_PATH.fullmatch(value) + # What: call any with segment and split and value and value and value; why: _check_endpoint consumes the any return value while evaluating or any(segment in {".", ".."} for segment in value.split("/")). + or any(segment in {".", ".."} for segment in value.split("/")) + # What: complete the enclosing predicate with if not isinstance value str or len value 256; why: _check_endpoint groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _check_endpoint stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f field must be an absolute portion of the enclosing predicate; why: this clause remains in _check_endpoint\'s enclosing expression so its grouping and evaluation order stay intact. + f"{field} must be an absolute ASCII path without query, fragment, or traversal" + # What: complete the CatalogError call with field; why: _check_endpoint groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: return value from _check_endpoint; why: _check_endpoint exposes value so its caller can continue with the function\'s computed outcome. + return value + + +# What: define _proxy_template around value and field; why: its direct callers call _proxy_template for proxy template and rely on this exact input and result contract. +def _proxy_template(value: object, field: str) -> str: + # What: gate on isinstance and value and str and len before catalog error and field; why: _proxy_template admits catalog error and field only for this predicate and excludes the opposite state. + if not isinstance(value, str) or len(value) > 512: + # What: raise CatalogError for the caller; why: _proxy_template stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must be a safe loopback HTTP URL template") + # What: compute match from fullmatch and value and proxy template; why: if match is later reads match, so _proxy_template must retain the computed value under that name. + match = _PROXY_TEMPLATE.fullmatch(value) + # What: gate on match before catalog error and field; why: _proxy_template admits catalog error and field only for this predicate and excludes the opposite state. + if match is None: + # What: raise CatalogError for the caller; why: _proxy_template stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f field must be http port portion of the enclosing predicate; why: this clause remains in _proxy_template\'s enclosing expression so its grouping and evaluation order stay intact. + f"{field} must be http://127.0.0.1:${{PORT}} with an optional safe path prefix" + # What: complete the CatalogError call with field; why: _proxy_template groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: compute prefix from group and match and value and prefix; why: if any segment in for segment later reads prefix, so _proxy_template must retain the computed value under that name. + prefix = match.group("prefix") or "" + # What: gate on any and segment and split and prefix before catalog error and field; why: _proxy_template admits catalog error and field only for this predicate and excludes the opposite state. + if any(segment in {".", ".."} for segment in prefix.split("/")): + # What: raise CatalogError for the caller; why: _proxy_template stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f field must be http port portion of the enclosing predicate; why: this clause remains in _proxy_template\'s enclosing expression so its grouping and evaluation order stay intact. + f"{field} must be http://127.0.0.1:${{PORT}} with an optional safe path prefix" + # What: complete the CatalogError call with field; why: _proxy_template groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: gate on prefix before prefix; why: _proxy_template admits prefix only for this predicate and excludes the opposite state. + if prefix == "/": + # What: compute prefix from value; why: return default proxy prefix later reads prefix, so _proxy_template must retain the computed value under that name. + prefix = "" + # What: return default proxy and prefix from _proxy_template; why: _proxy_template exposes default proxy and prefix so its caller can continue with the function\'s computed outcome. + return DEFAULT_PROXY + prefix + + +# What: define _upstream_model_name around value and field; why: its direct callers call _upstream_model_name for upstream model name and rely on this exact input and result contract. +def _upstream_model_name(value: object, field: str) -> str | None: + # What: gate on value before the computed value; why: _upstream_model_name admits the computed value only for this predicate and excludes the opposite state. + if value is None: + # What: return no value from _upstream_model_name; why: _upstream_model_name returns no value to callers that depend on its completed result. + return None + # What: gate on value and any and isinstance and str and len before catalog error and field; why: _upstream_model_name admits catalog error and field only for this predicate and excludes the opposite state. + if ( + # What: call isinstance with value and str; why: _upstream_model_name invokes isinstance while performing or not value; the call advances that operation through its result or side effect. + not isinstance(value, str) + # What: apply the or not value portion of the enclosing predicate; why: this clause remains in _upstream_model_name\'s enclosing expression so its grouping and evaluation order stay intact. + or not value + # What: call len with value; why: _upstream_model_name invokes len while performing or value value strip; the call advances that operation through its result or side effect. + or len(value) > 256 + # What: call value.strip with the declared inputs; why: _upstream_model_name invokes value.strip while performing or any ord character or ord; the call advances that operation through its result or side effect. + or value != value.strip() + # What: call any with character and value and ord and 32 and 127; why: _upstream_model_name consumes the any return value while evaluating or any(ord(character) < 32 or ord(character) == 127 for character in val. + or any(ord(character) < 32 or ord(character) == 127 for character in value) + # What: complete the enclosing predicate with if not isinstance value str or not value or; why: _upstream_model_name groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _upstream_model_name stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f field must be a non empty portion of the enclosing predicate; why: this clause remains in _upstream_model_name\'s enclosing expression so its grouping and evaluation order stay intact. + f"{field} must be a non-empty trimmed string without control characters" + # What: complete the CatalogError call with field; why: _upstream_model_name groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: return value from _upstream_model_name; why: _upstream_model_name exposes value so its caller can continue with the function\'s computed outcome. + return value + + +# What: define _metadata_json around value and field; why: its direct callers call _metadata_json for metadata json and rely on this exact input and result contract. +def _metadata_json(value: object, field: str) -> str: + # What: gate on isinstance and value and dict and all and key before catalog error and field; why: _metadata_json admits catalog error and field only for this predicate and excludes the opposite state. + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + # What: raise CatalogError for the caller; why: _metadata_json stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must be a table with string keys") + # What: establish the handler boundary for the protected operation; why: _metadata_json routes failures to type error and value error while preserving cleanup and success flow. + try: + # What: return dumps and value and json and false and false from _metadata_json; why: _metadata_json exposes dumps and value and json and false and false so its caller can continue with the function\'s computed outcome. + return json.dumps( + # What: supply ensure ascii to json.dumps; why: _metadata_json binds this false value to json.dumps's ensure ascii input. + value, ensure_ascii=False, allow_nan=False, separators=(",", ":"), sort_keys=True + # What: complete the json.dumps call with ensure ascii and allow nan and separators and sort keys; why: _metadata_json groups the supplied clauses as one json.dumps call before its value is consumed. + ) + # What: handle type error and value error by raise catalog error f field must be json compatible; why: _metadata_json converts that failure into this concrete recovery, response, or cleanup behavior. + except (TypeError, ValueError) as exc: + # What: raise CatalogError for the caller; why: _metadata_json stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must be JSON-compatible") from exc + + +# What: define _public_text around value and field; why: its direct callers call _public_text for public text and rely on this exact input and result contract. +def _public_text(value: object, field: str) -> str | None: + # What: gate on value before the computed value; why: _public_text admits the computed value only for this predicate and excludes the opposite state. + if value is None: + # What: return no value from _public_text; why: _public_text returns no value to callers that depend on its completed result. + return None + # What: gate on value and isinstance and str before catalog error and field; why: _public_text admits catalog error and field only for this predicate and excludes the opposite state. + if not isinstance(value, str) or "\x00" in value: + # What: raise CatalogError for the caller; why: _public_text stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must be a string without NUL") + # What: return strip and value from _public_text; why: _public_text exposes strip and value so its caller can continue with the function\'s computed outcome. + return value.strip() or None + + +# What: define _request_field_path around value and field; why: its direct callers call _request_field_path for request field path and rely on this exact input and result contract. +def _request_field_path(value: object, field: str) -> tuple[str, ...]: + # What: gate on isinstance and value and str and len before catalog error and field; why: _request_field_path admits catalog error and field only for this predicate and excludes the opposite state. + if not isinstance(value, str) or len(value) > 128: + # What: raise CatalogError for the caller; why: _request_field_path stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must use safe dot-delimited JSON object paths") + # What: compute path from tuple and split and value and value; why: if not path or len path later reads path, so _request_field_path must retain the computed value under that name. + path = tuple(value.split(".")) + # What: gate on path and len and all and fullmatch and part before catalog error and field; why: _request_field_path admits catalog error and field only for this predicate and excludes the opposite state. + if not path or len(path) > 16 or not all(_SIMPLE_NAME.fullmatch(part) for part in path): + # What: raise CatalogError for the caller; why: _request_field_path stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must use safe dot-delimited JSON object paths") + # What: return path from _request_field_path; why: _request_field_path exposes path so its caller can continue with the function\'s computed outcome. + return path + + +# What: define _request_fields around name and key and value; why: its direct callers call _request_fields for request fields and rely on this exact input and result contract. +def _request_fields(name: str, key: str, value: object) -> tuple[RequestField, ...]: + # What: compute field from name and key and models and value; why: raise catalog error f field must be later reads field, so _request_fields must retain the computed value under that name. + field = f"models.{name}.{key}" + # What: gate on isinstance and value and dict and len before catalog error and field; why: _request_fields admits catalog error and field only for this predicate and excludes the opposite state. + if not isinstance(value, dict) or len(value) > 64: + # What: raise CatalogError for the caller; why: _request_fields stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must be a table with at most 64 JSON field assignments") + # What: initialize hard as an empty runtime accumulator; why: _request_fields appends or maps entries into it during soft if is soft else hard path operation before consuming the aggregate. + hard: dict[tuple[str, ...], RequestField] = {} + # What: initialize soft as an empty runtime accumulator; why: _request_fields appends or maps entries into it during soft if is soft else hard path operation before consuming the aggregate. + soft: dict[tuple[str, ...], RequestField] = {} + # What: iterate across items and value to perform is soft and isinstance and raw key and str and endswith; why: _request_fields repeats the body only while or for the loop header admits an iteration. + for raw_key, raw_value in value.items(): + # What: compute is soft from isinstance and raw key and str and endswith and value; why: raw key if is soft else raw key later reads is soft, so _request_fields must retain the computed value under that name. + is_soft = isinstance(raw_key, str) and raw_key.endswith("?") + # What: compute path from request field path and field and is soft and raw key and 1; why: if path model later reads path, so _request_fields must retain the computed value under that name. + path = _request_field_path( + # What: apply the raw key if is soft else raw key portion of path; why: _request_fields uses this clause to evaluate path as one grouped value. + raw_key[:-1] if is_soft else raw_key, + # What: apply the field portion of path; why: _request_fields uses this clause to evaluate path as one grouped value. + field, + # What: complete the _request_field_path call with is soft and field; why: _request_fields groups the supplied clauses as one _request_field_path call before its value is consumed. + ) + # What: gate on path before catalog error and field; why: _request_fields admits catalog error and field only for this predicate and excludes the opposite state. + if path == ("model",): + # What: raise CatalogError for the caller; why: _request_fields stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must not set model") + # What: establish the handler boundary for the protected operation; why: _request_fields routes failures to type error and value error while preserving cleanup and success flow. + try: + # What: compute value json from dumps and raw value and json and false and false; why: if len value json encode utf 8 536 later reads value json, so _request_fields must retain the computed value under that name. + value_json = json.dumps( + # What: apply the raw value portion of value json; why: _request_fields uses this clause to evaluate value json as one grouped value. + raw_value, + # What: supply allow nan to json.dumps; why: _request_fields binds this false value to json.dumps's allow nan input. + allow_nan=False, + # What: supply ensure ascii to json.dumps; why: _request_fields binds this false value to json.dumps's ensure ascii input. + ensure_ascii=False, + # What: supply separators to json.dumps; why: _request_fields binds this value and value value to json.dumps's separators input. + separators=(",", ":"), + # What: supply sort keys to json.dumps; why: _request_fields binds this true value to json.dumps's sort keys input. + sort_keys=True, + # What: complete the json.dumps call with allow nan and ensure ascii and separators and sort keys; why: _request_fields groups the supplied clauses as one json.dumps call before its value is consumed. + ) + # What: handle type error and value error by raise catalog error f field raw key must be; why: _request_fields converts that failure into this concrete recovery, response, or cleanup behavior. + except (TypeError, ValueError) as exc: + # What: raise CatalogError for the caller; why: _request_fields stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.{raw_key} must be JSON-compatible") from exc + # What: gate on len and encode and value json before catalog error and field and raw key; why: _request_fields admits catalog error and field and raw key only for this predicate and excludes the opposite state. + if len(value_json.encode("utf-8")) > 65_536: + # What: raise CatalogError for the caller; why: _request_fields stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.{raw_key} exceeds the 65536-byte value limit") + # What: compute operation from request field and path and value json and is soft; why: soft if is soft else hard path later reads operation, so _request_fields must retain the computed value under that name. + operation = RequestField(path, value_json, is_soft) + # What: compute result entry from operation; why: the enclosing return or state update later reads result entry, so _request_fields must retain the computed value under that name. + (soft if is_soft else hard)[path] = operation + # What: iterate across intersection and soft and set and hard to perform pop and path and soft; why: _request_fields repeats the body only while or for the loop header admits an iteration. + for path in set(hard).intersection(soft): + # What: call soft.pop with path; why: _request_fields invokes soft.pop while performing return tuple hard path for path; the call advances that operation through its result or side effect. + soft.pop(path) + # What: return tuple and hard and path and soft from _request_fields; why: _request_fields exposes tuple and hard and path and soft so its caller can continue with the function\'s computed outcome. + return tuple(hard[path] for path in sorted(hard)) + tuple( + # What: call sorted with soft; why: _request_fields consumes the sorted return value while evaluating soft[path] for path in sorted(soft). + soft[path] for path in sorted(soft) + # What: complete the tuple call with soft; why: _request_fields groups the supplied clauses as one tuple call before its value is consumed. + ) + + +# What: define _request_fields_by_id around name and value; why: its direct callers call _request_fields_by_id for request fields by id and rely on this exact input and result contract. +def _request_fields_by_id( + # What: declare the name input for _request_fields_by_id; why: _request_fields_by_id consumes name during field f models name set fields by id, so callers must bind it with the other signature inputs. + name: str, value: object +# What: complete the enclosing predicate collection with tuple and str and request field and the named fixture input; why: _request_fields_by_id groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. +) -> tuple[tuple[str, tuple[RequestField, ...]], ...]: + # What: compute field from name and models and set fields by id; why: raise catalog error f field must be later reads field, so _request_fields_by_id must retain the computed value under that name. + field = f"models.{name}.set_fields_by_id" + # What: gate on isinstance and value and dict and len before catalog error and field; why: _request_fields_by_id admits catalog error and field only for this predicate and excludes the opposite state. + if not isinstance(value, dict) or len(value) > 64: + # What: raise CatalogError for the caller; why: _request_fields_by_id stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must be a table with at most 64 model IDs") + # What: initialize result as an empty runtime accumulator; why: _request_fields_by_id appends or maps entries into it during result append model id request fields name f set fields by id model id before consuming the aggregate. + result = [] + # What: iterate across items and value to perform model id and model id; why: _request_fields_by_id repeats the body only while or for the loop header admits an iteration. + for model_id, fields in value.items(): + # What: compute model id from model id and model id; why: result append model id request fields name f set fields by id later reads model id, so _request_fields_by_id must retain the computed value under that name. + model_id = _model_id(model_id) + # What: preserve the exact result append model id request fields name f set fields by id literal fragment; why: _request_fields_by_id passes this fragment verbatim through result.append((model_id, _request_fields(name, f"set_fields_by_id.{model, because changing it would alter a protocol payload, serialized fi. + result.append((model_id, _request_fields(name, f"set_fields_by_id.{model_id}", fields))) + # What: return tuple and sorted and result from _request_fields_by_id; why: _request_fields_by_id exposes tuple and sorted and result so its caller can continue with the function\'s computed outcome. + return tuple(sorted(result)) + + +# What: define _capabilities around name and value; why: its direct callers call _capabilities for capabilities and rely on this exact input and result contract. +def _capabilities(name: str, value: object) -> ModelCapabilities: + # What: compute field from name and models and capabilities; why: raise catalog error f field must be later reads field, so _capabilities must retain the computed value under that name. + field = f"models.{name}.capabilities" + # What: gate on isinstance and value and dict before catalog error and field; why: _capabilities admits catalog error and field only for this predicate and excludes the opposite state. + if not isinstance(value, dict): + # What: raise CatalogError for the caller; why: _capabilities stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must be a table") + # What: compute unknown from sorted and set and value and in and out; why: if unknown later reads unknown, so _capabilities must retain the computed value under that name. + unknown = sorted(set(value) - {"in", "out", "tools", "context"}) + # What: gate on unknown before catalog error and field and join and unknown; why: _capabilities admits catalog error and field and join and unknown only for this predicate and excludes the opposite state. + if unknown: + # What: raise CatalogError for the caller; why: _capabilities stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}: unsupported keys: {', '.join(unknown)}") + + # What: define modalities around key; why: its direct callers call modalities for modalities and rely on this exact input and result contract. + def modalities(key: str) -> tuple[str, ...]: + # What: compute raw from get and key and value; why: if not isinstance raw list or later reads raw, so modalities must retain the computed value under that name. + raw = value.get(key, []) + # What: gate on isinstance and raw and list and all and item before catalog error and field and key; why: modalities admits catalog error and field and key only for this predicate and excludes the opposite state. + if not isinstance(raw, list) or not all(isinstance(item, str) for item in raw): + # What: raise CatalogError for the caller; why: modalities stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.{key} must be an array of supported modalities") + # What: gate on len and raw and set before catalog error and field and key; why: modalities admits catalog error and field and key only for this predicate and excludes the opposite state. + if len(set(raw)) != len(raw): + # What: raise CatalogError for the caller; why: modalities stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.{key} must not contain duplicates") + # What: compute unsupported from sorted and set and raw and text; why: if unsupported later reads unsupported, so modalities must retain the computed value under that name. + unsupported = sorted(set(raw) - {"text"}) + # What: gate on unsupported before catalog error and field and key and join and unsupported; why: modalities admits catalog error and field and key and join and unsupported only for this predicate and excludes the opposite state. + if unsupported: + # What: raise CatalogError for the caller; why: modalities stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: call operation.join with unsupported; why: modalities consumes the operation.join return value while evaluating f"{field}.{key} contains unsupported modalities: {', '.join(unsupported). + f"{field}.{key} contains unsupported modalities: {', '.join(unsupported)}" + # What: complete the CatalogError call with field; why: modalities groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: return tuple and raw from modalities; why: modalities exposes tuple and raw so its caller can continue with the function\'s computed outcome. + return tuple(raw) + + # What: compute tools from get and value and tools and false; why: if not isinstance tools bool later reads tools, so _capabilities must retain the computed value under that name. + tools = value.get("tools", False) + # What: gate on isinstance and tools and bool before catalog error and field; why: _capabilities admits catalog error and field only for this predicate and excludes the opposite state. + if not isinstance(tools, bool): + # What: raise CatalogError for the caller; why: _capabilities stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.tools must be a boolean") + # What: compute context from get and value and context and 0; why: not isinstance context int later reads context, so _capabilities must retain the computed value under that name. + context = value.get("context", 0) + # What: gate on isinstance and context and bool and int before catalog error and field; why: _capabilities admits catalog error and field only for this predicate and excludes the opposite state. + if ( + # What: call isinstance with context and int; why: _capabilities invokes isinstance while performing or isinstance context bool; the call advances that operation through its result or side effect. + not isinstance(context, int) + # What: call isinstance with context and bool; why: _capabilities invokes isinstance while performing or context; the call advances that operation through its result or side effect. + or isinstance(context, bool) + # What: apply the or context portion of the enclosing predicate; why: this clause remains in _capabilities\'s enclosing expression so its grouping and evaluation order stay intact. + or context < 0 + # What: complete the enclosing predicate with if not isinstance context int or isinstance context bool; why: _capabilities groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _capabilities stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.context must be a nonnegative integer") + # What: return model capabilities and tools and context and modalities and in from _capabilities; why: _capabilities exposes model capabilities and tools and context and modalities and in so its caller can continue with the function\'s computed outcome. + return ModelCapabilities(modalities("in"), modalities("out"), tools, context) + + +# What: define _routing_profile around name and value; why: its direct callers call _routing_profile for routing profile and rely on this exact input and result contract. +def _routing_profile(name: str, value: object) -> RoutingProfile: + # What: compute field from name and profiles; why: raise catalog error f field must be later reads field, so _routing_profile must retain the computed value under that name. + field = f"profiles.{name}" + # What: gate on isinstance and value and dict before catalog error and field; why: _routing_profile admits catalog error and field only for this predicate and excludes the opposite state. + if not isinstance(value, dict): + # What: raise CatalogError for the caller; why: _routing_profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must be a table with description and pins") + # What: compute unknown from sorted and set and value and description and pins; why: if unknown later reads unknown, so _routing_profile must retain the computed value under that name. + unknown = sorted(set(value) - {"description", "pins"}) + # What: gate on unknown before catalog error and field and join and unknown; why: _routing_profile admits catalog error and field and join and unknown only for this predicate and excludes the opposite state. + if unknown: + # What: raise CatalogError for the caller; why: _routing_profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}: unsupported keys: {', '.join(unknown)}") + # What: compute description from get and value and description; why: if description is not and later reads description, so _routing_profile must retain the computed value under that name. + description = value.get("description") + # What: gate on description and isinstance and str before catalog error and field; why: _routing_profile admits catalog error and field only for this predicate and excludes the opposite state. + if description is not None and ( + # What: call isinstance with description and str; why: _routing_profile consumes the isinstance return value while evaluating not isinstance(description, str) or "\x00" in description. + not isinstance(description, str) or "\x00" in description + # What: complete the enclosing predicate with description is not and not isinstance description str or; why: _routing_profile groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _routing_profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.description must be a string without NUL") + # What: compute raw pins from get and value and pins; why: if not isinstance raw pins dict or later reads raw pins, so _routing_profile must retain the computed value under that name. + raw_pins = value.get("pins") + # What: gate on raw pins and isinstance and dict before catalog error and field; why: _routing_profile admits catalog error and field only for this predicate and excludes the opposite state. + if not isinstance(raw_pins, dict) or not raw_pins: + # What: raise CatalogError for the caller; why: _routing_profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.pins must contain at least one entry") + # What: initialize pins as an empty runtime accumulator; why: _routing_profile appends or maps entries into it during raise catalog error f field pins pin must before consuming the aggregate. + pins: list[tuple[str, str | None]] = [] + # What: iterate across items and raw pins to perform pin and model id and raw pin; why: _routing_profile repeats the body only while or for the loop header admits an iteration. + for raw_pin, raw_target in raw_pins.items(): + # What: compute pin from model id and raw pin; why: raise catalog error f field pins pin later reads pin, so _routing_profile must retain the computed value under that name. + pin = _model_id(raw_pin) + # What: gate on isinstance and raw target and str before catalog error and field and pin; why: _routing_profile admits catalog error and field and pin only for this predicate and excludes the opposite state. + if not isinstance(raw_target, str): + # What: raise CatalogError for the caller; why: _routing_profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.pins.{pin} must be a model ID or empty string") + # What: compute target from raw target and model id; why: pins append pin target later reads target, so _routing_profile must retain the computed value under that name. + target = _model_id(raw_target) if raw_target else None + # What: call pins.append with pin and target; why: _routing_profile invokes pins.append while performing return routing profile name tuple sorted pins; the call advances that operation through its result or side effect. + pins.append((pin, target)) + # What: return routing profile and name and tuple and description from _routing_profile; why: _routing_profile exposes routing profile and name and tuple and description so its caller can continue with the function\'s computed outcome. + return RoutingProfile(name, tuple(sorted(pins)), description or None) + + +# What: define _selector around name and value; why: its direct callers call _selector for selector and rely on this exact input and result contract. +def _selector(name: str, value: object) -> ModelSelector: + # What: compute field from name and selectors; why: raise catalog error f field must be later reads field, so _selector must retain the computed value under that name. + field = f"selectors.{name}" + # What: gate on isinstance and value and dict before catalog error and field; why: _selector admits catalog error and field only for this predicate and excludes the opposite state. + if not isinstance(value, dict): + # What: raise CatalogError for the caller; why: _selector stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field} must be a table") + # What: compute unknown from sorted and set and value and strategy and targets; why: if unknown later reads unknown, so _selector must retain the computed value under that name. + unknown = sorted( + # What: call set with value; why: _selector consumes the set return value while evaluating set(value) - {"strategy", "targets", "name", "description", "unlisted",. + set(value) - {"strategy", "targets", "name", "description", "unlisted", "metadata"} + # What: complete the sorted call with set; why: _selector groups the supplied clauses as one sorted call before its value is consumed. + ) + # What: gate on unknown before catalog error and field and join and unknown; why: _selector admits catalog error and field and join and unknown only for this predicate and excludes the opposite state. + if unknown: + # What: raise CatalogError for the caller; why: _selector stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}: unsupported keys: {', '.join(unknown)}") + # What: compute strategy from get and value and strategy; why: if strategy spillover later reads strategy, so _selector must retain the computed value under that name. + strategy = value.get("strategy") + # What: gate on strategy before catalog error and field; why: _selector admits catalog error and field only for this predicate and excludes the opposite state. + if strategy == "spillover": + # What: raise CatalogError for the caller; why: _selector stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f field strategy spillover requires multi resident portion of the enclosing predicate; why: this clause remains in _selector\'s enclosing expression so its grouping and evaluation order stay intact. + f"{field}.strategy spillover requires multi-resident or peer capacity and is unsupported" + # What: complete the CatalogError call with field; why: _selector groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: gate on strategy before catalog error and field; why: _selector admits catalog error and field only for this predicate and excludes the opposite state. + if strategy not in {"pin", "warm"}: + # What: raise CatalogError for the caller; why: _selector stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.strategy must be pin or warm") + # What: compute targets from get and value and targets; why: not isinstance targets list later reads targets, so _selector must retain the computed value under that name. + targets = value.get("targets") + # What: gate on targets and isinstance and list and len and all before catalog error and field; why: _selector admits catalog error and field only for this predicate and excludes the opposite state. + if ( + # What: call isinstance with targets and list; why: _selector invokes isinstance while performing or not targets; the call advances that operation through its result or side effect. + not isinstance(targets, list) + # What: apply the or not targets portion of the enclosing predicate; why: this clause remains in _selector\'s enclosing expression so its grouping and evaluation order stay intact. + or not targets + # What: call len with targets; why: _selector invokes len while performing or not all valid model id target for; the call advances that operation through its result or side effect. + or len(targets) > 64 + # What: call all with valid model id and target and targets; why: _selector consumes the all return value while evaluating or not all(_valid_model_id(target) for target in targets). + or not all(_valid_model_id(target) for target in targets) + # What: complete the enclosing predicate with if not isinstance targets list or not targets or; why: _selector groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _selector stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.targets must contain 1 to 64 valid model IDs") + # What: compute display name from get and value and name; why: for key candidate in name display name later reads display name, so _selector must retain the computed value under that name. + display_name = value.get("name") + # What: compute description from get and value and description; why: for key candidate in name display name later reads description, so _selector must retain the computed value under that name. + description = value.get("description") + # What: iterate across display name and description to perform candidate and catalog error and isinstance and str and field; why: _selector repeats the body only while or for the loop header admits an iteration. + for key, candidate in (("name", display_name), ("description", description)): + # What: gate on candidate and isinstance and str before catalog error and field and key; why: _selector admits catalog error and field and key only for this predicate and excludes the opposite state. + if candidate is not None and ( + # What: call isinstance with candidate and str; why: _selector consumes the isinstance return value while evaluating not isinstance(candidate, str) or "\x00" in candidate. + not isinstance(candidate, str) or "\x00" in candidate + # What: complete the enclosing predicate with candidate is not and not isinstance candidate str or; why: _selector groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise CatalogError for the caller; why: _selector stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.{key} must be a string without NUL") + # What: compute unlisted from get and value and unlisted and false; why: if not isinstance unlisted bool later reads unlisted, so _selector must retain the computed value under that name. + unlisted = value.get("unlisted", False) + # What: gate on isinstance and unlisted and bool before catalog error and field; why: _selector admits catalog error and field only for this predicate and excludes the opposite state. + if not isinstance(unlisted, bool): + # What: raise CatalogError for the caller; why: _selector stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError(f"{field}.unlisted must be a boolean") + # What: compute metadata json from metadata json and get and value and field and metadata; why: metadata json later reads metadata json, so _selector must retain the computed value under that name. + metadata_json = _metadata_json(value.get("metadata", {}), f"{field}.metadata") + # What: return model selector and name and strategy and unlisted from _selector; why: _selector exposes model selector and name and strategy and unlisted so its caller can continue with the function\'s computed outcome. + return ModelSelector( + # What: apply the name portion of the enclosing predicate; why: this clause remains in _selector\'s enclosing expression so its grouping and evaluation order stay intact. + name, + # What: apply the strategy portion of the enclosing predicate; why: this clause remains in _selector\'s enclosing expression so its grouping and evaluation order stay intact. + strategy, + # What: call tuple with targets; why: _selector invokes tuple while performing display name or; the call advances that operation through its result or side effect. + tuple(targets), + # What: apply the display name or portion of the enclosing predicate; why: this clause remains in _selector\'s enclosing expression so its grouping and evaluation order stay intact. + display_name or None, + # What: apply the description or portion of the enclosing predicate; why: this clause remains in _selector\'s enclosing expression so its grouping and evaluation order stay intact. + description or None, + # What: apply the unlisted portion of the enclosing predicate; why: this clause remains in _selector\'s enclosing expression so its grouping and evaluation order stay intact. + unlisted, + # What: apply the metadata json portion of the enclosing predicate; why: this clause remains in _selector\'s enclosing expression so its grouping and evaluation order stay intact. + metadata_json, + # What: complete the ModelSelector call with name and strategy and tuple and display name and description; why: _selector groups the supplied clauses as one ModelSelector call before its value is consumed. + ) diff --git a/python/freetoken/daemon/client.py b/python/freetoken/daemon/client.py index 6616778e72..b5ce5360bb 100644 --- a/python/freetoken/daemon/client.py +++ b/python/freetoken/daemon/client.py @@ -21,10 +21,21 @@ # prepare-stop (15s transport budget) + default SIGTERM grace (10s) + reap wait (10s), # with enough HTTP scheduling slack that a valid lifecycle transaction does not look failed. DEFAULT_LIFECYCLE_TIMEOUT = 40.0 +# What: compute default profile timeout from 1920 0; why: return default profile timeout later reads default profile timeout, so client must retain the computed value under that name. +DEFAULT_PROFILE_TIMEOUT = 1920.0 # replacement + recovery readiness (2 * 900s), lifecycle margin # Positional verbs that mean "act as a client"; anything else (bare, or a flag like --host) runs # the server. Kept in one place so the server dispatcher and this parser agree. -CLIENT_VERBS = ("self", "status", "health", "metrics", "stats", "start", "stop", "switch", "logs") +# What: compute client verbs from self and status and health and metrics and stats; why: the enclosing return or state update later reads client verbs, so client must retain the computed value under that name. +CLIENT_VERBS = ( + # What: apply the status health metrics stats models routing profiles portion of client verbs; why: client uses this clause to evaluate client verbs as one grouped value. + "self", "status", "health", "metrics", "stats", "models", "routing-profiles", + # What: apply the activate routing profile clear routing profile start stop shutdown portion of client verbs; why: client uses this clause to evaluate client verbs as one grouped value. + "activate-routing-profile", "clear-routing-profile", "start", "stop", "shutdown", + # What: apply the switch start profile switch profile logs portion of client verbs; why: client uses this clause to evaluate client verbs as one grouped value. + "switch", "start-profile", "switch-profile", "logs", +# What: complete the CLIENT_VERBS collection with self and status and health and metrics; why: client groups the supplied clauses as one CLIENT_VERBS collection before its value is consumed. +) class ClientError(Exception): @@ -36,9 +47,14 @@ def __init__(self, message: str, *, exit_code: int = 1) -> None: def _effective_timeout(verb: str, configured: float | None) -> float: if configured is not None: return configured + # What: gate on verb before default profile timeout; why: _effective_timeout admits default profile timeout only for this predicate and excludes the opposite state. + if verb in {"start-profile", "switch-profile"}: + # What: return default profile timeout from _effective_timeout; why: _effective_timeout exposes default profile timeout so its caller can continue with the function\'s computed outcome. + return DEFAULT_PROFILE_TIMEOUT return ( DEFAULT_LIFECYCLE_TIMEOUT - if verb in {"stop", "switch"} + # What: apply the if verb in stop shutdown switch portion of the enclosing predicate; why: this clause remains in _effective_timeout\'s enclosing expression so its grouping and evaluation order stay intact. + if verb in {"stop", "shutdown", "switch", "start-profile", "switch-profile"} else DEFAULT_TIMEOUT ) @@ -124,7 +140,8 @@ def _build_parser(prog: str) -> argparse.ArgumentParser: "--timeout", type=float, default=None, - help="HTTP timeout (default 10s; stop/switch 40s)", + # What: preserve the exact help http timeout default s stop literal fragment; why: _build_parser passes this fragment verbatim through help="HTTP timeout (default 10s; stop/switch 40s; profiles 960s)", because changing it would alter a protocol payload, serialized fixture, or public message. + help="HTTP timeout (default 10s; stop/switch 40s; profiles 960s)", ) p = argparse.ArgumentParser(prog=prog, description="Control a running ft daemon") @@ -134,12 +151,58 @@ def _build_parser(prog: str) -> argparse.ArgumentParser: sub.add_parser("health", parents=[common], help="Proxied serve health (GET /engine/health)") sub.add_parser("metrics", parents=[common], help="Engine footprint (GET /engine/metrics)") sub.add_parser("stats", parents=[common], help="Proxied serve stats (GET /engine/stats)") + # What: call sub.add_parser with models; why: _build_parser invokes sub.add_parser while performing models parents common; the call advances that operation through its result or side effect. + sub.add_parser( + # What: preserve the exact models parents common literal fragment; why: _build_parser passes this fragment verbatim through "models", parents=[common], because changing it would alter a protocol payload, serialized fixture, or public message. + "models", parents=[common], + # What: preserve the exact help list named freetoken swap model profiles literal fragment; why: _build_parser passes this fragment verbatim through help="List named freetoken-swap model profiles (GET /router/profiles)", because changing it would alter a protocol payload, serialized fixture, or public message. + help="List named freetoken-swap model profiles (GET /router/profiles)", + # What: complete the sub.add_parser call with parents and help; why: _build_parser groups the supplied clauses as one sub.add_parser call before its value is consumed. + ) + # What: call sub.add_parser with routing profiles; why: _build_parser invokes sub.add_parser while performing routing profiles parents common; the call advances that operation through its result or side effect. + sub.add_parser( + # What: preserve the exact routing profiles parents common literal fragment; why: _build_parser passes this fragment verbatim through "routing-profiles", parents=[common], because changing it would alter a protocol payload, serialized fixture, or public message. + "routing-profiles", parents=[common], + # What: preserve the exact help list runtime model id pin profiles literal fragment; why: _build_parser passes this fragment verbatim through help="List runtime model-ID pin profiles (GET /router/profiles)", because changing it would alter a protocol payload, serialized fixture, or public message. + help="List runtime model-ID pin profiles (GET /router/profiles)", + # What: complete the sub.add_parser call with parents and help; why: _build_parser groups the supplied clauses as one sub.add_parser call before its value is consumed. + ) + # What: compute activate routing from add parser and sub and common and activate routing profile and activate; why: activate routing add argument name help routing profile name later reads activate routing, so _build_parser must retain the computed value under that name. + activate_routing = sub.add_parser( + # What: supply parents to sub.add_parser; why: _build_parser binds this common value to sub.add_parser's parents input. + "activate-routing-profile", parents=[common], + # What: supply help to sub.add_parser; why: _build_parser binds this activate and a and runtime and model id value to sub.add_parser's help input. + help="Activate a runtime model-ID pin profile", + # What: complete the sub.add_parser call with parents and help; why: _build_parser groups the supplied clauses as one sub.add_parser call before its value is consumed. + ) + # What: preserve the exact activate routing add argument name help routing profile name literal fragment; why: _build_parser passes this fragment verbatim through activate_routing.add_argument("name", help="Routing profile name"), because changing it would alter a protocol payload, serialized fixture, or public me. + activate_routing.add_argument("name", help="Routing profile name") + # What: call sub.add_parser with clear routing profile; why: _build_parser invokes sub.add_parser while performing clear routing profile parents common; the call advances that operation through its result or side effect. + sub.add_parser( + # What: preserve the exact clear routing profile parents common literal fragment; why: _build_parser passes this fragment verbatim through "clear-routing-profile", parents=[common], because changing it would alter a protocol payload, serialized fixture, or public message. + "clear-routing-profile", parents=[common], + # What: preserve the exact help clear the active runtime model id literal fragment; why: _build_parser passes this fragment verbatim through help="Clear the active runtime model-ID pin profile", because changing it would alter a protocol payload, serialized fixture, or public message. + help="Clear the active runtime model-ID pin profile", + # What: complete the sub.add_parser call with parents and help; why: _build_parser groups the supplied clauses as one sub.add_parser call before its value is consumed. + ) stop = sub.add_parser("stop", parents=[common], help="Stop the serve (POST /engine/stop)") stop.add_argument( "--force", action="store_true", help="stop even if final accounting cannot be sealed (may lose the unobserved token tail)", ) + # What: compute shutdown from add parser and sub and common and shutdown and stop; why: shutdown add argument later reads shutdown, so _build_parser must retain the computed value under that name. + shutdown = sub.add_parser("shutdown", parents=[common], help="Stop the serve and daemon (POST /shutdown)") + # What: call shutdown.add_argument with force; why: _build_parser invokes shutdown.add_argument while performing force; the call advances that operation through its result or side effect. + shutdown.add_argument( + # What: preserve the exact force literal fragment; why: _build_parser passes this fragment verbatim through "--force", because changing it would alter a protocol payload, serialized fixture, or public message. + "--force", + # What: preserve the exact action store true literal fragment; why: _build_parser passes this fragment verbatim through action="store_true", because changing it would alter a protocol payload, serialized fixture, or public message. + action="store_true", + # What: preserve the exact help stop even if final accounting literal fragment; why: _build_parser passes this fragment verbatim through help="stop even if final accounting cannot be sealed (may lose the unobs, because changing it would alter a protocol payload, serialized fixture, or public message. + help="stop even if final accounting cannot be sealed (may lose the unobserved token tail)", + # What: complete the shutdown.add_argument call with action and help; why: _build_parser groups the supplied clauses as one shutdown.add_argument call before its value is consumed. + ) for name in ("start", "switch"): sp = sub.add_parser(name, parents=[common], help=f"POST /engine/{name}") sp.add_argument("model", help="Model path/id") @@ -153,6 +216,16 @@ def _build_parser(prog: str) -> argparse.ArgumentParser: # Everything after `--` is forwarded verbatim to ft serve (opaque passthrough): # ft daemon start MODEL --port 1919 -- --moe-cache-auto --graph 256 sp.add_argument("serve_args", nargs="*", default=[], help="Extra ft serve args (after --)") + # What: iterate across the computed value to perform sp and add parser and name and sub and common; why: _build_parser repeats the body only while or for the loop header admits an iteration. + for name in ("start-profile", "switch-profile"): + # What: compute sp from add parser and name and sub and common and post; why: sp add argument name help named model profile later reads sp, so _build_parser must retain the computed value under that name. + sp = sub.add_parser(name, parents=[common], help=f"POST /engine/{name}") + # What: preserve the exact sp add argument name help named model profile literal fragment; why: _build_parser passes this fragment verbatim through sp.add_argument("name", help="Named model profile from the daemon catalo, because changing it would alter a protocol payload, serialized fixture, or public message. + sp.add_argument("name", help="Named model profile from the daemon catalog") + # What: gate on name before add argument and sp; why: _build_parser admits add argument and sp only for this predicate and excludes the opposite state. + if name == "switch-profile": + # What: preserve the exact sp add argument force action store true help replace literal fragment; why: _build_parser passes this fragment verbatim through sp.add_argument("--force", action="store_true", help="replace even if fi, because changing it would alter a protocol payload, serialized fixture, or pub. + sp.add_argument("--force", action="store_true", help="replace even if final accounting cannot be sealed") lg = sub.add_parser("logs", parents=[common], help="Stream engine logs (SSE, GET /engine/logs)") lg.add_argument("--since", type=int, default=0, help="Replay from this seq cursor") return p @@ -171,23 +244,60 @@ def main(argv: Sequence[str] | None = None, *, prog: str = "ft daemon") -> int: "health": ("GET", "/engine/health", None), "metrics": ("GET", "/engine/metrics", None), "stats": ("GET", "/engine/stats", None), + # What: map the models field as get and router and profiles; why: main carries models through table into method path body table args verb. + "models": ("GET", "/router/profiles", None), + # What: map the routing profiles field as get and router and profiles; why: main carries routing profiles through table into method path body table args verb. + "routing-profiles": ("GET", "/router/profiles", None), "stop": ( "POST", "/engine/stop", {"force": True} if getattr(args, "force", False) else {}, ), + # What: map the shutdown field as getattr and args and post and shutdown and force; why: main carries shutdown through table into method path body table args verb. + "shutdown": ( + # What: apply the post portion of table; why: main uses this clause to evaluate table as one grouped value. + "POST", + # What: apply the shutdown portion of table; why: main uses this clause to evaluate table as one grouped value. + "/shutdown", + # What: map the force field as true; why: main carries force through table into method path body table args verb. + {"force": True} if getattr(args, "force", False) else {}, + # What: complete the table collection with post and shutdown and getattr and args and force and false and force; why: main groups the supplied clauses as one table collection before its value is consumed. + ), } - if args.verb in ("start", "switch"): + # What: gate on verb and args before method and path and body and name and args; why: main admits method and path and body and name and args only for this predicate and excludes the opposite state. + if args.verb == "activate-routing-profile": + # What: map the name field as name and args; why: main carries name through method and path and body into method path body put router profiles active. + method, path, body = "PUT", "/router/profiles/active", {"name": args.name} + # What: gate on verb and args before method and path and body; why: main admits method and path and body only for this predicate and excludes the opposite state. + elif args.verb == "clear-routing-profile": + # What: map the name field as the fixture input; why: main carries name through method and path and body into method path post f engine args verb. + method, path, body = "PUT", "/router/profiles/active", {"name": None} + # What: gate on verb and args before body and dict and model and str and any; why: main admits body and dict and model and str and any only for this predicate and excludes the opposite state. + elif args.verb in ("start", "switch"): body: dict[str, Any] = {"model": args.model, "args": list(args.serve_args)} if args.port is not None: body["port"] = args.port if args.verb == "switch" and args.force: body["force"] = True method, path = "POST", f"/engine/{args.verb}" + # What: gate on verb and args before body and name and args; why: main admits body and name and args only for this predicate and excludes the opposite state. + elif args.verb in ("start-profile", "switch-profile"): + # What: map the name field as name and args; why: main carries name through body into body force true. + body = {"name": args.name} + # What: gate on force and verb and args before body; why: main admits body only for this predicate and excludes the opposite state. + if args.verb == "switch-profile" and args.force: + # What: compute body entry from true; why: method path body table args verb later reads body entry, so main must retain the computed value under that name. + body["force"] = True + # What: compute method and path from verb and args and post and engine; why: method path body table args verb later reads method and path, so main must retain the computed value under that name. + method, path = "POST", f"/engine/{args.verb}" else: method, path, body = table[args.verb] doc = _request_json(method, args.url, path, body=body, token=args.token, timeout=timeout) print(json.dumps(doc, ensure_ascii=False, indent=2, sort_keys=True)) + # What: gate on verb and args and get and doc before the computed value; why: main admits the computed value only for this predicate and excludes the opposite state. + if args.verb in {"start-profile", "switch-profile"} and not doc.get("readiness", {}).get("ready"): + # What: return 1 from main; why: main exposes 1 so its caller can continue with the function\'s computed outcome. + return 1 return 0 except ClientError as exc: print(str(exc), file=sys.stderr) diff --git a/python/freetoken/daemon/inference_proxy.py b/python/freetoken/daemon/inference_proxy.py new file mode 100644 index 0000000000..2e7c16e169 --- /dev/null +++ b/python/freetoken/daemon/inference_proxy.py @@ -0,0 +1,335 @@ +"""Small request-preserving HTTP bridge from freetoken-swap to ``ft serve``. + +No inference dependency is imported here. The daemon only parses the request +JSON long enough to select an allowlisted profile, then forwards the original +bytes and safe HTTP headers to the selected FreeToken engine. +""" +# What: document small request preserving http bridge from freetoken swap in the inference_proxy docstring; why: introspection and maintainers read this exact docstring fragment to understand inference proxy behavior without executing it. +# What: document no inference dependency is imported here in the inference_proxy docstring; why: introspection and maintainers read this exact docstring fragment to understand inference proxy behavior without executing it. +# What: document json long enough to select an in the inference_proxy docstring; why: introspection and maintainers read this exact docstring fragment to understand inference proxy behavior without executing it. +# What: document bytes and safe http headers to in the inference_proxy docstring; why: introspection and maintainers read this exact docstring fragment to understand inference proxy behavior without executing it. +# What: preserve the paragraph boundary in the the inference_proxy docstring; why: introspection and maintainers read this paragraph break to understand inference proxy behavior without executing it. + +# What: enable postponed evaluation of annotations; why: type hints in inference_proxy can reference runtime types without eager imports or forward-reference failures. +from __future__ import annotations + +# What: import json for request model using json; why: request_model uses json loads, making that imported dependency available to its named operation. +import json + +# What: import re for open upstream using re; why: open_upstream uses re fullmatch, making that imported dependency available to its named operation. +import re + +# What: import iterator and mapping for chunks and forward headers using typing and iterator and mapping; why: chunks and forward_headers uses the iterator annotation in chunks and the mapping annotation in forward headers, making that imported dependency available to its named operation. +from collections.abc import Iterator, Mapping + +# What: import dataclass and field for response state using dataclasses; why: UpstreamResponse needs generated initialization plus an internal non-constructor cancellation flag. +from dataclasses import dataclass +from dataclasses import field as dataclass_field + +# What: import httperror for open upstream using urllib and error and httperror; why: open_upstream uses the httperror annotation in open upstream, making that imported dependency available to its named operation. +from urllib.error import HTTPError + +# What: import request and urlopen for open upstream using urllib and request and request and urlopen; why: open_upstream uses request and urlopen, making that imported dependency available to its named operation. +from urllib.request import Request, urlopen + +# What: import request field for set fields using catalog and request field; why: _set_fields uses the request field annotation in set fields, making that imported dependency available to its named operation. +from .catalog import RequestField + + +# What: define RequestModelError as the owner of its declared state; why: daemon callers use this class boundary so those methods share one request model error state invariant. +class RequestModelError(ValueError): + """The request cannot be routed because it has no valid model identifier.""" +# What: document the request cannot be routed because in the RequestModelError docstring; why: introspection and maintainers read this exact docstring fragment to understand request model error behavior without executing it. + + +# What: compute hop by hop from connection and content length and host and keep alive and proxy authenticate; why: excluded hop by hop local auth headers later reads hop by hop, so inference_proxy must retain the computed value under that name. +_HOP_BY_HOP = {"connection", "content-length", "host", "keep-alive", "proxy-authenticate", + # What: apply the proxy authorization te trailer transfer encoding upgrade portion of hop by hop; why: inference_proxy uses this clause to evaluate hop by hop as one grouped value. + "proxy-authorization", "te", "trailer", "transfer-encoding", "upgrade"} +# What: compute local auth headers from authorization and x api key and x ft token; why: excluded hop by hop local auth headers later reads local auth headers, so inference_proxy must retain the computed value under that name. +_LOCAL_AUTH_HEADERS = {"authorization", "x-api-key", "x-ft-token"} + + +# What: define request_model around body; why: its direct callers call request_model for request model and rely on this exact input and result contract. +def request_model(body: bytes) -> str: + # What: establish the handler boundary for the protected operation; why: request_model routes failures to unicode decode error and jsondecode error and json while preserving cleanup and success flow. + try: + # What: compute doc from loads and body and json; why: model doc get model if isinstance doc later reads doc, so request_model must retain the computed value under that name. + doc = json.loads(body) + # What: handle unicode decode error and jsondecode error and json by raise request model error request body must be valid; why: request_model converts that failure into this concrete recovery, response, or cleanup behavior. + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + # What: raise RequestModelError for the caller; why: request_model stops this rejected path before it can mutate state, dispatch work, or report success. + raise RequestModelError("request body must be valid JSON with a model string") from exc + # What: compute model from isinstance and doc and dict and get and model; why: if not isinstance model str or later reads model, so request_model must retain the computed value under that name. + model = doc.get("model") if isinstance(doc, dict) else None + # What: gate on model and isinstance and str and strip before request model error; why: request_model admits request model error only for this predicate and excludes the opposite state. + if not isinstance(model, str) or not model.strip() or "\x00" in model: + # What: raise RequestModelError for the caller; why: request_model stops this rejected path before it can mutate state, dispatch work, or report success. + raise RequestModelError("request body must include a non-empty model string") + # What: return model from request_model; why: request_model exposes model so its caller can continue with the function\'s computed outcome. + return model + + +# What: define _path_parent around doc and path and create; why: its direct callers call _path_parent for path parent and rely on this exact input and result contract. +def _path_parent(doc: dict, path: tuple[str, ...], *, create: bool) -> dict | None: + # What: compute current from doc; why: child current get part later reads current, so _path_parent must retain the computed value under that name. + current = doc + # What: iterate across path to perform child and get and part and current; why: _path_parent repeats the body only while or for the loop header admits an iteration. + for part in path[:-1]: + # What: compute child from get and part and current; why: if not isinstance child dict later reads child, so _path_parent must retain the computed value under that name. + child = current.get(part) + # What: gate on isinstance and child and dict before create; why: _path_parent admits create only for this predicate and excludes the opposite state. + if not isinstance(child, dict): + # What: gate on create before the computed value; why: _path_parent admits the computed value only for this predicate and excludes the opposite state. + if not create: + # What: return no value from _path_parent; why: _path_parent returns no value to callers that depend on its completed result. + return None + # What: initialize child as an empty runtime accumulator; why: _path_parent appends or maps entries into it during current part child before consuming the aggregate. + child = {} + # What: compute current entry from child; why: current child later reads current entry, so _path_parent must retain the computed value under that name. + current[part] = child + # What: compute current from child; why: return current later reads current, so _path_parent must retain the computed value under that name. + current = child + # What: return current from _path_parent; why: _path_parent exposes current so its caller can continue with the function\'s computed outcome. + return current + + +# What: define _set_fields around doc and fields; why: its direct callers call _set_fields for set fields and rely on this exact input and result contract. +def _set_fields(doc: dict, fields: tuple[RequestField, ...]) -> None: + # What: iterate across fields to perform parent and path parent and doc and path and field; why: _set_fields repeats the body only while or for the loop header admits an iteration. + for field in fields: + # What: compute parent from path parent and doc and path and field and true; why: assert parent is not later reads parent, so _set_fields must retain the computed value under that name. + parent = _path_parent(doc, field.path, create=True) + # What: assert that parent is not group delimiter; why: _set_fields requires parent is not group delimiter to be true, so a false result stops the invalid state. + assert parent is not None + # What: compute leaf from path and field and 1; why: if field soft and leaf in parent later reads leaf, so _set_fields must retain the computed value under that name. + leaf = field.path[-1] + # What: gate on soft and field and leaf and parent before the computed value; why: _set_fields admits the computed value only for this predicate and excludes the opposite state. + if field.soft and leaf in parent: + # What: apply the continue portion of the enclosing predicate; why: this clause remains in _set_fields\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: compute parent entry from value and field; why: the enclosing return or state update later reads parent entry, so _set_fields must retain the computed value under that name. + parent[leaf] = field.value() + + +# What: define filter_request_body around body and drop fields and set fields and set fields by id and requested model and rewrite model; why: its direct callers call filter_request_body for filter request body and rely on this exact input and result contract. +def filter_request_body( + # What: declare the body input for filter_request_body; why: filter_request_body consumes body during return body, so callers must bind it with the other signature inputs. + body: bytes, + # What: declare the drop fields input for filter_request_body; why: filter_request_body consumes drop fields during for field in drop fields, so callers must bind it with the other signature inputs. + drop_fields: tuple[str, ...], + # What: declare the set fields input for filter_request_body; why: filter_request_body consumes set fields during set fields doc set fields, so callers must bind it with the other signature inputs. + set_fields: tuple[RequestField, ...] = (), + # What: declare the set fields by id input for filter_request_body; why: filter_request_body consumes set fields by id during by id dict set fields by id get requested model, so callers must bind it with the other signature inputs. + set_fields_by_id: tuple[tuple[str, tuple[RequestField, ...]], ...] = (), + # What: mark the remaining parameters as keyword-only; why: filter_request_body prevents callers from confusing adjacent lifecycle and timing arguments. + *, + # What: declare the requested model input for filter_request_body; why: filter_request_body consumes requested model during by id dict set fields by id get requested model, so callers must bind it with the other signature inputs. + requested_model: str | None = None, + # What: declare the rewrite model input for filter_request_body; why: filter_request_body consumes rewrite model during if rewrite model is not, so callers must bind it with the other signature inputs. + rewrite_model: str | None = None, +# What: complete the enclosing predicate with bytes; why: filter_request_body groups the supplied clauses as one enclosing predicate expression before its value is consumed. +) -> bytes: + """Apply safe configured JSON-field transformations in pinned order. + + The default empty policy returns the original bytes exactly. A requested + rewrite runs before drop/global/by-ID fields, matching the pinned filter + order. There is no expression or hook language, so a catalog cannot + execute code in the daemon. + """ + # What: document apply safe configured json field transformations in in the filter_request_body docstring; why: introspection and maintainers read this exact docstring fragment to understand filter request body behavior without executing it. + # What: document the default empty policy returns the in the filter_request_body docstring; why: introspection and maintainers read this exact docstring fragment to understand filter request body behavior without executing it. + # What: document rewrite runs before drop global by id in the filter_request_body docstring; why: introspection and maintainers read this exact docstring fragment to understand filter request body behavior without executing it. + # What: document order there is no expression or in the filter_request_body docstring; why: introspection and maintainers read this exact docstring fragment to understand filter request body behavior without executing it. + # What: document execute code in the daemon in the filter_request_body docstring; why: introspection and maintainers read this exact docstring fragment to understand filter request body behavior without executing it. + # What: preserve the paragraph boundary in the the filter_request_body docstring; why: introspection and maintainers read this paragraph break to understand filter request body behavior without executing it. + # What: compute by id from get and requested model and dict and set fields by id; why: if rewrite model is and not drop fields later reads by id, so filter_request_body must retain the computed value under that name. + by_id = dict(set_fields_by_id).get(requested_model, ()) + # What: gate on rewrite model and drop fields and set fields and by id before body; why: filter_request_body admits body only for this predicate and excludes the opposite state. + if rewrite_model is None and not drop_fields and not set_fields and not by_id: + # What: return body from filter_request_body; why: filter_request_body exposes body so its caller can continue with the function\'s computed outcome. + return body + # What: establish the handler boundary for the protected operation; why: filter_request_body routes failures to unicode decode error and jsondecode error and json while preserving cleanup and success flow. + try: + # What: compute doc from loads and body and json; why: if not isinstance doc dict later reads doc, so filter_request_body must retain the computed value under that name. + doc = json.loads(body) + # What: handle unicode decode error and jsondecode error and json by raise request model error request body must be valid; why: filter_request_body converts that failure into this concrete recovery, response, or cleanup behavior. + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + # What: raise RequestModelError for the caller; why: filter_request_body stops this rejected path before it can mutate state, dispatch work, or report success. + raise RequestModelError("request body must be valid JSON") from exc + # What: gate on isinstance and doc and dict before request model error; why: filter_request_body admits request model error only for this predicate and excludes the opposite state. + if not isinstance(doc, dict): + # What: raise RequestModelError for the caller; why: filter_request_body stops this rejected path before it can mutate state, dispatch work, or report success. + raise RequestModelError("request body must be a JSON object") + # What: gate on rewrite model before rewrite model and doc; why: filter_request_body admits rewrite model and doc only for this predicate and excludes the opposite state. + if rewrite_model is not None: + # What: compute doc entry from rewrite model; why: parent path parent doc path create later reads doc entry, so filter_request_body must retain the computed value under that name. + doc["model"] = rewrite_model + # What: iterate across drop fields to perform path and tuple and split and field; why: filter_request_body repeats the body only while or for the loop header admits an iteration. + for field in drop_fields: + # What: compute path from tuple and split and field and value; why: parent path parent doc path create later reads path, so filter_request_body must retain the computed value under that name. + path = tuple(field.split(".")) + # What: compute parent from path parent and doc and path and false; why: if parent is not later reads parent, so filter_request_body must retain the computed value under that name. + parent = _path_parent(doc, path, create=False) + # What: gate on parent before pop and parent and path; why: filter_request_body admits pop and parent and path only for this predicate and excludes the opposite state. + if parent is not None: + # What: call parent.pop with path and 1 and the named fixture input; why: filter_request_body invokes parent.pop while performing set fields doc set fields; the call advances that operation through its result or side effect. + parent.pop(path[-1], None) + # What: call _set_fields with doc and set fields; why: filter_request_body invokes _set_fields while performing set fields doc by id; the call advances that operation through its result or side effect. + _set_fields(doc, set_fields) + # What: call _set_fields with doc and by id; why: filter_request_body invokes _set_fields while performing return json dumps doc separators ensure ascii encode; the call advances that operation through its result or side effect. + _set_fields(doc, by_id) + # What: return encode and dumps and doc and json and utf 8 from filter_request_body; why: filter_request_body exposes encode and dumps and doc and json and utf 8 so its caller can continue with the function\'s computed outcome. + return json.dumps(doc, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +# What: define forward_headers around headers; why: its direct callers call forward_headers for forward headers and rely on this exact input and result contract. +def forward_headers(headers: Mapping[str, str]) -> dict[str, str]: + """Preserve application headers without forwarding daemon authentication. + + The router terminates its bearer/Basic/``x-api-key`` credential and optional + ``X-FT-Token`` locally. None is an engine credential, so forwarding one would + disclose a control-plane secret to the child process and its logs. + """ + # What: document preserve application headers without forwarding daemon in the forward_headers docstring; why: introspection and maintainers read this exact docstring fragment to understand forward headers behavior without executing it. + # What: document the router terminates its bearer basic in the forward_headers docstring; why: introspection and maintainers read this exact docstring fragment to understand forward headers behavior without executing it. + # What: document x ft token locally is an engine credential in the forward_headers docstring; why: introspection and maintainers read this exact docstring fragment to understand forward headers behavior without executing it. + # What: document disclose a control plane secret to the in the forward_headers docstring; why: introspection and maintainers read this exact docstring fragment to understand forward headers behavior without executing it. + # What: preserve the paragraph boundary in the the forward_headers docstring; why: introspection and maintainers read this paragraph break to understand forward headers behavior without executing it. + # What: compute excluded from hop by hop and local auth headers; why: return key value for key value later reads excluded, so forward_headers must retain the computed value under that name. + excluded = _HOP_BY_HOP | _LOCAL_AUTH_HEADERS + # What: return key and value and items and excluded from forward_headers; why: forward_headers exposes key and value and items and excluded so its caller can continue with the function\'s computed outcome. + return {key: value for key, value in headers.items() if key.lower() not in excluded} + + +# What: define response_headers around headers; why: its direct callers call response_headers for response headers and rely on this exact input and result contract. +def response_headers(headers: Mapping[str, str]) -> dict[str, str]: + """Remove only hop-by-hop fields from an engine response. + + Local router credentials are an inbound-only concern. A response may + legitimately contain an application authentication challenge or similarly + named metadata, which must retain normal upstream-header semantics. + """ + # What: document remove only hop by hop fields from an in the response_headers docstring; why: introspection and maintainers read this exact docstring fragment to understand response headers behavior without executing it. + # What: document local router credentials are an inbound only in the response_headers docstring; why: introspection and maintainers read this exact docstring fragment to understand response headers behavior without executing it. + # What: document legitimately contain an application authentication challenge in the response_headers docstring; why: introspection and maintainers read this exact docstring fragment to understand response headers behavior without executing it. + # What: document named metadata which must retain normal in the response_headers docstring; why: introspection and maintainers read this exact docstring fragment to understand response headers behavior without executing it. + # What: preserve the paragraph boundary in the the response_headers docstring; why: introspection and maintainers read this paragraph break to understand response headers behavior without executing it. + # What: return key and value and items and hop by hop from response_headers; why: response_headers exposes key and value and items and hop by hop so its caller can continue with the function\'s computed outcome. + return {key: value for key, value in headers.items() if key.lower() not in _HOP_BY_HOP} + + +# What: generate dataclass initialization and value semantics for UpstreamResponse; why: UpstreamResponse acts as a typed state record with consistent construction, comparison, and representation. +@dataclass +# What: define UpstreamResponse as the owner of chunks and close; why: daemon callers use this class boundary so those methods share one upstream response state invariant. +class UpstreamResponse: + # What: compute status from the named fixture input; why: status raw getcode later reads status, so inference_proxy must retain the computed value under that name. + status: int + # What: compute headers from the named fixture input; why: def open upstream port int path and query str later reads headers, so inference_proxy must retain the computed value under that name. + headers: dict[str, str] + # What: compute raw from the named fixture input; why: chunk self raw read size later reads raw, so inference_proxy must retain the computed value under that name. + raw: object + # What: track whether an explicit close has interrupted the response; why: a concurrent socket close may surface as a low-level read error that represents expected cancellation rather than a server failure. + _closed: bool = dataclass_field(default=False, init=False, repr=False) + + # What: define chunks around size; why: its direct callers call chunks for chunks and rely on this exact input and result contract. + def chunks(self, size: int = 64 * 1024) -> Iterator[bytes]: + # What: establish the handler boundary for the protected operation; why: UpstreamResponse.chunks routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: iterate across the computed value to perform chunk and read and size and raw; why: chunks repeats the body only while or for the loop header admits an iteration. + while True: + # What: read the next upstream block while distinguishing cancellation races; why: closing Python's HTTP response from another thread can invalidate its internal file pointer during this call. + try: + # What: retain the next raw response block; why: non-empty data must continue through the streaming iterator unchanged. + chunk = self.raw.read(size) + # What: handle close-induced low-level stream state errors; why: expected cancellation should terminate cleanly while unrelated transport defects still propagate. + except (AttributeError, ValueError): + # What: re-raise when no explicit close occurred; why: only a proven cancellation race may be converted into normal end-of-stream behavior. + if not self._closed: + # What: propagate the unexpected read failure; why: callers must retain visibility into genuine upstream corruption or implementation defects. + raise + # What: end iteration after a concurrent explicit close; why: cancellation already owns cleanup and should not emit an ASGI exception traceback. + return + # What: gate on chunk before the computed value; why: chunks admits the computed value only for this predicate and excludes the opposite state. + if not chunk: + # What: apply the break portion of the enclosing predicate; why: this clause remains in chunks\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: apply the yield chunk portion of the enclosing predicate; why: this clause remains in chunks\'s enclosing expression so its grouping and evaluation order stay intact. + yield chunk + # What: run self close on every exit path; why: chunks performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: call self.close with the declared inputs; why: chunks invokes self.close while performing the enclosing return; the call advances that operation through its result or side effect. + self.close() + + # What: define close around the current object state; why: its direct callers call close for close and rely on this exact input and result contract. + def close(self) -> None: + # What: mark the response closed before touching the raw transport; why: a blocked reader awakened by raw.close can observe cancellation before it raises from invalid internal state. + self._closed = True + # What: compute close from getattr and raw and close; why: if close is not later reads close, so close must retain the computed value under that name. + close = getattr(self.raw, "close", None) + # What: gate on close before close; why: close admits close only for this predicate and excludes the opposite state. + if close is not None: + # What: call close with the declared inputs; why: close invokes close while performing the enclosing return; the call advances that operation through its result or side effect. + close() + + +# What: define open_upstream around port and path and query and headers and body and method and timeout s and base url; why: its direct callers call open_upstream for open upstream and rely on this exact input and result contract. +def open_upstream(*, port: int, path_and_query: str, headers: Mapping[str, str], body: bytes, + # What: declare the method input for open_upstream; why: open_upstream consumes method during method method, so callers must bind it with the other signature inputs. + method: str = "POST", timeout_s: float = 900.0, + # What: declare the base url input for open_upstream; why: open_upstream consumes base url during base url base url or owned base, so callers must bind it with the other signature inputs. + base_url: str | None = None) -> UpstreamResponse: + # What: compute owned base from port and http; why: base url base url or owned base later reads owned base, so open_upstream must retain the computed value under that name. + owned_base = f"http://127.0.0.1:{port}" + # What: compute base url from base url and owned base; why: rf http port a za z0 9 base url later reads base url, so open_upstream must retain the computed value under that name. + base_url = base_url or owned_base + # What: gate on any and fullmatch and base url and re and segment before value error; why: open_upstream admits value error only for this predicate and excludes the opposite state. + if ( + # What: call re.fullmatch with port and http and a za z0 9 and value and base url; why: open_upstream invokes re.fullmatch while performing rf http port a za z0 9 base url; the call advances that operation through its result or side effect. + re.fullmatch( + # What: apply the rf http port a za z0 9 base url portion of the enclosing predicate; why: this clause remains in open_upstream\'s enclosing expression so its grouping and evaluation order stay intact. + rf"http://127\.0\.0\.1:{port}(?:/[A-Za-z0-9._~-]+)*", base_url + # What: complete the re.fullmatch call with port and base url; why: open_upstream groups the supplied clauses as one re.fullmatch call before its value is consumed. + ) + # What: apply the is portion of the enclosing predicate; why: this clause remains in open_upstream\'s enclosing expression so its grouping and evaluation order stay intact. + is None + # What: call any with segment and split and base url and value and value; why: open_upstream consumes the any return value while evaluating or any(segment in {".", ".."} for segment in base_url.split("/")). + or any(segment in {".", ".."} for segment in base_url.split("/")) + # What: complete the enclosing predicate with if re fullmatch f http 127 0 0 1 port; why: open_upstream groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise ValueError for the caller; why: open_upstream stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("upstream base URL must target the manager-owned loopback port") + # What: gate on startswith and path and query before value error; why: open_upstream admits value error only for this predicate and excludes the opposite state. + if not path_and_query.startswith("/"): + # What: raise ValueError for the caller; why: open_upstream stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("upstream path must be absolute") + # What: compute request from request and body and method and path and query; why: raw urlopen request timeout timeout s later reads request, so open_upstream must retain the computed value under that name. + request = Request( + # What: call base_url.rstrip with value; why: open_upstream invokes base_url.rstrip while performing data body; the call advances that operation through its result or side effect. + f"{base_url.rstrip('/')}{path_and_query}", + # What: supply data to Request; why: open_upstream binds this body value to Request's data input. + data=body, + # What: supply headers to forward_headers; why: open_upstream binds this forward headers and headers value to forward_headers's headers input. + headers=forward_headers(headers), + # What: supply method to Request; why: open_upstream binds this method value to Request's method input. + method=method, + # What: complete the Request call with data and headers and method; why: open_upstream groups the supplied clauses as one Request call before its value is consumed. + ) + # What: establish the handler boundary for the protected operation; why: open_upstream routes failures to httperror while preserving cleanup and success flow. + try: + # What: compute raw from urlopen and request and timeout s; why: raw exc later reads raw, so open_upstream must retain the computed value under that name. + raw = urlopen(request, timeout=timeout_s) + # What: handle httperror by raw exc; why: open_upstream converts that failure into this concrete recovery, response, or cleanup behavior. + except HTTPError as exc: + # What: compute raw from exc; why: status raw getcode later reads raw, so open_upstream must retain the computed value under that name. + raw = exc + # What: return upstream response and raw and getcode and response headers from open_upstream; why: open_upstream exposes upstream response and raw and getcode and response headers so its caller can continue with the function\'s computed outcome. + return UpstreamResponse( + # What: supply status to raw.getcode; why: open_upstream binds this getcode and raw value to raw.getcode's status input. + status=raw.getcode(), + # What: supply headers to response_headers; why: open_upstream binds this response headers and dict and items and headers value to response_headers's headers input. + headers=response_headers(dict(raw.headers.items())), + # What: supply raw to UpstreamResponse; why: open_upstream binds this raw value to UpstreamResponse's raw input. + raw=raw, + # What: complete the UpstreamResponse call with status and headers and raw; why: open_upstream groups the supplied clauses as one UpstreamResponse call before its value is consumed. + ) diff --git a/python/freetoken/daemon/metrics.py b/python/freetoken/daemon/metrics.py index a2a42d03bd..05211efcda 100644 --- a/python/freetoken/daemon/metrics.py +++ b/python/freetoken/daemon/metrics.py @@ -1,28 +1,69 @@ -"""The engine's OWN footprint. Boundary: only the serve tree's RAM/VRAM — system-wide host -telemetry is not this daemon's job. +"""The engine's own process-tree footprint, never system-wide host telemetry. -RAM = summed PSS across the serve process group (shared pages counted once, the honest number). -VRAM = per-process GPU memory for those pids, via ``pynvml`` if importable (optional), else -parsed from ``nvidia-smi``, else 0. All best-effort and off the event loop — a missing GPU or -absent NVML returns 0, never an error.""" +RAM is summed Linux PSS. VRAM is per-process GPU memory from NVML/``nvidia-smi`` or +``amd-smi``. Byte fields remain integers for API compatibility; availability fields prevent an +unavailable best-effort probe from being misrepresented as a measured zero. +""" +# What: document the engine s own process tree footprint in the metrics docstring; why: introspection and maintainers read this exact docstring fragment to understand metrics behavior without executing it. +# What: document ram is summed linux pss vram in the metrics docstring; why: introspection and maintainers read this exact docstring fragment to understand metrics behavior without executing it. +# What: document amd smi byte fields remain integers for in the metrics docstring; why: introspection and maintainers read this exact docstring fragment to understand metrics behavior without executing it. +# What: document unavailable best effort probe from being misrepresented in the metrics docstring; why: introspection and maintainers read this exact docstring fragment to understand metrics behavior without executing it. +# What: preserve the paragraph boundary in the the metrics docstring; why: introspection and maintainers read this paragraph break to understand metrics behavior without executing it. from __future__ import annotations +# What: import json for amd smi process vram using json; why: _amd_smi_process_vram uses json loads, making that imported dependency available to its named operation. +import json import subprocess import threading import time -from typing import Callable +from collections.abc import Callable + +# What: import Path for process fdinfo discovery; why: Linux DRM exposes privacy-bounded per-process GPU memory through files beneath each owned PID. +from pathlib import Path from . import osproc def engine_footprint(pid: int | None) -> dict: if pid is None: - return {"ramBytes": 0, "vramBytes": 0, "pids": []} + # What: return ram bytes and vram bytes and pids and ram available and vram available from engine_footprint; why: engine_footprint exposes ram bytes and vram bytes and pids and ram available and vram available so its caller can continue with the function\'s computed outcome. + return { + # What: map the ram bytes field as 0; why: engine_footprint carries ram bytes into "ramBytes": 0, "vramBytes": 0, "pids": []. + "ramBytes": 0, "vramBytes": 0, "pids": [], + # What: map the ram available field as false; why: engine_footprint carries ram available into "ramAvailable": False, "vramAvailable": False. + "ramAvailable": False, "vramAvailable": False, + # What: map the ram source field as the fixture input; why: engine_footprint carries ram source into "ramSource": None, "vramSource": None. + "ramSource": None, "vramSource": None, + # What: complete the enclosing predicate mapping with ram bytes and vram bytes and pids and ram available and vram available; why: engine_footprint groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } pids = osproc.tree_pids(pid) - ram = sum(osproc.read_pss_bytes(p) for p in pids) - vram = vram_bytes_for_pids(pids) - return {"ramBytes": ram, "vramBytes": vram, "pids": pids} + # What: compute ram parts from read pss bytes if available and p and pids and osproc; why: ram available bool ram parts and all value later reads ram parts, so engine_footprint must retain the computed value under that name. + ram_parts = [osproc.read_pss_bytes_if_available(p) for p in pids] + # What: compute ram available from bool and ram parts and all and value; why: ram available ram available later reads ram available, so engine_footprint must retain the computed value under that name. + ram_available = bool(ram_parts) and all(value is not None for value in ram_parts) + # What: compute ram from sum and value and ram parts and 0; why: ram bytes ram later reads ram, so engine_footprint must retain the computed value under that name. + ram = sum(value or 0 for value in ram_parts) + # What: compute vram and vram available and vram source from vram measurement for pids and pids; why: vram bytes vram later reads vram and vram available and vram source, so engine_footprint must retain the computed value under that name. + vram, vram_available, vram_source = _vram_measurement_for_pids(pids) + # What: return ram and vram and pids and ram available from engine_footprint; why: engine_footprint exposes ram and vram and pids and ram available so its caller can continue with the function\'s computed outcome. + return { + # What: map the ram bytes field as ram; why: engine_footprint carries ram bytes into "ramBytes": ram. + "ramBytes": ram, + # What: map the vram bytes field as vram; why: engine_footprint carries vram bytes into "vramBytes": vram. + "vramBytes": vram, + # What: map the pids field as pids; why: engine_footprint carries pids into "pids": pids. + "pids": pids, + # What: map the ram available field as ram available; why: engine_footprint carries ram available into "ramAvailable": ram_available. + "ramAvailable": ram_available, + # What: map the vram available field as vram available; why: engine_footprint carries vram available into "vramAvailable": vram_available. + "vramAvailable": vram_available, + # What: map the ram source field as ram available and proc smaps rollup pss; why: engine_footprint carries ram source into "ramSource": "proc-smaps-rollup-pss" if ram_available else None. + "ramSource": "proc-smaps-rollup-pss" if ram_available else None, + # What: map the vram source field as vram source; why: engine_footprint carries vram source into "vramSource": vram_source. + "vramSource": vram_source, + # What: complete the enclosing predicate mapping with ram bytes and vram bytes and pids and ram available and vram available; why: engine_footprint groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } class FootprintCache: @@ -48,15 +89,121 @@ def get(self, pid: int | None) -> dict: def vram_bytes_for_pids(pids: list[int]) -> int: + # What: return vram measurement for pids and pids and 0 from vram_bytes_for_pids; why: vram_bytes_for_pids exposes vram measurement for pids and pids and 0 so its caller can continue with the function\'s computed outcome. + return _vram_measurement_for_pids(pids)[0] + + +# What: define _vram_measurement_for_pids around pids; why: its direct callers call _vram_measurement_for_pids for vram measurement for pids and rely on this exact input and result contract. +def _vram_measurement_for_pids(pids: list[int]) -> tuple[int, bool, str | None]: want = set(pids) if not want: - return 0 - usage = _nvml_process_vram() - if usage is None: - usage = _smi_process_vram() - if not usage: - return 0 - return sum(nbytes for p, nbytes in usage.items() if p in want) + # What: return 0 and false from _vram_measurement_for_pids; why: _vram_measurement_for_pids exposes 0 and false so its caller can continue with the function\'s computed outcome. + return 0, False, None + # What: read Linux DRM fdinfo for the exact owned process tree; why: AMD APUs expose per-process VRAM and GTT there even when amd-smi omits HIP clients. + drm_usage = _drm_fdinfo_process_vram(sorted(want)) + # What: accept a supported DRM probe even when measured use is zero; why: an empty measured result is distinct from unavailable telemetry. + if drm_usage is not None: + # What: sum only requested process entries and report the precise source; why: the router must not leak or count unrelated host workloads. + return sum(drm_usage.get(pid, 0) for pid in want), True, "drm-fdinfo-vram-gtt" + # What: compute available source from the named fixture input; why: available source available source or source later reads available source, so _vram_measurement_for_pids must retain the computed value under that name. + available_source = None + # What: iterate across nvml process vram and smi process vram and amd smi process vram to perform usage and probe; why: _vram_measurement_for_pids repeats the body only while or for the loop header admits an iteration. + for source, probe in ( + # What: apply the nvml nvml process vram portion of the enclosing predicate; why: this clause remains in _vram_measurement_for_pids\'s enclosing expression so its grouping and evaluation order stay intact. + ("nvml", _nvml_process_vram), + # What: apply the nvidia smi smi process vram portion of the enclosing predicate; why: this clause remains in _vram_measurement_for_pids\'s enclosing expression so its grouping and evaluation order stay intact. + ("nvidia-smi", _smi_process_vram), + # What: apply the amd smi amd smi process vram portion of the enclosing predicate; why: this clause remains in _vram_measurement_for_pids\'s enclosing expression so its grouping and evaluation order stay intact. + ("amd-smi", _amd_smi_process_vram), + # What: complete the enclosing predicate collection with nvml process vram and nvml and smi process vram and nvidia smi and amd smi process vram and amd smi; why: _vram_measurement_for_pids groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. + ): + # What: compute usage from probe; why: if usage is not later reads usage, so _vram_measurement_for_pids must retain the computed value under that name. + usage = probe() + # What: gate on usage before any and source and pid and usage and want; why: _vram_measurement_for_pids admits any and source and pid and usage and want only for this predicate and excludes the opposite state. + if usage is not None: + # What: gate on any and pid and usage and want before source and sum and nbytes and p and items; why: _vram_measurement_for_pids admits source and sum and nbytes and p and items only for this predicate and excludes the opposite state. + if any(pid in usage for pid in want): + # What: return source and sum and nbytes and p from _vram_measurement_for_pids; why: _vram_measurement_for_pids exposes source and sum and nbytes and p so its caller can continue with the function\'s computed outcome. + return sum(nbytes for p, nbytes in usage.items() if p in want), True, source + # What: compute available source from available source and source; why: if available source is not later reads available source, so _vram_measurement_for_pids must retain the computed value under that name. + available_source = available_source or source + # What: gate on available source before available source; why: _vram_measurement_for_pids admits available source only for this predicate and excludes the opposite state. + if available_source is not None: + # What: return available source and 0 and true from _vram_measurement_for_pids; why: _vram_measurement_for_pids exposes available source and 0 and true so its caller can continue with the function\'s computed outcome. + return 0, True, available_source + # What: return 0 and false from _vram_measurement_for_pids; why: _vram_measurement_for_pids exposes 0 and false so its caller can continue with the function\'s computed outcome. + return 0, False, None + + +# What: define the Linux DRM fdinfo process-memory probe; why: the target AMD host needs owned-process GPU evidence without requiring privileged or vendor-specific tooling. +def _drm_fdinfo_process_vram(pids: list[int], proc_root: Path = Path("/proc")) -> dict[int, int] | None: + """Return each requested PID's deduplicated DRM VRAM plus GTT bytes when available.""" + # What: initialize measured usage by PID; why: callers need a process-scoped mapping compatible with the existing VRAM probe contract. + usage: dict[int, int] = {} + # What: track whether any valid DRM memory record was readable; why: a measured zero must remain distinguishable from an unavailable probe. + available = False + # What: inspect only the already-owned process IDs; why: telemetry must not enumerate or expose unrelated host processes. + for pid in pids: + # What: initialize per-process DRM client identities; why: one DRM client may appear on several file descriptors and must not be double-counted. + seen_clients: set[tuple[str, str]] = set() + # What: resolve the PID's fdinfo directory; why: Linux publishes per-file-descriptor DRM accounting at this stable procfs location. + fdinfo_dir = proc_root / str(pid) / "fdinfo" + # What: establish a race-safe directory read boundary; why: owned worker processes or descriptors may exit while a periodic sample runs. + try: + # What: snapshot current fdinfo entries; why: each readable descriptor may carry one DRM client accounting record. + entries = list(fdinfo_dir.iterdir()) + # What: treat an unreadable or vanished process as unavailable for this PID; why: monitoring must never break routing during lifecycle changes. + except OSError: + # What: continue to the remaining owned PIDs; why: sibling workers can still provide valid process-tree evidence. + continue + # What: inspect each owned descriptor record; why: the DRM driver attaches memory accounting to fdinfo rather than the process directory itself. + for entry in entries: + # What: establish a race-safe descriptor read boundary; why: descriptors may close between directory enumeration and content access. + try: + # What: read the small procfs record with replacement decoding; why: malformed text must not crash the router's telemetry thread. + lines = entry.read_text(encoding="utf-8", errors="replace").splitlines() + # What: ignore descriptors that vanish or deny access; why: periodic monitoring must tolerate normal process activity. + except OSError: + # What: continue to the next descriptor; why: one transient fd does not invalidate other owned DRM records. + continue + # What: initialize normalized fdinfo fields; why: parsing only named keys avoids depending on line order or unrelated driver counters. + fields: dict[str, str] = {} + # What: parse each colon-delimited fdinfo line; why: DRM accounting is represented as named textual scalars. + for line in lines: + # What: skip lines without a field delimiter; why: unrelated or malformed records cannot contribute trustworthy memory values. + if ":" not in line: + # What: continue parsing the remaining lines; why: valid DRM fields may still be present later in the record. + continue + # What: split once into a normalized key and value; why: units and future values may contain additional punctuation. + key, value = line.split(":", 1) + # What: retain the stripped field pair; why: procfs aligns values with tabs and spaces that are not part of the measurement. + fields[key.strip()] = value.strip() + # What: identify memory-bearing DRM records; why: ordinary descriptors must not be mistaken for measured zero GPU clients. + memory_keys = tuple(key for key in ("drm-memory-vram", "drm-memory-gtt") if key in fields) + # What: skip descriptors without GPU memory fields; why: availability requires an actual DRM memory accounting record. + if not memory_keys: + # What: continue to the next descriptor; why: another fd may own the process's DRM client. + continue + # What: mark the probe available after a valid DRM memory record; why: zero-valued records are still authoritative measurements. + available = True + # What: derive a stable per-process client identity; why: duplicate descriptors for one DRM client report identical totals. + client = (fields.get("drm-client-id", entry.name), fields.get("drm-pdev", "")) + # What: skip a client already counted for this PID; why: summing duplicate fdinfo records would inflate GPU memory evidence. + if client in seen_clients: + # What: continue to the next descriptor; why: distinct clients still need to be included. + continue + # What: record the client before summing; why: every DRM client contributes at most once per process sample. + seen_clients.add(client) + # What: parse the available VRAM and GTT fields into bytes; why: AMD APUs commonly hold model allocations in GTT while discrete allocations use VRAM. + values = [_memory_bytes(fields[key]) for key in memory_keys] + # What: reject a malformed client record; why: partial or unknown units must not become false precision. + if any(value is None for value in values): + # What: continue without counting the malformed record; why: other valid clients can still produce bounded evidence. + continue + # What: add the deduplicated client total to its owned PID; why: the public metric contract reports aggregate process-tree GPU memory bytes. + usage[pid] = usage.get(pid, 0) + sum(int(value) for value in values if value is not None) + # What: return the mapping only when DRM accounting was observed; why: None preserves the existing fallback chain to NVML and vendor SMI tools. + return usage if available else None # NVML is initialized ONCE and held for the daemon's life — nvmlInit()+nvmlShutdown() on every @@ -83,6 +230,8 @@ def _nvml_process_vram() -> dict[int, int] | None: if not pynvml: return None out: dict[int, int] = {} + # What: compute queried from false; why: queried later reads queried, so _nvml_process_vram must retain the computed value under that name. + queried = False try: count = pynvml.nvmlDeviceGetCount() for i in range(count): @@ -98,18 +247,21 @@ def _nvml_process_vram() -> dict[int, int] | None: used = getattr(proc, "usedGpuMemory", None) if used: # None == "not available", per NVML out[int(proc.pid)] = out.get(int(proc.pid), 0) + int(used) + # What: compute queried from true; why: return out if queried else later reads queried, so _nvml_process_vram must retain the computed value under that name. + queried = True break - except Exception: # noqa: BLE001 + except Exception: # noqa: BLE001, S112 -- another NVML ABI getter may still succeed. continue except Exception: # noqa: BLE001 return out or None - # Empty → NVML enumeration gave nothing usable (e.g. every process getter raised on a - # driver/MIG mismatch); signal that with None so the nvidia-smi fallback still runs, matching - # the error path above. - return out or None + # A successfully queried empty process list is a real zero. If every getter failed, + # ``queried`` stays false and the command-line fallbacks still run. + # What: return queried and out from _nvml_process_vram; why: _nvml_process_vram exposes queried and out so its caller can continue with the function\'s computed outcome. + return out if queried else None -def _smi_process_vram() -> dict[int, int]: +# What: define _smi_process_vram around the current object state; why: its direct callers call _smi_process_vram for smi process vram and rely on this exact input and result contract. +def _smi_process_vram() -> dict[int, int] | None: try: out = subprocess.run( [ @@ -120,15 +272,169 @@ def _smi_process_vram() -> dict[int, int]: capture_output=True, text=True, timeout=3.0, + # What: keep a nonzero command status as inspectable data; why: missing telemetry is handled without raising or hiding daemon health. + check=False, ) except (OSError, subprocess.SubprocessError): - return {} + # What: return no value from _smi_process_vram; why: _smi_process_vram returns no value to callers that depend on its completed result. + return None if out.returncode != 0: - return {} + # What: return no value from _smi_process_vram; why: _smi_process_vram returns no value to callers that depend on its completed result. + return None usage: dict[int, int] = {} + # What: compute malformed from false; why: malformed later reads malformed, so _smi_process_vram must retain the computed value under that name. + malformed = False for line in out.stdout.splitlines(): parts = [p.strip() for p in line.split(",")] if len(parts) != 2 or not parts[0].isdigit() or not parts[1].isdigit(): + # What: gate on strip and line before malformed; why: _smi_process_vram admits malformed only for this predicate and excludes the opposite state. + if line.strip(): + # What: compute malformed from true; why: return if malformed else usage later reads malformed, so _smi_process_vram must retain the computed value under that name. + malformed = True continue usage[int(parts[0])] = usage.get(int(parts[0]), 0) + int(parts[1]) * 1024 * 1024 # MiB - return usage + # What: return malformed and usage from _smi_process_vram; why: _smi_process_vram exposes malformed and usage so its caller can continue with the function\'s computed outcome. + return None if malformed else usage + + +# What: define _memory_bytes around value; why: its direct callers call _memory_bytes for memory bytes and rely on this exact input and result contract. +def _memory_bytes(value) -> int | None: + """Parse AMD SMI's version-dependent JSON scalar or ``{value, unit}`` form.""" + # What: document parse amd smi s version dependent json in the _memory_bytes docstring; why: introspection and maintainers read this exact docstring fragment to understand memory bytes behavior without executing it. + # What: gate on isinstance and value and dict before unit and get and value; why: _memory_bytes admits unit and get and value only for this predicate and excludes the opposite state. + if isinstance(value, dict) and "value" in value: + # What: compute unit from get and value and unit and b; why: value unit parts parts if len later reads unit, so _memory_bytes must retain the computed value under that name. + unit = value.get("unit", "B") + # What: compute value from value and value; why: elif isinstance value str later reads value, so _memory_bytes must retain the computed value under that name. + value = value["value"] + # What: gate on isinstance and value and str before parts and split and strip and value; why: _memory_bytes admits parts and split and strip and value only for this predicate and excludes the opposite state. + elif isinstance(value, str): + # What: compute parts from split and strip and value; why: if not parts later reads parts, so _memory_bytes must retain the computed value under that name. + parts = value.strip().split() + # What: gate on parts before the computed value; why: _memory_bytes admits the computed value only for this predicate and excludes the opposite state. + if not parts: + # What: return no value from _memory_bytes; why: _memory_bytes returns no value to callers that depend on its completed result. + return None + # What: compute value and unit from parts and len and 0 and b and 1; why: if isinstance value bool later reads value and unit, so _memory_bytes must retain the computed value under that name. + value, unit = parts[0], parts[1] if len(parts) > 1 else "B" + # What: select the remaining branch that performs unit b; why: _memory_bytes covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: compute unit from b; why: scale scales get str unit strip lower later reads unit, so _memory_bytes must retain the computed value under that name. + unit = "B" + # What: gate on isinstance and value and bool before the computed value; why: _memory_bytes admits the computed value only for this predicate and excludes the opposite state. + if isinstance(value, bool): + # What: return no value from _memory_bytes; why: _memory_bytes returns no value to callers that depend on its completed result. + return None + # What: establish the handler boundary for the protected operation; why: _memory_bytes routes failures to type error and value error while preserving cleanup and success flow. + try: + # What: compute amount from float and value; why: if scale is or amount later reads amount, so _memory_bytes must retain the computed value under that name. + amount = float(value) + # What: handle type error and value error by return; why: _memory_bytes converts that failure into this concrete recovery, response, or cleanup behavior. + except (TypeError, ValueError): + # What: return no value from _memory_bytes; why: _memory_bytes returns no value to callers that depend on its completed result. + return None + # What: compute scales from b and kb and mb and gb and tb; why: scale scales get str unit strip lower later reads scales, so _memory_bytes must retain the computed value under that name. + scales = { + # What: map the b field as 1; why: _memory_bytes carries b through scales into scale scales get str unit strip lower. + "b": 1, "kb": 1000, "mb": 1000**2, "gb": 1000**3, "tb": 1000**4, + # What: map the kib field as 1024; why: _memory_bytes carries kib through scales into scale scales get str unit strip lower. + "kib": 1024, "mib": 1024**2, "gib": 1024**3, "tib": 1024**4, + # What: complete the scales mapping with b and kb and mb and gb and tb; why: _memory_bytes groups the supplied clauses as one scales mapping before its value is consumed. + } + # What: compute scale from get and scales and lower and strip; why: if scale is or amount later reads scale, so _memory_bytes must retain the computed value under that name. + scale = scales.get(str(unit).strip().lower()) + # What: gate on scale and amount before the computed value; why: _memory_bytes admits the computed value only for this predicate and excludes the opposite state. + if scale is None or amount < 0: + # What: return no value from _memory_bytes; why: _memory_bytes returns no value to callers that depend on its completed result. + return None + # What: return int and amount and scale from _memory_bytes; why: _memory_bytes exposes int and amount and scale so its caller can continue with the function\'s computed outcome. + return int(amount * scale) + + +# What: define _amd_smi_process_vram around the current object state; why: its direct callers call _amd_smi_process_vram for amd smi process vram and rely on this exact input and result contract. +def _amd_smi_process_vram() -> dict[int, int] | None: + """Read process VRAM from the documented ``amd-smi process --json`` schema.""" + # What: document read process vram from the documented in the _amd_smi_process_vram docstring; why: introspection and maintainers read this exact docstring fragment to understand amd smi process vram behavior without executing it. + # What: establish the handler boundary for the protected operation; why: _amd_smi_process_vram routes failures to oserror and subprocess error and subprocess while preserving cleanup and success flow. + try: + # What: compute out from run and subprocess and amd smi and process and json; why: if out returncode later reads out, so _amd_smi_process_vram must retain the computed value under that name. + out = subprocess.run( + # What: apply the amd smi process json general portion of out; why: _amd_smi_process_vram uses this clause to evaluate out as one grouped value. + ["amd-smi", "process", "--json", "--general"], + # What: supply capture output to subprocess.run; why: _amd_smi_process_vram binds this true value to subprocess.run's capture output input. + capture_output=True, + # What: supply text to subprocess.run; why: _amd_smi_process_vram binds this true value to subprocess.run's text input. + text=True, + # What: supply timeout to subprocess.run; why: _amd_smi_process_vram binds this 3 0 value to subprocess.run's timeout input. + timeout=3.0, + # What: keep a nonzero command status as inspectable data; why: missing AMD telemetry is handled without raising or hiding daemon health. + check=False, + # What: complete the subprocess.run call with capture output and text and timeout; why: _amd_smi_process_vram groups the supplied clauses as one subprocess.run call before its value is consumed. + ) + # What: handle oserror and subprocess error and subprocess by return; why: _amd_smi_process_vram converts that failure into this concrete recovery, response, or cleanup behavior. + except (OSError, subprocess.SubprocessError): + # What: return no value from _amd_smi_process_vram; why: _amd_smi_process_vram returns no value to callers that depend on its completed result. + return None + # What: gate on returncode and out before the computed value; why: _amd_smi_process_vram admits the computed value only for this predicate and excludes the opposite state. + if out.returncode != 0: + # What: return no value from _amd_smi_process_vram; why: _amd_smi_process_vram returns no value to callers that depend on its completed result. + return None + # What: establish the handler boundary for the protected operation; why: _amd_smi_process_vram routes failures to jsondecode error and type error and json while preserving cleanup and success flow. + try: + # What: compute doc from loads and stdout and json and out; why: visit doc later reads doc, so _amd_smi_process_vram must retain the computed value under that name. + doc = json.loads(out.stdout) + # What: handle jsondecode error and type error and json by return; why: _amd_smi_process_vram converts that failure into this concrete recovery, response, or cleanup behavior. + except (json.JSONDecodeError, TypeError): + # What: return no value from _amd_smi_process_vram; why: _amd_smi_process_vram returns no value to callers that depend on its completed result. + return None + + # What: initialize usage as an empty runtime accumulator; why: _amd_smi_process_vram appends or maps entries into it during usage pid usage get pid 0 vram before consuming the aggregate. + usage: dict[int, int] = {} + # What: compute saw process from false; why: nonlocal saw process saw vram later reads saw process, so _amd_smi_process_vram must retain the computed value under that name. + saw_process = False + # What: compute saw vram from false; why: nonlocal saw process saw vram later reads saw vram, so _amd_smi_process_vram must retain the computed value under that name. + saw_vram = False + + # What: define visit around node; why: its direct callers call visit for visit and rely on this exact input and result contract. + def visit(node) -> None: + # What: apply the nonlocal saw process saw vram portion of the enclosing predicate; why: this clause remains in visit\'s enclosing expression so its grouping and evaluation order stay intact. + nonlocal saw_process, saw_vram + # What: gate on isinstance and node and dict before fields and value and lower and key and items; why: visit admits fields and value and lower and key and items only for this predicate and excludes the opposite state. + if isinstance(node, dict): + # What: compute fields from value and lower and key and items; why: pid fields get pid later reads fields, so visit must retain the computed value under that name. + fields = {str(key).lower(): value for key, value in node.items()} + # What: compute pid from get and fields and pid; why: if isinstance pid int and not later reads pid, so visit must retain the computed value under that name. + pid = fields.get("pid") + # What: compute memory from get and fields and memory usage; why: if isinstance memory dict later reads memory, so visit must retain the computed value under that name. + memory = fields.get("memory_usage") + # What: gate on isinstance and pid and int and bool before saw process; why: visit admits saw process only for this predicate and excludes the opposite state. + if isinstance(pid, int) and not isinstance(pid, bool): + # What: compute saw process from true; why: the enclosing return or state update later reads saw process, so visit must retain the computed value under that name. + saw_process = True + # What: gate on isinstance and memory and dict before memory fields and value and lower and key and items; why: visit admits memory fields and value and lower and key and items only for this predicate and excludes the opposite state. + if isinstance(memory, dict): + # What: compute memory fields from value and lower and key and items; why: vram memory bytes memory fields get vram mem later reads memory fields, so visit must retain the computed value under that name. + memory_fields = {str(key).lower(): value for key, value in memory.items()} + # What: compute vram from memory bytes and get and memory fields and vram mem; why: if vram is not later reads vram, so visit must retain the computed value under that name. + vram = _memory_bytes(memory_fields.get("vram_mem")) + # What: gate on vram before saw vram; why: visit admits saw vram only for this predicate and excludes the opposite state. + if vram is not None: + # What: compute saw vram from true; why: the enclosing return or state update later reads saw vram, so visit must retain the computed value under that name. + saw_vram = True + # What: compute usage entry from vram and get and pid and usage and 0; why: the enclosing return or state update later reads usage entry, so visit must retain the computed value under that name. + usage[pid] = usage.get(pid, 0) + vram + # What: iterate across values and node to perform visit and child; why: visit repeats the body only while or for the loop header admits an iteration. + for child in node.values(): + # What: call visit with child; why: visit invokes visit while performing elif isinstance node list; the call advances that operation through its result or side effect. + visit(child) + # What: gate on isinstance and node and list before child and node and visit; why: visit admits child and node and visit only for this predicate and excludes the opposite state. + elif isinstance(node, list): + # What: iterate across node to perform visit and child; why: visit repeats the body only while or for the loop header admits an iteration. + for child in node: + # What: call visit with child; why: visit invokes visit while performing the enclosing return; the call advances that operation through its result or side effect. + visit(child) + + # What: call visit with doc; why: _amd_smi_process_vram invokes visit while performing return if saw process and not saw vram; the call advances that operation through its result or side effect. + visit(doc) + # What: return usage and saw process and saw vram from _amd_smi_process_vram; why: _amd_smi_process_vram exposes usage and saw process and saw vram so its caller can continue with the function\'s computed outcome. + return None if saw_process and not saw_vram else usage diff --git a/python/freetoken/daemon/osproc.py b/python/freetoken/daemon/osproc.py index acf6447efa..f04a321873 100644 --- a/python/freetoken/daemon/osproc.py +++ b/python/freetoken/daemon/osproc.py @@ -156,6 +156,39 @@ def read_pss_bytes(pid: int) -> int: return 0 +# What: define read_pss_bytes_if_available around pid; why: its direct callers call read_pss_bytes_if_available for read pss bytes if available and rely on this exact input and result contract. +def read_pss_bytes_if_available(pid: int) -> int | None: + """PSS in bytes, or ``None`` when this host/process cannot provide it. + + Unlike :func:`read_pss_bytes`, this preserves the distinction between an + actual zero and an unavailable ``/proc`` measurement for observability + callers that must not present a safe default as measured data. + """ + # What: document pss in bytes or when this in the read_pss_bytes_if_available docstring; why: introspection and maintainers read this exact docstring fragment to understand read pss bytes if available behavior without executing it. + # What: document unlike func read pss bytes this preserves the in the read_pss_bytes_if_available docstring; why: introspection and maintainers read this exact docstring fragment to understand read pss bytes if available behavior without executing it. + # What: document actual zero and an unavailable proc in the read_pss_bytes_if_available docstring; why: introspection and maintainers read this exact docstring fragment to understand read pss bytes if available behavior without executing it. + # What: document callers that must not present a in the read_pss_bytes_if_available docstring; why: introspection and maintainers read this exact docstring fragment to understand read pss bytes if available behavior without executing it. + # What: preserve the paragraph boundary in the the read_pss_bytes_if_available docstring; why: introspection and maintainers read this paragraph break to understand read pss bytes if available behavior without executing it. + # What: compute raw from read proc and pid and smaps rollup; why: if not raw later reads raw, so read_pss_bytes_if_available must retain the computed value under that name. + raw = _read_proc(pid, "smaps_rollup") + # What: gate on raw before the computed value; why: read_pss_bytes_if_available admits the computed value only for this predicate and excludes the opposite state. + if not raw: + # What: return no value from read_pss_bytes_if_available; why: read_pss_bytes_if_available returns no value to callers that depend on its completed result. + return None + # What: iterate across splitlines and raw to perform startswith and parts and line and split and isdigit; why: read_pss_bytes_if_available repeats the body only while or for the loop header admits an iteration. + for line in raw.splitlines(): + # What: gate on startswith and line before parts and split and line; why: read_pss_bytes_if_available admits parts and split and line only for this predicate and excludes the opposite state. + if line.startswith("Pss:"): + # What: compute parts from split and line; why: if len parts and parts isdigit later reads parts, so read_pss_bytes_if_available must retain the computed value under that name. + parts = line.split() + # What: gate on isdigit and len and parts before int and parts; why: read_pss_bytes_if_available admits int and parts only for this predicate and excludes the opposite state. + if len(parts) >= 2 and parts[1].isdigit(): + # What: return int and parts and 1024 and 1 from read_pss_bytes_if_available; why: read_pss_bytes_if_available exposes int and parts and 1024 and 1 so its caller can continue with the function\'s computed outcome. + return int(parts[1]) * 1024 + # What: return no value from read_pss_bytes_if_available; why: read_pss_bytes_if_available returns no value to callers that depend on its completed result. + return None + + def is_ft_serve_on_port(pid: int, port: int, *, starttime: int | None = None) -> bool: """Verify ``pid`` is (still) an ``ft serve`` bound to ``port`` — the re-adoption / liveness identity check. Requires: alive, unchanged start time (PID-reuse diff --git a/python/freetoken/daemon/performance.py b/python/freetoken/daemon/performance.py new file mode 100644 index 0000000000..db4efa1b04 --- /dev/null +++ b/python/freetoken/daemon/performance.py @@ -0,0 +1,223 @@ +"""Bounded periodic history for the owned engine process tree only.""" +# What: document bounded periodic history for the owned in the performance docstring; why: introspection and maintainers read this exact docstring fragment to understand performance behavior without executing it. + +# What: enable postponed evaluation of annotations; why: type hints in performance can reference runtime types without eager imports or forward-reference failures. +from __future__ import annotations + +# What: import deque for init using collections and deque; why: __init__ uses deque, making that imported dependency available to its named operation. +from collections import deque +# What: import datetime and timezone for current and sample once using datetime and datetime and timezone; why: current and sample_once uses the datetime annotation in current and timezone utc, making that imported dependency available to its named operation. +from datetime import datetime, timezone +# What: import threading for init using threading; why: __init__ uses threading lock, making that imported dependency available to its named operation. +import threading +# What: import time for init using time; why: __init__ uses time time, making that imported dependency available to its named operation. +import time +# What: import callable for init using typing and callable; why: __init__ uses the callable annotation in init, making that imported dependency available to its named operation. +from typing import Callable + + +# What: define PerformanceMonitor as the owner of __init__ and start and stop and reconfigure and sample_once; why: daemon callers use this class boundary so those methods share one performance monitor state invariant. +class PerformanceMonitor: + """Sample a privacy-bounded probe for at most one hour.""" +# What: document sample a privacy bounded probe for at in the PerformanceMonitor docstring; why: introspection and maintainers read this exact docstring fragment to understand performance monitor behavior without executing it. + + # What: define __init__ around sample fn and every s and disabled and wall now; why: its direct callers call __init__ for init and rely on this exact input and result contract. + def __init__( + # What: declare the self input for __init__; why: __init__ consumes self during self sample fn sample fn, so callers must bind it with the other signature inputs. + self, sample_fn: Callable[[], dict], *, every_s: float = 5.0, + # What: declare the disabled input for __init__; why: __init__ consumes disabled during self disabled disabled, so callers must bind it with the other signature inputs. + disabled: bool = False, wall_now: Callable[[], float] = time.time, + # What: complete the enclosing predicate with group delimiter; why: PerformanceMonitor.__init__ groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) -> None: + # What: compute sample fn from sample fn; why: the enclosing return or state update later reads sample fn, so __init__ must retain the computed value under that name. + self._sample_fn = sample_fn + # What: compute wall now from wall now; why: the enclosing return or state update later reads wall now, so __init__ must retain the computed value under that name. + self._wall_now = wall_now + # What: compute lock from lock and threading; why: the enclosing return or state update later reads lock, so __init__ must retain the computed value under that name. + self._lock = threading.Lock() + # What: compute stop from the named fixture input; why: the enclosing return or state update later reads stop, so __init__ must retain the computed value under that name. + self._stop: threading.Event | None = None + # What: compute thread from the named fixture input; why: the enclosing return or state update later reads thread, so __init__ must retain the computed value under that name. + self._thread: threading.Thread | None = None + # What: compute started from false; why: the enclosing return or state update later reads started, so __init__ must retain the computed value under that name. + self._started = False + # What: compute rows from deque; why: the enclosing return or state update later reads rows, so __init__ must retain the computed value under that name. + self._rows: deque[dict] = deque() + # What: compute error from the named fixture input; why: the enclosing return or state update later reads error, so __init__ must retain the computed value under that name. + self._error: str | None = None + # What: compute every s from every s; why: the enclosing return or state update later reads every s, so __init__ must retain the computed value under that name. + self._every_s = every_s + # What: compute disabled from disabled; why: the enclosing return or state update later reads disabled, so __init__ must retain the computed value under that name. + self._disabled = disabled + # What: compute capacity from max and int and every s and 1 and 3600; why: the enclosing return or state update later reads capacity, so __init__ must retain the computed value under that name. + self._capacity = max(1, int(3600 / every_s)) + + # What: define start around the current object state; why: its direct callers call start for start and rely on this exact input and result contract. + def start(self) -> None: + # What: enter the lock managed context before self started; why: start releases this resource or lock after self started on both success and failure paths. + with self._lock: + # What: compute started from true; why: the enclosing return or state update later reads started, so start must retain the computed value under that name. + self._started = True + # What: gate on disabled and thread before the computed value; why: start admits the computed value only for this predicate and excludes the opposite state. + if self._disabled or self._thread is not None: + # What: return no value from start; why: start returns no value to callers that depend on its completed result. + return + # What: compute stop from event and threading; why: self stop stop later reads stop, so start must retain the computed value under that name. + stop = threading.Event() + # What: compute stop from stop; why: the enclosing return or state update later reads stop, so start must retain the computed value under that name. + self._stop = stop + # What: compute thread from thread and threading and run and stop and ft daemon performance; why: self thread start later reads thread, so start must retain the computed value under that name. + self._thread = threading.Thread( + # What: supply target to threading.Thread; why: start binds this run value to threading.Thread's target input. + target=self._run, args=(stop,), name="ft-daemon-performance", daemon=True + # What: complete the threading.Thread call with target and args and name and daemon; why: PerformanceMonitor.start groups the supplied clauses as one threading.Thread call before its value is consumed. + ) + # What: call self._thread.start with the declared inputs; why: start invokes self._thread.start while performing the enclosing return; the call advances that operation through its result or side effect. + self._thread.start() + + # What: define stop around the current object state; why: its direct callers call stop for stop and rely on this exact input and result contract. + def stop(self) -> None: + # What: enter the lock managed context before self started; why: stop releases this resource or lock after self started on both success and failure paths. + with self._lock: + # What: compute started from false; why: the enclosing return or state update later reads started, so stop must retain the computed value under that name. + self._started = False + # What: compute thread from thread; why: if thread is not later reads thread, so stop must retain the computed value under that name. + thread = self._thread + # What: compute thread from the named fixture input; why: the enclosing return or state update later reads thread, so stop must retain the computed value under that name. + self._thread = None + # What: compute stop from stop; why: if stop is not later reads stop, so stop must retain the computed value under that name. + stop = self._stop + # What: compute stop from the named fixture input; why: the enclosing return or state update later reads stop, so stop must retain the computed value under that name. + self._stop = None + # What: gate on stop before set and stop; why: stop admits set and stop only for this predicate and excludes the opposite state. + if stop is not None: + # What: call stop.set with the declared inputs; why: stop invokes stop.set while performing if thread is not; the call advances that operation through its result or side effect. + stop.set() + # What: gate on thread before join and thread and max and min and every s; why: stop admits join and thread and max and min and every s only for this predicate and excludes the opposite state. + if thread is not None: + # What: supply timeout to thread.join; why: stop binds this max and min and every s and 1 0 and 5 0 value to thread.join's timeout input. + thread.join(timeout=max(1.0, min(self._every_s, 5.0))) + + # What: define reconfigure around every s and disabled; why: its direct callers call reconfigure for reconfigure and rely on this exact input and result contract. + def reconfigure(self, every_s: float, disabled: bool) -> None: + # What: enter the lock managed context before changed every s self every s or disabled self disabled; why: reconfigure releases this resource or lock after changed every s self every s or disabled self disabled on both success and failure paths. + with self._lock: + # What: compute changed from every s and every s and disabled and disabled; why: if not changed later reads changed, so reconfigure must retain the computed value under that name. + changed = every_s != self._every_s or disabled != self._disabled + # What: compute restart from started; why: if restart later reads restart, so reconfigure must retain the computed value under that name. + restart = self._started + # What: gate on changed before the computed value; why: reconfigure admits the computed value only for this predicate and excludes the opposite state. + if not changed: + # What: return no value from reconfigure; why: reconfigure returns no value to callers that depend on its completed result. + return + # What: call self.stop with the declared inputs; why: reconfigure invokes self.stop while performing with self lock; the call advances that operation through its result or side effect. + self.stop() + # What: enter the lock managed context before self every s every s; why: reconfigure releases this resource or lock after self every s every s on both success and failure paths. + with self._lock: + # What: compute every s from every s; why: the enclosing return or state update later reads every s, so reconfigure must retain the computed value under that name. + self._every_s = every_s + # What: compute disabled from disabled; why: the enclosing return or state update later reads disabled, so reconfigure must retain the computed value under that name. + self._disabled = disabled + # What: compute capacity from max and int and every s and 1 and 3600; why: the enclosing return or state update later reads capacity, so reconfigure must retain the computed value under that name. + self._capacity = max(1, int(3600 / every_s)) + # What: call self._rows.clear with the declared inputs; why: reconfigure invokes self._rows.clear while performing self error; the call advances that operation through its result or side effect. + self._rows.clear() + # What: compute error from the named fixture input; why: the enclosing return or state update later reads error, so reconfigure must retain the computed value under that name. + self._error = None + # What: gate on restart before start; why: reconfigure admits start only for this predicate and excludes the opposite state. + if restart: + # What: call self.start with the declared inputs; why: reconfigure invokes self.start while performing the enclosing return; the call advances that operation through its result or side effect. + self.start() + + # What: define sample_once around the current object state; why: its direct callers call sample_once for sample once and rely on this exact input and result contract. + def sample_once(self) -> None: + # What: establish the handler boundary for the protected operation; why: PerformanceMonitor.sample_once routes failures to exception while preserving cleanup and success flow. + try: + # What: compute measured from sample fn; why: ram bytes int measured get ram bytes later reads measured, so sample_once must retain the computed value under that name. + measured = self._sample_fn() + # What: compute row from replace and int and bool and get; why: self rows append row later reads row, so sample_once must retain the computed value under that name. + row = { + # What: map the timestamp field as replace and isoformat and fromtimestamp and utc; why: PerformanceMonitor.sample_once carries timestamp through row into self rows append row. + "timestamp": datetime.fromtimestamp( + # What: call self._wall_now with the declared inputs; why: sample_once invokes self._wall_now while performing isoformat replace z; the call advances that operation through its result or side effect. + self._wall_now(), timezone.utc + # What: apply the isoformat replace z portion of row; why: sample_once uses this clause to evaluate row as one grouped value. + ).isoformat().replace("+00:00", "Z"), + # What: map the scope field as engine process tree; why: PerformanceMonitor.sample_once carries scope through row into self rows append row. + "scope": "engine-process-tree", + # What: map the ram bytes field as int and get and measured and ram bytes and 0; why: PerformanceMonitor.sample_once carries ram bytes through row into self rows append row. + "ram_bytes": int(measured.get("ramBytes", 0)), + # What: map the vram bytes field as int and get and measured and vram bytes and 0; why: PerformanceMonitor.sample_once carries vram bytes through row into self rows append row. + "vram_bytes": int(measured.get("vramBytes", 0)), + # What: map the ram available field as bool and get and measured and ram available and false; why: PerformanceMonitor.sample_once carries ram available through row into self rows append row. + "ram_available": bool(measured.get("ramAvailable", False)), + # What: map the vram available field as bool and get and measured and vram available and false; why: PerformanceMonitor.sample_once carries vram available through row into self rows append row. + "vram_available": bool(measured.get("vramAvailable", False)), + # What: map the ram source field as get and measured and ram source; why: PerformanceMonitor.sample_once carries ram source through row into self rows append row. + "ram_source": measured.get("ramSource"), + # What: map the vram source field as get and measured and vram source; why: PerformanceMonitor.sample_once carries vram source through row into self rows append row. + "vram_source": measured.get("vramSource"), + # What: complete the row mapping with timestamp and scope and ram bytes and vram bytes and ram available; why: PerformanceMonitor.sample_once groups the supplied clauses as one row mapping before its value is consumed. + } + # What: handle exception by with self lock; why: PerformanceMonitor.sample_once converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception: # noqa: BLE001 - monitoring must never break routing + # What: enter the lock managed context before self error sample failed; why: sample_once releases this resource or lock after self error sample failed on both success and failure paths. + with self._lock: + # What: compute error from sample failed; why: self error later reads error, so sample_once must retain the computed value under that name. + self._error = "sample_failed" + # What: return no value from sample_once; why: sample_once returns no value to callers that depend on its completed result. + return + # What: enter the lock managed context before self rows append row; why: sample_once releases this resource or lock after self rows append row on both success and failure paths. + with self._lock: + # What: call self._rows.append with row; why: sample_once invokes self._rows.append while performing while len self rows self capacity; the call advances that operation through its result or side effect. + self._rows.append(row) + # What: iterate across capacity and len and rows to perform popleft and rows; why: sample_once repeats the body only while or for the loop header admits an iteration. + while len(self._rows) > self._capacity: + # What: call self._rows.popleft with the declared inputs; why: sample_once invokes self._rows.popleft while performing self error; the call advances that operation through its result or side effect. + self._rows.popleft() + # What: compute error from the named fixture input; why: the enclosing return or state update later reads error, so sample_once must retain the computed value under that name. + self._error = None + + # What: define current around after; why: its direct callers call current for current and rely on this exact input and result contract. + def current(self, *, after: datetime | None = None) -> dict: + # What: compute cutoff from after and timestamp; why: if cutoff is not later reads cutoff, so current must retain the computed value under that name. + cutoff = after.timestamp() if after is not None else None + # What: enter the lock managed context before rows dict row for row in; why: current releases this resource or lock after rows dict row for row in on both success and failure paths. + with self._lock: + # What: compute rows from dict and row and rows; why: rows later reads rows, so current must retain the computed value under that name. + rows = [dict(row) for row in self._rows] + # What: compute state from every s and error and disabled and enabled and every s; why: return state sys stats rows gpu stats later reads state, so current must retain the computed value under that name. + state = { + # What: map the enabled field as disabled; why: PerformanceMonitor.current carries enabled through state into return state sys stats rows gpu stats. + "enabled": not self._disabled, + # What: map the every s field as every s; why: PerformanceMonitor.current carries every s through state into return state sys stats rows gpu stats. + "everyS": self._every_s, + # What: map the retention s field as 3600; why: PerformanceMonitor.current carries retention s through state into return state sys stats rows gpu stats. + "retentionS": 3600, + # What: map the healthy field as error; why: PerformanceMonitor.current carries healthy through state into return state sys stats rows gpu stats. + "healthy": self._error is None, + # What: map the error field as error; why: PerformanceMonitor.current carries error through state into return state sys stats rows gpu stats. + "error": self._error, + # What: complete the state mapping with enabled and every s and retention s and healthy and error; why: PerformanceMonitor.current groups the supplied clauses as one state mapping before its value is consumed. + } + # What: gate on cutoff before rows and row and cutoff and timestamp and fromisoformat; why: current admits rows and row and cutoff and timestamp and fromisoformat only for this predicate and excludes the opposite state. + if cutoff is not None: + # What: compute rows from row and rows and cutoff and timestamp; why: row for row in rows later reads rows, so current must retain the computed value under that name. + rows = [ + # What: apply the row for row in rows portion of rows; why: current uses this clause to evaluate rows as one grouped value. + row for row in rows + # What: call operation.timestamp with the declared inputs; why: current invokes operation.timestamp while performing cutoff; the call advances that operation through its result or side effect. + if datetime.fromisoformat(row["timestamp"].replace("Z", "+00:00")).timestamp() + # What: apply the cutoff portion of rows; why: current uses this clause to evaluate rows as one grouped value. + > cutoff + # What: complete the rows expression with rows row for row in rows if datetime fromisoformat row; why: PerformanceMonitor.current groups the supplied clauses as one rows expression before its value is consumed. + ] + # What: map the sys stats field as rows; why: PerformanceMonitor.current carries sys stats into return {**state, "sys_stats": rows, "gpu_stats": []}. + return {**state, "sys_stats": rows, "gpu_stats": []} + + # What: define _run around stop; why: its direct callers call _run for run and rely on this exact input and result contract. + def _run(self, stop: threading.Event) -> None: + # What: iterate across wait and every s and stop to perform sample once; why: _run repeats the body only while or for the loop header admits an iteration. + while not stop.wait(self._every_s): + # What: call self.sample_once with the declared inputs; why: _run invokes self.sample_once while performing the enclosing return; the call advances that operation through its result or side effect. + self.sample_once() diff --git a/python/freetoken/daemon/proxy.py b/python/freetoken/daemon/proxy.py index 9e382a4d72..9673d2c44e 100644 --- a/python/freetoken/daemon/proxy.py +++ b/python/freetoken/daemon/proxy.py @@ -63,6 +63,24 @@ def __init__( def health(self, port: int) -> dict: return self._cached("health", "/health", port) + # What: define an uncached health probe for the active engine port; why: readiness checks must bypass a replaced generation's cached response before accepting the new process. + def fresh_health(self, port: int) -> dict: + """Read this generation, never a cached response from a replaced engine.""" + # What: document read this generation never a cached in the fresh_health docstring; why: introspection and maintainers read this exact docstring fragment to understand fresh health behavior without executing it. + # What: return fetch and port and health from fresh_health; why: fresh_health exposes fetch and port and health so its caller can continue with the function\'s computed outcome. + return self._fetch("/health", port) + + # What: define fresh_readiness around the active port and probe path; why: callers route health probes through fresh_health and fetch other readiness paths directly, preserving generation-local evidence. + def fresh_readiness(self, port: int, path: str) -> dict: + """Probe a validated profile path without reusing prior-generation state.""" + # What: document probe a validated profile path without in the fresh_readiness docstring; why: introspection and maintainers read this exact docstring fragment to understand fresh readiness behavior without executing it. + # What: gate on path before fresh health and port; why: fresh_readiness admits fresh health and port only for this predicate and excludes the opposite state. + if path == "/health": + # What: return fresh health and port from fresh_readiness; why: fresh_readiness exposes fresh health and port so its caller can continue with the function\'s computed outcome. + return self.fresh_health(port) + # What: return fetch and path and port from fresh_readiness; why: fresh_readiness exposes fetch and path and port so its caller can continue with the function\'s computed outcome. + return self._fetch(path, port) + def stats(self, port: int) -> dict: return self._cached("stats", "/v1/stats", port) @@ -114,7 +132,16 @@ def _urlopen(url: str, timeout: float) -> dict: req = urllib.request.Request(url, headers={"Accept": "application/json"}, method="GET") with urllib.request.urlopen(req, timeout=timeout) as resp: raw = resp.read() - return json.loads(raw.decode("utf-8")) + # What: establish the handler boundary for the protected operation; why: ServeProbe._urlopen routes failures to unicode decode error and jsondecode error and json while preserving cleanup and success flow. + try: + # What: return loads and json and decode and raw and utf 8 from _urlopen; why: _urlopen exposes loads and json and decode and raw and utf 8 so its caller can continue with the function\'s computed outcome. + return json.loads(raw.decode("utf-8")) + # What: handle unicode decode error and jsondecode error and json by return; why: ServeProbe._urlopen converts that failure into this concrete recovery, response, or cleanup behavior. + except (UnicodeDecodeError, json.JSONDecodeError): + # A custom readiness endpoint follows HTTP-status semantics. Do + # not retain or surface an arbitrary successful response body. + # What: return no value from _urlopen; why: _urlopen returns no value to callers that depend on its completed result. + return {} @staticmethod def _urlopen_prepare(url: str, timeout: float) -> dict: diff --git a/python/freetoken/daemon/readiness.py b/python/freetoken/daemon/readiness.py new file mode 100644 index 0000000000..498ce0a135 --- /dev/null +++ b/python/freetoken/daemon/readiness.py @@ -0,0 +1,108 @@ +"""Wait for a newly launched FreeToken serve to report its own readiness. + +The daemon never treats a listening socket as ready. ``/health`` is the +engine's lifecycle authority and reports ``loading``, ``ok``, or ``error``. +This helper intentionally does not kill an engine on timeout: model loads can +be slow, and the existing manager must keep the still-visible process available +for logs, diagnosis, or an explicit operator stop. +""" +# What: document wait for a newly launched free token in the readiness docstring; why: introspection and maintainers read this exact docstring fragment to understand readiness behavior without executing it. +# What: document the daemon never treats a listening in the readiness docstring; why: introspection and maintainers read this exact docstring fragment to understand readiness behavior without executing it. +# What: document engine s lifecycle authority and reports in the readiness docstring; why: introspection and maintainers read this exact docstring fragment to understand readiness behavior without executing it. +# What: document this helper intentionally does not kill in the readiness docstring; why: introspection and maintainers read this exact docstring fragment to understand readiness behavior without executing it. +# What: document be slow and the existing manager in the readiness docstring; why: introspection and maintainers read this exact docstring fragment to understand readiness behavior without executing it. +# What: document for logs diagnosis or an explicit in the readiness docstring; why: introspection and maintainers read this exact docstring fragment to understand readiness behavior without executing it. +# What: preserve the paragraph boundary in the the readiness docstring; why: introspection and maintainers read this paragraph break to understand readiness behavior without executing it. + +# What: enable postponed evaluation of annotations; why: type hints in readiness can reference runtime types without eager imports or forward-reference failures. +from __future__ import annotations + +# What: import time for wait for ready using time; why: wait_for_ready uses time monotonic, making that imported dependency available to its named operation. +import time +from collections.abc import Callable + +# What: import any and callable for wait for ready using typing and any and callable; why: wait_for_ready uses the any annotation in wait for ready and the callable annotation in wait for ready, making that imported dependency available to its named operation. +from typing import Any + + +# What: define wait_for_ready around manager and probe and pid and port and timeout s and path and now and sleep; why: its direct callers call wait_for_ready for wait for ready and rely on this exact input and result contract. +def wait_for_ready( + # What: declare the manager input for wait_for_ready; why: wait_for_ready consumes manager during state manager status, so callers must bind it with the other signature inputs. + manager, + # What: declare the probe input for wait_for_ready; why: wait_for_ready consumes probe during probe fresh health port, so callers must bind it with the other signature inputs. + probe, + # What: mark the remaining parameters as keyword-only; why: wait_for_ready prevents callers from confusing adjacent lifecycle and timing arguments. + *, + # What: declare the pid input for wait_for_ready; why: wait_for_ready consumes pid during if not state get running or pid, so callers must bind it with the other signature inputs. + pid: int | None, + # What: declare the port input for wait_for_ready; why: wait_for_ready consumes port during probe fresh health port, so callers must bind it with the other signature inputs. + port: int, + # What: declare the timeout s input for wait_for_ready; why: wait_for_ready consumes timeout s during deadline now timeout s, so callers must bind it with the other signature inputs. + timeout_s: float, + # What: declare the path input for wait_for_ready; why: wait_for_ready consumes path during if path health, so callers must bind it with the other signature inputs. + path: str = "/health", + # What: declare the now input for wait_for_ready; why: wait_for_ready consumes now during deadline now timeout s, so callers must bind it with the other signature inputs. + now: Callable[[], float] = time.monotonic, + # What: declare the sleep input for wait_for_ready; why: wait_for_ready consumes sleep during sleep min remaining, so callers must bind it with the other signature inputs. + sleep: Callable[[float], None] = time.sleep, +# What: declare a heterogeneous readiness-result mapping; why: callers receive ready, reason, and health fields whose values include booleans, strings, and nested health data. +) -> dict[str, Any]: + # What: compute deadline from timeout s and now; why: remaining deadline now later reads deadline, so wait_for_ready must retain the computed value under that name. + deadline = now() + timeout_s + # What: map the reachable field as false; why: wait_for_ready carries reachable through last into return ready false reason superseded health last. + last: dict[str, Any] = {"reachable": False, "status": "unreachable"} + # What: poll readiness until an explicit terminal condition returns; why: supersession, ready, engine-error, and timeout returns bound this loop despite its unconditional header. + while True: + # What: compute state from status and manager; why: if not state get running or pid later reads state, so wait_for_ready must retain the computed value under that name. + state = manager.status() + # What: gate on get and pid and state before last; why: wait_for_ready admits last only for this predicate and excludes the opposite state. + if not state.get("running") or (pid is not None and state.get("pid") != pid): + # What: map the ready field as false; why: wait_for_ready carries ready into return {"ready": False, "reason": "superseded", "health": last}. + return {"ready": False, "reason": "superseded", "health": last} + # What: compute last from path and fresh health and port and fresh readiness; why: return ready reason superseded health last later reads last, so wait_for_ready must retain the computed value under that name. + last = ( + # What: call probe.fresh_health with port; why: wait_for_ready invokes probe.fresh_health while performing if path health; the call advances that operation through its result or side effect. + probe.fresh_health(port) + # What: apply the if path health portion of last; why: wait_for_ready uses this clause to evaluate last as one grouped value. + if path == "/health" + # What: call probe.fresh_readiness with port and path; why: wait_for_ready consumes the probe.fresh_readiness return value while evaluating else probe.fresh_readiness(port, path). + else probe.fresh_readiness(port, path) + # What: complete the last expression with last probe fresh health port if path equals health else probe fresh readiness; why: wait_for_ready groups the supplied clauses as one last expression before its value is consumed. + ) + # Replacement or exit can happen while the HTTP request is in flight. + # What: compute state from status and manager; why: if not state get running or pid later reads state, so wait_for_ready must retain the computed value under that name. + state = manager.status() + # What: gate on get and pid and state before last; why: wait_for_ready admits last only for this predicate and excludes the opposite state. + if not state.get("running") or (pid is not None and state.get("pid") != pid): + # What: map the ready field as false; why: wait_for_ready carries ready into return {"ready": False, "reason": "superseded", "health": last}. + return {"ready": False, "reason": "superseded", "health": last} + # What: gate on get and last and path before last; why: wait_for_ready admits last only for this predicate and excludes the opposite state. + if last.get("reachable") and ( + # What: apply the path health portion of the enclosing predicate; why: this clause remains in wait_for_ready\'s enclosing expression so its grouping and evaluation order stay intact. + # What: accept a custom endpoint only when its HTTP status succeeded; why: a reachable 503 means the model is still loading, not ready. + (path != "/health" and last.get("status") != "error") + # What: apply the or portion of the enclosing predicate; why: this clause remains in wait_for_ready\'s enclosing expression so its grouping and evaluation order stay intact. + or ( + # What: call last.get with status; why: wait_for_ready invokes last.get while performing and last get maintenance serving serving; the call advances that operation through its result or side effect. + last.get("status") == "ok" + # What: call last.get with maintenance and serving; why: wait_for_ready consumes the last.get return value while evaluating and last.get("maintenance", "serving") == "serving". + and last.get("maintenance", "serving") == "serving" + # What: complete the enclosing predicate with path differs from health or last get status equals ok; why: wait_for_ready groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) + # What: complete the enclosing predicate with last get reachable and path differs from health or last get; why: wait_for_ready groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: map the ready field as true; why: wait_for_ready carries ready into return {"ready": True, "health": last}. + return {"ready": True, "health": last} + # What: gate on get and last before last; why: wait_for_ready admits last only for this predicate and excludes the opposite state. + # What: treat an explicit health-document error as terminal while retrying custom endpoint HTTP failures; why: `/ready` commonly returns 503 during normal model loading. + if path == "/health" and last.get("status") == "error": + # What: map the ready field as false; why: wait_for_ready carries ready into return {"ready": False, "reason": "engine-error", "health": last}. + return {"ready": False, "reason": "engine-error", "health": last} + # What: compute remaining from deadline and now; why: if remaining later reads remaining, so wait_for_ready must retain the computed value under that name. + remaining = deadline - now() + # What: gate on remaining before last; why: wait_for_ready admits last only for this predicate and excludes the opposite state. + if remaining <= 0: + # What: map the ready field as false; why: wait_for_ready carries ready into return {"ready": False, "reason": "timeout", "health": last}. + return {"ready": False, "reason": "timeout", "health": last} + # What: pace the next readiness poll with the injected sleep callback; why: the quarter-second cap avoids busy-waiting while the remaining deadline prevents oversleeping the timeout. + sleep(min(0.25, remaining)) diff --git a/python/freetoken/daemon/router.py b/python/freetoken/daemon/router.py new file mode 100644 index 0000000000..f4dbcdffa4 --- /dev/null +++ b/python/freetoken/daemon/router.py @@ -0,0 +1,1997 @@ +"""Native, transport-independent admission and model activation for freetoken-swap. + +The router owns the decision to retain an already ready engine or to make a +safe lifecycle transition before an inference request is forwarded. It does +not implement HTTP itself: keeping this boundary small makes FIFO priority, +leases, readiness, and rollback directly testable without a model runtime. +""" +# What: document native transport independent admission and model activation in the router docstring; why: introspection and maintainers read this exact docstring fragment to understand router behavior without executing it. +# What: document the router owns the decision to in the router docstring; why: introspection and maintainers read this exact docstring fragment to understand router behavior without executing it. +# What: document safe lifecycle transition before an inference in the router docstring; why: introspection and maintainers read this exact docstring fragment to understand router behavior without executing it. +# What: document not implement http itself keeping this in the router docstring; why: introspection and maintainers read this exact docstring fragment to understand router behavior without executing it. +# What: document leases readiness and rollback directly testable in the router docstring; why: introspection and maintainers read this exact docstring fragment to understand router behavior without executing it. +# What: preserve the paragraph boundary in the the router docstring; why: introspection and maintainers read this paragraph break to understand router behavior without executing it. + +# What: enable postponed evaluation of annotations; why: type hints in router can reference runtime types without eager imports or forward-reference failures. +from __future__ import annotations + +# What: import socket for allocate loopback port using socket; why: allocate_loopback_port uses socket socket, making that imported dependency available to its named operation. +import socket + +# What: import threading for init using threading; why: __init__ uses threading condition, making that imported dependency available to its named operation. +import threading + +# What: import time for acquire using time; why: acquire uses time monotonic, making that imported dependency available to its named operation. +import time + +# What: import callable for init using typing and callable; why: __init__ uses the callable annotation in init, making that imported dependency available to its named operation. +from collections.abc import Callable + +# What: import dataclass and field for module initialization using dataclasses and dataclass and field; why: module initialization uses the dataclass annotation in module initialization and field, making that imported dependency available to its named operation. +from dataclasses import dataclass, field + +# What: import from catalog import DEFAULT CHECK ENDPOINT CatalogError ModelCatalog ModelProfile; why: this module calls or annotates these symbols in the branch-created operations below. +from .catalog import DEFAULT_CHECK_ENDPOINT, CatalogError, ModelCatalog, ModelProfile + +# What: import wait for ready for init using readiness and wait for ready; why: __init__ uses the wait for ready annotation in init, making that imported dependency available to its named operation. +from .readiness import wait_for_ready + +# What: import conflict and switch launch error for acquire using serve manager and conflict and switch launch error; why: acquire uses the conflict annotation in acquire and the switch launch error annotation in acquire, making that imported dependency available to its named operation. +from .serve_manager import Conflict, SwitchLaunchError + +# What: compute default profile concurrency limit from 10; why: default profile concurrency limit default profile concurrency limit later reads default profile concurrency limit, so router must retain the computed value under that name. +DEFAULT_PROFILE_CONCURRENCY_LIMIT = 10 + + +# What: define allocate_loopback_port around the current object state; why: its direct callers call allocate_loopback_port for allocate loopback port and rely on this exact input and result contract. +def allocate_loopback_port() -> int: + """Ask the kernel for an ephemeral loopback TCP port pair. + + The listener is intentionally closed before the child starts: FreeToken's + serve process, not the daemon, must own the listening socket. The manager + serializes the immediately following launch. FreeToken also reserves the + next port for its local distributed store, so both adjacent ports must be + available. A local process can still win the unavoidable post-check bind + race, in which case readiness fails closed and rollback applies. + """ + # What: document ask the kernel for an ephemeral in the allocate_loopback_port docstring; why: introspection and maintainers read this exact docstring fragment to understand allocate loopback port behavior without executing it. + # What: document the listener is intentionally closed before in the allocate_loopback_port docstring; why: introspection and maintainers read this exact docstring fragment to understand allocate loopback port behavior without executing it. + # What: document serve process not the daemon must in the allocate_loopback_port docstring; why: introspection and maintainers read this exact docstring fragment to understand allocate loopback port behavior without executing it. + # What: document serializes the immediately following launch a in the allocate_loopback_port docstring; why: introspection and maintainers read this exact docstring fragment to understand allocate loopback port behavior without executing it. + # What: document process can still win that unavoidable in the allocate_loopback_port docstring; why: introspection and maintainers read this exact docstring fragment to understand allocate loopback port behavior without executing it. + # What: document fails closed and the normal rollback in the allocate_loopback_port docstring; why: introspection and maintainers read this exact docstring fragment to understand allocate loopback port behavior without executing it. + # What: preserve the paragraph boundary in the the allocate_loopback_port docstring; why: introspection and maintainers read this paragraph break to understand allocate loopback port behavior without executing it. + # What: bound ephemeral-pair selection attempts; why: repeated adjacent-port conflicts must fail instead of looping forever. + for _ in range(64): + # What: reserve a kernel-selected candidate service port during validation; why: concurrent allocators cannot take the base port before its companion is checked. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as service_sock: + # What: forbid address reuse on the candidate service socket; why: the availability check must reflect an exclusive future listener. + service_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 0) + # What: ask the kernel for a loopback candidate; why: hard-coded ports would collide across parallel daemon instances. + service_sock.bind(("127.0.0.1", 0)) + # What: retain the selected service port; why: its adjacent distributed-store port must be validated before launch. + port = int(service_sock.getsockname()[1]) + # What: skip a terminal port with no valid successor; why: FreeToken cannot bind a distributed store above TCP port 65535. + if port >= 65535: + # What: retry with another kernel-selected port; why: only a complete adjacent pair is usable. + continue + # What: attempt to reserve the adjacent distributed-store port; why: FreeToken initializes its local process group on service port plus one. + try: + # What: hold the companion listener during validation; why: both required ports must be simultaneously available. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as distributed_sock: + # What: forbid address reuse on the companion socket; why: an existing listener must be detected as a conflict. + distributed_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 0) + # What: bind the exact companion port; why: a free base port alone previously produced EADDRINUSE during model switching. + distributed_sock.bind(("127.0.0.1", port + 1)) + # What: retry when the companion port is already occupied; why: the child would otherwise fail after an expensive model switch. + except OSError: + # What: continue bounded pair selection; why: another candidate may have both required listeners free. + continue + # What: return the validated base port after both temporary reservations close; why: the FreeToken child must own both real listeners. + return port + # What: fail after exhausting bounded pair-selection attempts; why: launching without a proven port pair would create a predictable lifecycle failure. + raise OSError("could not allocate adjacent loopback ports for FreeToken serve") + + +# What: define RoutingError as the owner of __init__; why: daemon callers use this class boundary so those methods share one routing error state invariant. +class RoutingError(RuntimeError): + """A request could not be admitted to a ready native engine.""" +# What: document a request could not be admitted in the RoutingError docstring; why: introspection and maintainers read this exact docstring fragment to understand routing error behavior without executing it. + + # What: define __init__ around code and detail and status code and recovery; why: its direct callers call __init__ for init and rely on this exact input and result contract. + def __init__(self, code: str, detail: str, *, status_code: int = 503, recovery: dict | None = None): + # What: call operation.__init__ with detail; why: __init__ invokes operation.__init__ while performing self code code; the call advances that operation through its result or side effect. + super().__init__(detail) + # What: compute code from code; why: the enclosing return or state update later reads code, so __init__ must retain the computed value under that name. + self.code = code + # What: compute status code from status code; why: the enclosing return or state update later reads status code, so __init__ must retain the computed value under that name. + self.status_code = status_code + # What: compute recovery from recovery; why: the enclosing return or state update later reads recovery, so __init__ must retain the computed value under that name. + self.recovery = recovery + + +# What: generate dataclass initialization and value semantics for RouteLease; why: RouteLease acts as a typed state record with consistent construction, comparison, and representation. +@dataclass +# What: define RouteLease as the owner of release and proxy_base_url; why: daemon callers use this class boundary so those methods share one route lease state invariant. +class RouteLease: + """One admitted request. Call :meth:`release` exactly once when it ends.""" +# What: document one admitted request call meth release in the RouteLease docstring; why: introspection and maintainers read this exact docstring fragment to understand route lease behavior without executing it. + + # What: compute router from the named fixture input; why: self router release later reads router, so router must retain the computed value under that name. + router: RoutingCoordinator + # What: compute profile from the named fixture input; why: return self profile proxy base url self port later reads profile, so router must retain the computed value under that name. + profile: ModelProfile + # What: compute port from the named fixture input; why: return self profile proxy base url self port later reads port, so router must retain the computed value under that name. + port: int + # What: compute pid from the named fixture input; why: state get pid later reads pid, so router must retain the computed value under that name. + pid: int | None + # What: compute model id from the named fixture input; why: model id profile selector id routing profile id pin id later reads model id, so router must retain the computed value under that name. + model_id: str | None = None + # What: compute selector id from the named fixture input; why: model id profile selector id routing profile id pin id later reads selector id, so router must retain the computed value under that name. + selector_id: str | None = None + # What: compute routing profile id from the named fixture input; why: model id profile selector id routing profile id pin id later reads routing profile id, so router must retain the computed value under that name. + routing_profile_id: str | None = None + # What: compute pin id from the named fixture input; why: model id profile selector id routing profile id pin id later reads pin id, so router must retain the computed value under that name. + pin_id: str | None = None + # What: compute released from field and false and false and false; why: if lease released or self leases later reads released, so router must retain the computed value under that name. + _released: bool = field(default=False, init=False, repr=False) + + # What: define release around the current object state; why: its direct callers call release for release and rely on this exact input and result contract. + def release(self) -> None: + # What: call self.router.release with the named fixture input; why: release invokes self.router.release while performing the enclosing return; the call advances that operation through its result or side effect. + self.router.release(self) + + # What: expose proxy_base_url as a read-only computed property; why: callers read proxy_base_url through attribute access while its getter retains control of the derived value. + @property + # What: define proxy_base_url around the current object state; why: the registered API client call proxy_base_url for proxy base url and rely on this exact input and result contract. + def proxy_base_url(self) -> str: + # What: return proxy base url and port and profile from proxy_base_url; why: proxy_base_url exposes proxy base url and port and profile so its caller can continue with the function\'s computed outcome. + return self.profile.proxy_base_url(self.port) + + +# What: define RoutingCoordinator as the owner of __init__ and _adopt_exact_catalog_resident and acquire and cancel_acquire and queue_position; why: daemon callers use this class boundary so those methods share one routing coordinator state invariant. +class RoutingCoordinator: + """Serialize unsafe swaps while allowing concurrent requests for one engine. + + A higher profile priority wins over lower priority requests that have not + begun an activation. Equal priorities use strict FIFO ordering. A swap is + never started while an admitted lease exists, which preserves streaming + requests and their cancellation semantics. + """ +# What: document serialize unsafe swaps while allowing concurrent in the RoutingCoordinator docstring; why: introspection and maintainers read this exact docstring fragment to understand routing coordinator behavior without executing it. +# What: document a higher profile priority wins over in the RoutingCoordinator docstring; why: introspection and maintainers read this exact docstring fragment to understand routing coordinator behavior without executing it. +# What: document begun an activation equal priorities use in the RoutingCoordinator docstring; why: introspection and maintainers read this exact docstring fragment to understand routing coordinator behavior without executing it. +# What: document never started while an admitted lease in the RoutingCoordinator docstring; why: introspection and maintainers read this exact docstring fragment to understand routing coordinator behavior without executing it. +# What: document requests and their cancellation semantics in the RoutingCoordinator docstring; why: introspection and maintainers read this exact docstring fragment to understand routing coordinator behavior without executing it. +# What: preserve the paragraph boundary in the the RoutingCoordinator docstring; why: introspection and maintainers read this paragraph break to understand routing coordinator behavior without executing it. + + # What: define __init__ around manager and catalog and probe and default port and ready fn and timer factory and port allocator; why: its direct callers call __init__ for init and rely on this exact input and result contract. + def __init__( + # What: declare the self input for __init__; why: __init__ consumes self during self manager manager, so callers must bind it with the other signature inputs. + self, + # What: declare the manager input for __init__; why: __init__ consumes manager during self manager manager, so callers must bind it with the other signature inputs. + manager, + # What: declare the catalog input for __init__; why: __init__ consumes catalog during self catalog catalog, so callers must bind it with the other signature inputs. + catalog: ModelCatalog, + # What: declare the probe input for __init__; why: __init__ consumes probe during self probe probe, so callers must bind it with the other signature inputs. + probe, + # What: mark the remaining parameters as keyword-only; why: __init__ prevents callers from confusing adjacent lifecycle and timing arguments. + *, + # What: declare the default port input for __init__; why: __init__ consumes default port during self default port default port, so callers must bind it with the other signature inputs. + default_port: int = 1919, + # What: declare the ready fn input for __init__; why: __init__ consumes ready fn during self ready fn ready fn, so callers must bind it with the other signature inputs. + ready_fn: Callable = wait_for_ready, + # What: declare the timer factory input for __init__; why: __init__ consumes timer factory during self timer factory timer factory or self new timer, so callers must bind it with the other signature inputs. + timer_factory: Callable[[float, Callable[[], None]], object] | None = None, + # What: declare the port allocator input for __init__; why: __init__ consumes port allocator during self port allocator port allocator, so callers must bind it with the other signature inputs. + port_allocator: Callable[[], int] = allocate_loopback_port, + # What: complete the enclosing predicate with group delimiter; why: RoutingCoordinator.__init__ groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) -> None: + # What: compute manager from manager; why: the enclosing return or state update later reads manager, so __init__ must retain the computed value under that name. + self._manager = manager + # What: compute catalog from catalog; why: the enclosing return or state update later reads catalog, so __init__ must retain the computed value under that name. + self._catalog = catalog + # What: compute probe from probe; why: the enclosing return or state update later reads probe, so __init__ must retain the computed value under that name. + self._probe = probe + # What: compute default port from default port; why: the enclosing return or state update later reads default port, so __init__ must retain the computed value under that name. + self._default_port = default_port + # What: compute ready fn from ready fn; why: the enclosing return or state update later reads ready fn, so __init__ must retain the computed value under that name. + self._ready_fn = ready_fn + # What: compute timer factory from timer factory and new timer; why: the enclosing return or state update later reads timer factory, so __init__ must retain the computed value under that name. + self._timer_factory = timer_factory or self._new_timer + # What: compute port allocator from port allocator; why: the enclosing return or state update later reads port allocator, so __init__ must retain the computed value under that name. + self._port_allocator = port_allocator + # What: compute cond from condition and threading and lock; why: the enclosing return or state update later reads cond, so __init__ must retain the computed value under that name. + self._cond = threading.Condition(threading.Lock()) + # What: compute next sequence from 0; why: the enclosing return or state update later reads next sequence, so __init__ must retain the computed value under that name. + self._next_sequence = 0 + # What: initialize pending as an empty runtime accumulator; why: RoutingCoordinator.__init__ appends or maps entries into it during the enclosing return or state update before consuming the aggregate. + self._pending: list[tuple[int, int, str]] = [] + # What: initialize pending by cancellation as an empty runtime accumulator; why: RoutingCoordinator.__init__ appends or maps entries into it during the enclosing return or state update before consuming the aggregate. + self._pending_by_cancellation: dict[ + # What: apply the threading event tuple tuple int int str portion of pending by cancellation; why: __init__ uses this clause to evaluate pending by cancellation as one grouped value. + threading.Event, tuple[tuple[int, int, str], ModelProfile] + # What: apply the grouped expression portion of pending by cancellation; why: __init__ uses this clause to evaluate pending by cancellation as one grouped value. + ] = {} + # What: compute leases from 0; why: the enclosing return or state update later reads leases, so __init__ must retain the computed value under that name. + self._leases = 0 + # What: compute reservations from 0; why: the enclosing return or state update later reads reservations, so __init__ must retain the computed value under that name. + self._reservations = 0 + # What: initialize profile reservations as an empty runtime accumulator; why: RoutingCoordinator.__init__ appends or maps entries into it during the enclosing return or state update before consuming the aggregate. + self._profile_reservations: dict[str, int] = {} + # What: compute active name from the named fixture input; why: the enclosing return or state update later reads active name, so __init__ must retain the computed value under that name. + self._active_name: str | None = None + # What: compute active routing profile from the named fixture input; why: the enclosing return or state update later reads active routing profile, so __init__ must retain the computed value under that name. + self._active_routing_profile: str | None = None + # What: compute activating name from the named fixture input; why: the enclosing return or state update later reads activating name, so __init__ must retain the computed value under that name. + self._activating_name: str | None = None + # What: compute activating port from the named fixture input; why: the enclosing return or state update later reads activating port, so __init__ must retain the computed value under that name. + self._activating_port: int | None = None + # What: compute switching from false; why: the enclosing return or state update later reads switching, so __init__ must retain the computed value under that name. + self._switching = False + # What: compute manual lifecycle owner from the named fixture input; why: the enclosing return or state update later reads manual lifecycle owner, so __init__ must retain the computed value under that name. + self._manual_lifecycle_owner: object | None = None + # What: compute manual lifecycle tokens from set; why: the enclosing return or state update later reads manual lifecycle tokens, so __init__ must retain the computed value under that name. + self._manual_lifecycle_tokens: set[object] = set() + # What: compute shutdown requested from false; why: the enclosing return or state update later reads shutdown requested, so __init__ must retain the computed value under that name. + self._shutdown_requested = False + # What: compute shutdown owner from the named fixture input; why: the enclosing return or state update later reads shutdown owner, so __init__ must retain the computed value under that name. + self._shutdown_owner: object | None = None + # What: compute idle timer from the named fixture input; why: the enclosing return or state update later reads idle timer, so __init__ must retain the computed value under that name. + self._idle_timer: object | None = None + # What: compute evictions from 0; why: the enclosing return or state update later reads evictions, so __init__ must retain the computed value under that name. + self._evictions = 0 + # What: compute admissions from 0; why: the enclosing return or state update later reads admissions, so __init__ must retain the computed value under that name. + self._admissions = 0 + # What: compute activations from 0; why: the enclosing return or state update later reads activations, so __init__ must retain the computed value under that name. + self._activations = 0 + # What: compute activation failures from 0; why: the enclosing return or state update later reads activation failures, so __init__ must retain the computed value under that name. + self._activation_failures = 0 + # What: compute cancellations from 0; why: the enclosing return or state update later reads cancellations, so __init__ must retain the computed value under that name. + self._cancellations = 0 + # What: compute terminal streams from 0; why: the enclosing return or state update later reads terminal streams, so __init__ must retain the computed value under that name. + self._terminal_streams = 0 + # What: compute last ttft ms from the named fixture input; why: the enclosing return or state update later reads last ttft ms, so __init__ must retain the computed value under that name. + self._last_ttft_ms: float | None = None + # What: compute last duration ms from the named fixture input; why: the enclosing return or state update later reads last duration ms, so __init__ must retain the computed value under that name. + self._last_duration_ms: float | None = None + # What: compute last activation ms from the named fixture input; why: the enclosing return or state update later reads last activation ms, so __init__ must retain the computed value under that name. + self._last_activation_ms: float | None = None + # What: compute last queue wait ms from the named fixture input; why: the enclosing return or state update later reads last queue wait ms, so __init__ must retain the computed value under that name. + self._last_queue_wait_ms: float | None = None + # What: compute last response bytes from the named fixture input; why: the enclosing return or state update later reads last response bytes, so __init__ must retain the computed value under that name. + self._last_response_bytes: int | None = None + # What: compute last proxy bytes per second from the named fixture input; why: the enclosing return or state update later reads last proxy bytes per second, so __init__ must retain the computed value under that name. + self._last_proxy_bytes_per_second: float | None = None + # What: call self._adopt_exact_catalog_resident with the declared inputs; why: __init__ invokes self._adopt_exact_catalog_resident while performing the enclosing return; the call advances that operation through its result or side effect. + self._adopt_exact_catalog_resident() + + # What: define _adopt_exact_catalog_resident around the current object state; why: its direct callers call _adopt_exact_catalog_resident for adopt exact catalog resident and rely on this exact input and result contract. + def _adopt_exact_catalog_resident(self) -> None: + """Bind one unambiguous catalog profile to a manager-re-adopted engine. + + Omitted ports match the configured default and dynamic-port profiles + match the concrete persisted port. If multiple profiles describe the + same process identity, fail closed rather than inventing which one owns + residency. + """ + # What: document bind one unambiguous catalog profile to in the _adopt_exact_catalog_resident docstring; why: introspection and maintainers read this exact docstring fragment to understand adopt exact catalog resident behavior without executing it. + # What: document omitted ports match the configured default in the _adopt_exact_catalog_resident docstring; why: introspection and maintainers read this exact docstring fragment to understand adopt exact catalog resident behavior without executing it. + # What: document match the concrete persisted port if in the _adopt_exact_catalog_resident docstring; why: introspection and maintainers read this exact docstring fragment to understand adopt exact catalog resident behavior without executing it. + # What: document same process identity fail closed rather in the _adopt_exact_catalog_resident docstring; why: introspection and maintainers read this exact docstring fragment to understand adopt exact catalog resident behavior without executing it. + # What: document residency in the _adopt_exact_catalog_resident docstring; why: introspection and maintainers read this exact docstring fragment to understand adopt exact catalog resident behavior without executing it. + # What: preserve the paragraph boundary in the the _adopt_exact_catalog_resident docstring; why: introspection and maintainers read this paragraph break to understand adopt exact catalog resident behavior without executing it. + # What: compute state from status and manager; why: port state get port later reads state, so _adopt_exact_catalog_resident must retain the computed value under that name. + state = self._manager.status() + # What: compute port from get and state and port; why: if not state get running or not later reads port, so _adopt_exact_catalog_resident must retain the computed value under that name. + port = state.get("port") + # What: gate on port and get and isinstance and int and state before the computed value; why: _adopt_exact_catalog_resident admits the computed value only for this predicate and excludes the opposite state. + if not state.get("running") or not isinstance(port, int) or port <= 0: + # What: return no value from _adopt_exact_catalog_resident; why: _adopt_exact_catalog_resident returns no value to callers that depend on its completed result. + return + # What: compute args from serve args and manager; why: and list profile args args later reads args, so _adopt_exact_catalog_resident must retain the computed value under that name. + args = self._manager.serve_args() + # What: compute matches from profile and profiles and catalog and model; why: if len matches later reads matches, so _adopt_exact_catalog_resident must retain the computed value under that name. + matches = [ + # What: call self._catalog.profiles with the declared inputs; why: _adopt_exact_catalog_resident invokes self._catalog.profiles while performing if profile model state get model; the call advances that operation through its result or side effect. + profile for profile in self._catalog.profiles() + # What: call state.get with model; why: _adopt_exact_catalog_resident invokes state.get while performing and; the call advances that operation through its result or side effect. + if profile.model == state.get("model") + # What: apply the and portion of matches; why: _adopt_exact_catalog_resident uses this clause to evaluate matches as one grouped value. + and ( + # What: apply the profile port port portion of matches; why: _adopt_exact_catalog_resident uses this clause to evaluate matches as one grouped value. + profile.port == port + # What: apply the or profile port portion of matches; why: _adopt_exact_catalog_resident uses this clause to evaluate matches as one grouped value. + or profile.port == 0 + # What: apply the or profile port is and port self default port portion of matches; why: _adopt_exact_catalog_resident uses this clause to evaluate matches as one grouped value. + or (profile.port is None and port == self._default_port) + # What: complete the matches expression with profile model equals state get model and profile port equals port or; why: RoutingCoordinator._adopt_exact_catalog_resident groups the supplied clauses as one matches expression before its value is consumed. + ) + # What: call list with args and profile; why: _adopt_exact_catalog_resident consumes the list return value while evaluating and list(profile.args) == args. + and list(profile.args) == args + # What: complete the matches expression with matches profile for profile in self catalog profiles if profile model equals; why: RoutingCoordinator._adopt_exact_catalog_resident groups the supplied clauses as one matches expression before its value is consumed. + ] + # What: gate on len and matches before the computed value; why: _adopt_exact_catalog_resident admits the computed value only for this predicate and excludes the opposite state. + if len(matches) != 1: + # What: return no value from _adopt_exact_catalog_resident; why: _adopt_exact_catalog_resident returns no value to callers that depend on its completed result. + return + # What: compute active name from name and matches and 0; why: the enclosing return or state update later reads active name, so _adopt_exact_catalog_resident must retain the computed value under that name. + self._active_name = matches[0].name + # What: call self._schedule_idle_eviction with the declared inputs; why: _adopt_exact_catalog_resident invokes self._schedule_idle_eviction while performing the enclosing return; the call advances that operation through its result or side effect. + self._schedule_idle_eviction() + + # What: define acquire around name and cancellation and on reserved and apply loading policy and apply routing profile; why: its direct callers call acquire for acquire and rely on this exact input and result contract. + def acquire( + # What: declare the self input for acquire; why: acquire consumes self while evaluating self, so callers must bind it with the other signature inputs. + self, + # What: declare the name input for acquire; why: acquire consumes name during name apply routing profile apply routing profile, so callers must bind it with the other signature inputs. + name: str, + # What: declare the cancellation input for acquire; why: acquire consumes cancellation during if cancellation is not, so callers must bind it with the other signature inputs. + cancellation: threading.Event | None = None, + # What: declare the on reserved input for acquire; why: acquire consumes on reserved during if on reserved is not, so callers must bind it with the other signature inputs. + on_reserved: Callable[[bool, int], None] | None = None, + # What: mark the remaining parameters as keyword-only; why: acquire prevents callers from confusing adjacent lifecycle and timing arguments. + *, + # What: declare the apply loading policy input for acquire; why: acquire consumes apply loading policy during on reserved loading enabled and cold if apply loading policy, so callers must bind it with the other signature inputs. + apply_loading_policy: bool = False, + # What: declare the apply routing profile input for acquire; why: acquire consumes apply routing profile during name apply routing profile apply routing profile, so callers must bind it with the other signature inputs. + apply_routing_profile: bool = True, + # What: complete the enclosing predicate with route lease; why: RoutingCoordinator.acquire groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) -> RouteLease: + """Return a lease only after *name* has a health-verified engine.""" + # What: document return a lease only after name in the acquire docstring; why: introspection and maintainers read this exact docstring fragment to understand acquire behavior without executing it. + # What: compute queued at from monotonic and time; why: self last queue wait ms round time monotonic queued at later reads queued at, so acquire must retain the computed value under that name. + queued_at = time.monotonic() + # What: enter the cond managed context before if self shutdown requested; why: acquire releases this resource or lock after if self shutdown requested on both success and failure paths. + with self._cond: + # What: gate on shutdown requested before routing error; why: acquire admits routing error only for this predicate and excludes the opposite state. + if self._shutdown_requested: + # What: raise RoutingError for the caller; why: RoutingCoordinator.acquire stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: supply status code to RoutingError; why: acquire binds this 503 value to RoutingError's status code input. + "router_shutting_down", "router shutdown is in progress", status_code=503 + # What: complete the RoutingError call with status code; why: RoutingCoordinator.acquire groups the supplied clauses as one RoutingError call before its value is consumed. + ) + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator.acquire routes failures to catalog error while preserving cleanup and success flow. + try: + # What: evaluate and capture model id profile selector id routing profile id pin id; why: the enclosing qualifier uses the captured result in its next validation or artifact step. + model_id, profile, selector_id, routing_profile_id, pin_id = ( + # What: call self._resolve_request_locked with name; why: acquire invokes self._resolve_request_locked while performing name apply routing profile apply routing profile; the call advances that operation through its result or side effect. + self._resolve_request_locked( + # What: supply apply routing profile to self._resolve_request_locked; why: acquire binds this apply routing profile value to self._resolve_request_locked's apply routing profile input. + name, apply_routing_profile=apply_routing_profile + # What: complete the self._resolve_request_locked call with apply routing profile; why: RoutingCoordinator.acquire groups the supplied clauses as one self._resolve_request_locked call before its value is consumed. + ) + # What: execute the grouped source fragment; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + ) + # What: handle catalog error by raise routing error unknown model str exc status code 404; why: RoutingCoordinator.acquire converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError as exc: + # What: raise RoutingError for the caller; why: RoutingCoordinator.acquire stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError("unknown_model", str(exc), status_code=404) from exc + # What: gate on cancellation and is set before routing error; why: acquire admits routing error only for this predicate and excludes the opposite state. + if cancellation is not None and cancellation.is_set(): + # What: raise RoutingError for the caller; why: RoutingCoordinator.acquire stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: supply status code to RoutingError; why: acquire binds this 409 value to RoutingError's status code input. + "request_cancelled", "request cancelled before admission", status_code=409 + # What: complete the RoutingError call with status code; why: RoutingCoordinator.acquire groups the supplied clauses as one RoutingError call before its value is consumed. + ) + # What: call self._reserve_concurrency_locked with profile; why: acquire invokes self._reserve_concurrency_locked while performing ticket profile priority self next sequence profile name; the call advances that operation through its result or side effect. + self._reserve_concurrency_locked(profile) + # What: compute ticket from next sequence and name and priority and profile; why: self pending append ticket later reads ticket, so acquire must retain the computed value under that name. + ticket = (-profile.priority, self._next_sequence, profile.name) + # What: compute next sequence from 1; why: the enclosing return or state update later reads next sequence, so acquire must retain the computed value under that name. + self._next_sequence += 1 + # What: call self._pending.append with ticket; why: acquire invokes self._pending.append while performing if cancellation is not; the call advances that operation through its result or side effect. + self._pending.append(ticket) + # What: gate on cancellation before pending by cancellation and cancellation and ticket and profile; why: acquire admits pending by cancellation and cancellation and ticket and profile only for this predicate and excludes the opposite state. + if cancellation is not None: + # What: compute pending by cancellation entry from ticket and profile; why: the enclosing return or state update later reads pending by cancellation entry, so acquire must retain the computed value under that name. + self._pending_by_cancellation[cancellation] = (ticket, profile) + # What: gate on on reserved before position and loading enabled and cold and base exception and send loading state; why: acquire admits position and loading enabled and cold and base exception and send loading state only for this predicate and excludes the opposite state. + if on_reserved is not None: + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator.acquire routes failures to base exception while preserving cleanup and success flow. + try: + # What: compute position from index and ticket and sorted and pending and 1; why: on reserved loading enabled and cold if apply loading policy later reads position, so acquire must retain the computed value under that name. + position = sorted(self._pending).index(ticket) + 1 + # What: compute loading enabled from send loading state and profile and settings and catalog; why: on reserved loading enabled and cold if apply loading policy later reads loading enabled, so acquire must retain the computed value under that name. + loading_enabled = ( + # What: apply the profile send loading state portion of loading enabled; why: acquire uses this clause to evaluate loading enabled as one grouped value. + profile.send_loading_state + # What: apply the if profile send loading state is not portion of loading enabled; why: acquire uses this clause to evaluate loading enabled as one grouped value. + if profile.send_loading_state is not None + # What: apply the else self catalog settings send loading state portion of loading enabled; why: acquire uses this clause to evaluate loading enabled as one grouped value. + else self._catalog.settings.send_loading_state + # What: execute the grouped source fragment; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + ) + # What: compute cold from active profile ready locked and profile; why: on reserved loading enabled and cold if apply loading policy later reads cold, so acquire must retain the computed value under that name. + cold = not self._active_profile_ready_locked(profile) + # What: call on_reserved with apply loading policy and cold and loading enabled and position; why: acquire invokes on_reserved while performing except base exception; the call advances that operation through its result or side effect. + on_reserved(loading_enabled and cold if apply_loading_policy else cold, position) + # What: handle base exception by self remove pending locked ticket cancellation; why: RoutingCoordinator.acquire converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException: + # What: call self._remove_pending_locked with ticket and cancellation; why: acquire invokes self._remove_pending_locked while performing self drop concurrency reservation locked profile; the call advances that operation through its result or side effect. + self._remove_pending_locked(ticket, cancellation) + # What: call self._drop_concurrency_reservation_locked with profile; why: acquire invokes self._drop_concurrency_reservation_locked while performing self cond notify all; the call advances that operation through its result or side effect. + self._drop_concurrency_reservation_locked(profile) + # What: call self._cond.notify_all with the declared inputs; why: acquire invokes self._cond.notify_all while performing raise; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: re-propagate the active failure to the caller; why: RoutingCoordinator.acquire stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: iterate across the computed value to perform shutdown requested and remove pending locked and ticket and cancellation and notify all; why: acquire repeats the body only while or for the loop header admits an iteration. + while True: + # What: execute if self shutdown requested; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + if self._shutdown_requested: + # What: gate on remove pending locked and ticket and cancellation before drop concurrency reservation locked and profile; why: acquire admits drop concurrency reservation locked and profile only for this predicate and excludes the opposite state. + if self._remove_pending_locked(ticket, cancellation): + # What: call self._drop_concurrency_reservation_locked with profile; why: acquire invokes self._drop_concurrency_reservation_locked while performing self cond notify all; the call advances that operation through its result or side effect. + self._drop_concurrency_reservation_locked(profile) + # What: call self._cond.notify_all with the declared inputs; why: acquire invokes self._cond.notify_all while performing raise routing error; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: raise RoutingError for the caller; why: RoutingCoordinator.acquire stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: supply status code to RoutingError; why: acquire binds this 503 value to RoutingError's status code input. + "router_shutting_down", "router shutdown is in progress", status_code=503 + # What: complete the RoutingError call with status code; why: RoutingCoordinator.acquire groups the supplied clauses as one RoutingError call before its value is consumed. + ) + # What: execute if cancellation is not None and cancellation is set; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + if cancellation is not None and cancellation.is_set(): + # What: gate on remove pending locked and ticket and cancellation before drop concurrency reservation locked and profile; why: acquire admits drop concurrency reservation locked and profile only for this predicate and excludes the opposite state. + if self._remove_pending_locked(ticket, cancellation): + # What: call self._drop_concurrency_reservation_locked with profile; why: acquire invokes self._drop_concurrency_reservation_locked while performing self cond notify all; the call advances that operation through its result or side effect. + self._drop_concurrency_reservation_locked(profile) + # What: call self._cond.notify_all with the declared inputs; why: acquire invokes self._cond.notify_all while performing raise routing error; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: raise RoutingError for the caller; why: RoutingCoordinator.acquire stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: supply status code to RoutingError; why: acquire binds this 409 value to RoutingError's status code input. + "request_cancelled", "request cancelled before admission", status_code=409 + # What: complete the RoutingError call with status code; why: RoutingCoordinator.acquire groups the supplied clauses as one RoutingError call before its value is consumed. + ) + # What: compute head from min and pending; why: if ticket head later reads head, so acquire must retain the computed value under that name. + head = min(self._pending) + # What: gate on ticket and head before wait and cond; why: acquire admits wait and cond only for this predicate and excludes the opposite state. + if ticket != head: + # What: call self._cond.wait with the declared inputs; why: acquire invokes self._cond.wait while performing continue; the call advances that operation through its result or side effect. + self._cond.wait() + # What: apply the continue portion of the enclosing predicate; why: this clause remains in acquire\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: gate on switching before wait and cond; why: acquire admits wait and cond only for this predicate and excludes the opposite state. + if self._switching: + # What: call self._cond.wait with the declared inputs; why: acquire invokes self._cond.wait while performing continue; the call advances that operation through its result or side effect. + self._cond.wait() + # What: apply the continue portion of the enclosing predicate; why: this clause remains in acquire\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator.acquire routes failures to base exception while preserving cleanup and success flow. + try: + # Dynamic binding happens only for the head ticket. Other + # cold requests then reuse the committed resident target. + # What: compute port from port for and profile; why: if self matches active profile port later reads port, so acquire must retain the computed value under that name. + port = self._port_for(profile) + # What: handle base exception by self remove pending locked ticket cancellation; why: RoutingCoordinator.acquire converts that failure into this concrete recovery, response, or cleanup behavior. + except BaseException: + # What: call self._remove_pending_locked with ticket and cancellation; why: acquire invokes self._remove_pending_locked while performing self drop concurrency reservation locked profile; the call advances that operation through its result or side effect. + self._remove_pending_locked(ticket, cancellation) + # What: call self._drop_concurrency_reservation_locked with profile; why: acquire invokes self._drop_concurrency_reservation_locked while performing self cond notify all; the call advances that operation through its result or side effect. + self._drop_concurrency_reservation_locked(profile) + # What: call self._cond.notify_all with the declared inputs; why: acquire invokes self._cond.notify_all while performing raise; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: re-propagate the active failure to the caller; why: RoutingCoordinator.acquire stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: gate on matches active and profile and port before cancel idle timer; why: acquire admits cancel idle timer only for this predicate and excludes the opposite state. + if self._matches_active(profile, port): + # What: call self._cancel_idle_timer with the declared inputs; why: acquire invokes self._cancel_idle_timer while performing self remove pending locked ticket cancellation; the call advances that operation through its result or side effect. + self._cancel_idle_timer() + # What: call self._remove_pending_locked with ticket and cancellation; why: acquire invokes self._remove_pending_locked while performing self leases; the call advances that operation through its result or side effect. + self._remove_pending_locked(ticket, cancellation) + # What: compute leases from 1; why: if self leases later reads leases, so acquire must retain the computed value under that name. + self._leases += 1 + # What: compute admissions from 1; why: self admissions later reads admissions, so acquire must retain the computed value under that name. + self._admissions += 1 + # What: compute last queue wait ms from round and queued at and monotonic and time and 3; why: self last queue wait ms round activated at queued at later reads last queue wait ms, so acquire must retain the computed value under that name. + self._last_queue_wait_ms = round((time.monotonic() - queued_at) * 1000, 3) + # What: compute state from status and manager; why: state get pid later reads state, so acquire must retain the computed value under that name. + state = self._manager.status() + # What: call self._cond.notify_all with the declared inputs; why: acquire invokes self._cond.notify_all while performing return route lease; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: return route lease and profile and port and get from acquire; why: acquire exposes route lease and profile and port and get so its caller can continue with the function\'s computed outcome. + return RouteLease( + # What: apply the grouped expression portion of the enclosing predicate; why: this clause remains in acquire\'s enclosing expression so its grouping and evaluation order stay intact. + self, + # What: apply the profile portion of the enclosing predicate; why: this clause remains in acquire\'s enclosing expression so its grouping and evaluation order stay intact. + profile, + # What: apply the port portion of the enclosing predicate; why: this clause remains in acquire\'s enclosing expression so its grouping and evaluation order stay intact. + port, + # What: call state.get with pid; why: acquire invokes state.get while performing model id model id; the call advances that operation through its result or side effect. + state.get("pid"), + # What: supply model id to RouteLease; why: acquire binds this model id value to RouteLease's model id input. + model_id=model_id, + # What: supply selector id to RouteLease; why: acquire binds this selector id value to RouteLease's selector id input. + selector_id=selector_id, + # What: supply routing profile id to RouteLease; why: acquire binds this routing profile id value to RouteLease's routing profile id input. + routing_profile_id=routing_profile_id, + # What: supply pin id to RouteLease; why: acquire binds this pin id value to RouteLease's pin id input. + pin_id=pin_id, + # What: complete the RouteLease call with model id and selector id and routing profile id and pin id; why: RoutingCoordinator.acquire groups the supplied clauses as one RouteLease call before its value is consumed. + ) + # What: gate on leases before wait and cond; why: acquire admits wait and cond only for this predicate and excludes the opposite state. + if self._leases: + # What: call self._cond.wait with the declared inputs; why: acquire invokes self._cond.wait while performing continue; the call advances that operation through its result or side effect. + self._cond.wait() + # What: apply the continue portion of the enclosing predicate; why: this clause remains in acquire\'s enclosing expression so its grouping and evaluation order stay intact. + continue + # What: compute block from capacity block and profile; why: if block is not later reads block, so acquire must retain the computed value under that name. + block = self._capacity_block(profile) + # What: gate on block before remove pending locked and ticket and cancellation; why: acquire admits remove pending locked and ticket and cancellation only for this predicate and excludes the opposite state. + if block is not None: + # What: call self._remove_pending_locked with ticket and cancellation; why: acquire invokes self._remove_pending_locked while performing self drop concurrency reservation locked profile; the call advances that operation through its result or side effect. + self._remove_pending_locked(ticket, cancellation) + # What: call self._drop_concurrency_reservation_locked with profile; why: acquire invokes self._drop_concurrency_reservation_locked while performing self cond notify all; the call advances that operation through its result or side effect. + self._drop_concurrency_reservation_locked(profile) + # What: call self._cond.notify_all with the declared inputs; why: acquire invokes self._cond.notify_all while performing raise routing error capacity unavailable block status code; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: raise RoutingError for the caller; why: RoutingCoordinator.acquire stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError("capacity_unavailable", block, status_code=409) + # What: compute switching from true; why: self switching later reads switching, so acquire must retain the computed value under that name. + self._switching = True + # What: compute activating name from name and profile; why: self activating name later reads activating name, so acquire must retain the computed value under that name. + self._activating_name = profile.name + # What: compute activating port from port; why: self activating port later reads activating port, so acquire must retain the computed value under that name. + self._activating_port = port + # What: call self._remove_pending_locked with ticket and cancellation; why: acquire invokes self._remove_pending_locked while performing break; the call advances that operation through its result or side effect. + self._remove_pending_locked(ticket, cancellation) + # What: apply the break portion of the enclosing predicate; why: this clause remains in acquire\'s enclosing expression so its grouping and evaluation order stay intact. + break + + # What: compute activated at from monotonic and time; why: self last queue wait ms round activated at queued at later reads activated at, so acquire must retain the computed value under that name. + activated_at = time.monotonic() + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator.acquire routes failures to exception while preserving cleanup and success flow. + try: + # What: compute pid from activate and profile and port; why: pid later reads pid, so acquire must retain the computed value under that name. + pid = self._activate(profile, port) + # What: handle exception by with self cond; why: RoutingCoordinator.acquire converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: + # What: enter the cond managed context before self switching; why: acquire releases this resource or lock after self switching on both success and failure paths. + with self._cond: + # What: compute switching from false; why: self switching later reads switching, so acquire must retain the computed value under that name. + self._switching = False + # What: compute activating name from the named fixture input; why: self activating name later reads activating name, so acquire must retain the computed value under that name. + self._activating_name = None + # What: compute activating port from the named fixture input; why: self activating port later reads activating port, so acquire must retain the computed value under that name. + self._activating_port = None + # What: compute activation failures from 1; why: the enclosing return or state update later reads activation failures, so acquire must retain the computed value under that name. + self._activation_failures += 1 + # What: call self._drop_concurrency_reservation_locked with profile; why: acquire invokes self._drop_concurrency_reservation_locked while performing self cond notify all; the call advances that operation through its result or side effect. + self._drop_concurrency_reservation_locked(profile) + # What: call self._cond.notify_all with the declared inputs; why: acquire invokes self._cond.notify_all while performing if isinstance exc routing error; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: gate on isinstance and exc and routing error before the computed value; why: acquire admits the computed value only for this predicate and excludes the opposite state. + if isinstance(exc, RoutingError): + # What: re-propagate the active failure to the caller; why: RoutingCoordinator.acquire stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: gate on isinstance and exc and switch launch error before exc and routing error and str and rollback; why: acquire admits exc and routing error and str and rollback only for this predicate and excludes the opposite state. + if isinstance(exc, SwitchLaunchError): + # What: raise RoutingError for the caller; why: RoutingCoordinator.acquire stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError("switch_launch_failed", str(exc), recovery=exc.rollback) from exc + # What: gate on isinstance and exc and conflict before exc and routing error and str; why: acquire admits exc and routing error and str only for this predicate and excludes the opposite state. + if isinstance(exc, Conflict): + # What: raise RoutingError for the caller; why: RoutingCoordinator.acquire stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError("serve_conflict", str(exc), status_code=409) from exc + # What: raise RoutingError for the caller; why: RoutingCoordinator.acquire stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError("activation_failed", str(exc)) from exc + # What: enter the cond managed context before self active name profile name; why: acquire releases this resource or lock after self active name profile name on both success and failure paths. + with self._cond: + # What: compute active name from name and profile; why: the enclosing return or state update later reads active name, so acquire must retain the computed value under that name. + self._active_name = profile.name + # What: compute activating name from the named fixture input; why: the enclosing return or state update later reads activating name, so acquire must retain the computed value under that name. + self._activating_name = None + # What: compute activating port from the named fixture input; why: the enclosing return or state update later reads activating port, so acquire must retain the computed value under that name. + self._activating_port = None + # What: compute switching from false; why: the enclosing return or state update later reads switching, so acquire must retain the computed value under that name. + self._switching = False + # What: call self._cancel_idle_timer with the declared inputs; why: acquire invokes self._cancel_idle_timer while performing self leases; the call advances that operation through its result or side effect. + self._cancel_idle_timer() + # What: compute leases from 1; why: the enclosing return or state update later reads leases, so acquire must retain the computed value under that name. + self._leases += 1 + # What: compute admissions from 1; why: the enclosing return or state update later reads admissions, so acquire must retain the computed value under that name. + self._admissions += 1 + # What: compute last queue wait ms from round and activated at and queued at and 3 and 1000; why: the enclosing return or state update later reads last queue wait ms, so acquire must retain the computed value under that name. + self._last_queue_wait_ms = round((activated_at - queued_at) * 1000, 3) + # What: compute last activation ms from round and activated at and monotonic and time and 3; why: the enclosing return or state update later reads last activation ms, so acquire must retain the computed value under that name. + self._last_activation_ms = round((time.monotonic() - activated_at) * 1000, 3) + # What: call self._cond.notify_all with the declared inputs; why: acquire invokes self._cond.notify_all while performing return route lease; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: return route lease and profile and port and pid from acquire; why: acquire exposes route lease and profile and port and pid so its caller can continue with the function\'s computed outcome. + return RouteLease( + # What: apply the grouped expression portion of the enclosing predicate; why: this clause remains in acquire\'s enclosing expression so its grouping and evaluation order stay intact. + self, + # What: apply the profile portion of the enclosing predicate; why: this clause remains in acquire\'s enclosing expression so its grouping and evaluation order stay intact. + profile, + # What: apply the port portion of the enclosing predicate; why: this clause remains in acquire\'s enclosing expression so its grouping and evaluation order stay intact. + port, + # What: apply the pid portion of the enclosing predicate; why: this clause remains in acquire\'s enclosing expression so its grouping and evaluation order stay intact. + pid, + # What: supply model id to RouteLease; why: acquire binds this model id value to RouteLease's model id input. + model_id=model_id, + # What: supply selector id to RouteLease; why: acquire binds this selector id value to RouteLease's selector id input. + selector_id=selector_id, + # What: supply routing profile id to RouteLease; why: acquire binds this routing profile id value to RouteLease's routing profile id input. + routing_profile_id=routing_profile_id, + # What: supply pin id to RouteLease; why: acquire binds this pin id value to RouteLease's pin id input. + pin_id=pin_id, + # What: complete the RouteLease call with model id and selector id and routing profile id and pin id; why: RoutingCoordinator.acquire groups the supplied clauses as one RouteLease call before its value is consumed. + ) + + # What: define cancel_acquire around cancellation; why: its direct callers call cancel_acquire for cancel acquire and rely on this exact input and result contract. + def cancel_acquire(self, cancellation: threading.Event) -> None: + """Atomically retire queued ownership, then wake its admission worker.""" + # What: document atomically retire queued ownership then wake in the cancel_acquire docstring; why: introspection and maintainers read this exact docstring fragment to understand cancel acquire behavior without executing it. + # What: enter the cond managed context before cancellation set; why: cancel_acquire releases this resource or lock after cancellation set on both success and failure paths. + with self._cond: + # What: call cancellation.set with the declared inputs; why: cancel_acquire invokes cancellation.set while performing pending self pending by cancellation get cancellation; the call advances that operation through its result or side effect. + cancellation.set() + # What: compute pending from get and cancellation and pending by cancellation; why: if pending is not later reads pending, so cancel_acquire must retain the computed value under that name. + pending = self._pending_by_cancellation.get(cancellation) + # What: gate on pending before pending and ticket and profile; why: cancel_acquire admits pending and ticket and profile only for this predicate and excludes the opposite state. + if pending is not None: + # What: compute ticket and profile from pending; why: if self remove pending locked ticket cancellation later reads ticket and profile, so cancel_acquire must retain the computed value under that name. + ticket, profile = pending + # What: gate on remove pending locked and ticket and cancellation before drop concurrency reservation locked and profile; why: cancel_acquire admits drop concurrency reservation locked and profile only for this predicate and excludes the opposite state. + if self._remove_pending_locked(ticket, cancellation): + # What: call self._drop_concurrency_reservation_locked with profile; why: cancel_acquire invokes self._drop_concurrency_reservation_locked while performing self cond notify all; the call advances that operation through its result or side effect. + self._drop_concurrency_reservation_locked(profile) + # What: call self._cond.notify_all with the declared inputs; why: cancel_acquire invokes self._cond.notify_all while performing the enclosing return; the call advances that operation through its result or side effect. + self._cond.notify_all() + + # What: define queue_position around cancellation; why: its direct callers call queue_position for queue position and rely on this exact input and result contract. + def queue_position(self, cancellation: threading.Event) -> int | None: + """Return the current one-based scheduler position for a reserved request.""" + # What: document return the current one based scheduler position in the queue_position docstring; why: introspection and maintainers read this exact docstring fragment to understand queue position behavior without executing it. + # What: enter the cond managed context before pending self pending by cancellation get cancellation; why: queue_position releases this resource or lock after pending self pending by cancellation get cancellation on both success and failure paths. + with self._cond: + # What: compute pending from get and cancellation and pending by cancellation; why: if pending is later reads pending, so queue_position must retain the computed value under that name. + pending = self._pending_by_cancellation.get(cancellation) + # What: gate on pending before the computed value; why: queue_position admits the computed value only for this predicate and excludes the opposite state. + if pending is None: + # What: return no value from queue_position; why: queue_position returns no value to callers that depend on its completed result. + return None + # What: compute ticket and profile from pending; why: return sorted self pending index ticket later reads ticket and profile, so queue_position must retain the computed value under that name. + ticket, _profile = pending + # What: return index and ticket and sorted and pending and 1 from queue_position; why: queue_position exposes index and ticket and sorted and pending and 1 so its caller can continue with the function\'s computed outcome. + return sorted(self._pending).index(ticket) + 1 + + # What: define loading_feedback_enabled around name; why: its direct callers call loading_feedback_enabled for loading feedback enabled and rely on this exact input and result contract. + def loading_feedback_enabled(self, name: str) -> bool: + """Resolve the per-profile loading setting over the global default atomically.""" + # What: document resolve the per profile loading setting over in the loading_feedback_enabled docstring; why: introspection and maintainers read this exact docstring fragment to understand loading feedback enabled behavior without executing it. + # What: enter the cond managed context before try; why: loading_feedback_enabled releases this resource or lock after try on both success and failure paths. + with self._cond: + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator.loading_feedback_enabled routes failures to catalog error while preserving cleanup and success flow. + try: + # What: compute and profile and and and from resolve request locked and name; why: the enclosing return or state update later reads and profile and and and, so loading_feedback_enabled must retain the computed value under that name. + _, profile, _, _, _ = self._resolve_request_locked(name) + # What: handle catalog error by return false; why: RoutingCoordinator.loading_feedback_enabled converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError: + # Admission owns the authoritative unknown-model response. A + # concurrent catalog replacement must not leak an exception + # from this optional pre-admission presentation policy. + # What: return false from loading_feedback_enabled; why: loading_feedback_enabled exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: gate on send loading state and profile before send loading state and profile; why: loading_feedback_enabled admits send loading state and profile only for this predicate and excludes the opposite state. + if profile.send_loading_state is not None: + # What: return send loading state and profile from loading_feedback_enabled; why: loading_feedback_enabled exposes send loading state and profile so its caller can continue with the function\'s computed outcome. + return profile.send_loading_state + # What: return send loading state and settings and catalog from loading_feedback_enabled; why: loading_feedback_enabled exposes send loading state and settings and catalog so its caller can continue with the function\'s computed outcome. + return self._catalog.settings.send_loading_state + + # What: define _resolve_request_locked around name and apply routing profile; why: its direct callers call _resolve_request_locked for resolve request locked and rely on this exact input and result contract. + def _resolve_request_locked( + # What: declare the self input for _resolve_request_locked; why: _resolve_request_locked consumes self during if apply routing profile and self active routing profile is not, so callers must bind it with the other signature inputs. + self, name: str, *, apply_routing_profile: bool = True + # What: complete the enclosing predicate collection with str and model profile and str and str; why: RoutingCoordinator._resolve_request_locked groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. + ) -> tuple[str, ModelProfile, str | None, str | None, str | None]: + # What: compute routing profile id from the named fixture input; why: routing profile id routing profile name later reads routing profile id, so _resolve_request_locked must retain the computed value under that name. + routing_profile_id = None + # What: compute pin id from the named fixture input; why: pin id name later reads pin id, so _resolve_request_locked must retain the computed value under that name. + pin_id = None + # What: gate on apply routing profile and active routing profile before routing profile and active routing profile and catalog; why: _resolve_request_locked admits routing profile and active routing profile and catalog only for this predicate and excludes the opposite state. + if apply_routing_profile and self._active_routing_profile is not None: + # What: compute routing profile from routing profile and active routing profile and catalog; why: if routing profile is not later reads routing profile, so _resolve_request_locked must retain the computed value under that name. + routing_profile = self._catalog.routing_profile(self._active_routing_profile) + # What: gate on routing profile before pinned and target and replacement and name and routing profile; why: _resolve_request_locked admits pinned and target and replacement and name and routing profile only for this predicate and excludes the opposite state. + if routing_profile is not None: + # What: compute pinned and target from replacement and name and routing profile; why: if pinned later reads pinned and target, so _resolve_request_locked must retain the computed value under that name. + pinned, target = routing_profile.replacement(name) + # What: gate on pinned before routing profile id and name and routing profile; why: _resolve_request_locked admits routing profile id and name and routing profile only for this predicate and excludes the opposite state. + if pinned: + # What: compute routing profile id from name and routing profile; why: return name self catalog get name routing profile id pin id later reads routing profile id, so _resolve_request_locked must retain the computed value under that name. + routing_profile_id = routing_profile.name + # What: compute pin id from name; why: return name self catalog get name routing profile id pin id later reads pin id, so _resolve_request_locked must retain the computed value under that name. + pin_id = name + # What: gate on target before catalog error and name and routing profile; why: _resolve_request_locked admits catalog error and name and routing profile only for this predicate and excludes the opposite state. + if target is None: + # What: raise CatalogError for the caller; why: RoutingCoordinator._resolve_request_locked stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f model id name r is portion of the enclosing predicate; why: this clause remains in _resolve_request_locked\'s enclosing expression so its grouping and evaluation order stay intact. + f"model ID {name!r} is disabled by routing profile {routing_profile.name!r}" + # What: complete the CatalogError call with name; why: RoutingCoordinator._resolve_request_locked groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: compute name from target; why: selector self catalog selector name later reads name, so _resolve_request_locked must retain the computed value under that name. + name = target + # What: compute selector from selector and name and catalog; why: if selector is later reads selector, so _resolve_request_locked must retain the computed value under that name. + selector = self._catalog.selector(name) + # What: gate on selector before name and routing profile id and pin id and get and catalog; why: _resolve_request_locked admits name and routing profile id and pin id and get and catalog only for this predicate and excludes the opposite state. + if selector is None: + # What: return name and routing profile id and pin id and get from _resolve_request_locked; why: _resolve_request_locked exposes name and routing profile id and pin id and get so its caller can continue with the function\'s computed outcome. + return name, self._catalog.get(name), None, routing_profile_id, pin_id + # What: gate on strategy and selector before target and targets and selector and profile and get; why: _resolve_request_locked admits target and targets and selector and profile and get only for this predicate and excludes the opposite state. + if selector.strategy == "warm": + # What: iterate across targets and selector to perform profile and get and target and catalog; why: _resolve_request_locked repeats the body only while or for the loop header admits an iteration. + for target in selector.targets: + # What: compute profile from get and target and catalog; why: if self active profile ready locked profile later reads profile, so _resolve_request_locked must retain the computed value under that name. + profile = self._catalog.get(target) + # What: gate on active profile ready locked and profile before target and profile and name and routing profile id and pin id; why: _resolve_request_locked admits target and profile and name and routing profile id and pin id only for this predicate and excludes the opposite state. + if self._active_profile_ready_locked(profile): + # What: return target and profile and name and routing profile id from _resolve_request_locked; why: _resolve_request_locked exposes target and profile and name and routing profile id so its caller can continue with the function\'s computed outcome. + return target, profile, selector.name, routing_profile_id, pin_id + # What: iterate across targets and selector to perform profile and get and target and catalog; why: _resolve_request_locked repeats the body only while or for the loop header admits an iteration. + for target in selector.targets: + # What: compute profile from get and target and catalog; why: if self activating name profile name later reads profile, so _resolve_request_locked must retain the computed value under that name. + profile = self._catalog.get(target) + # What: gate on activating name and name and profile before target and profile and name and routing profile id and pin id; why: _resolve_request_locked admits target and profile and name and routing profile id and pin id only for this predicate and excludes the opposite state. + if self._activating_name == profile.name: + # What: return target and profile and name and routing profile id from _resolve_request_locked; why: _resolve_request_locked exposes target and profile and name and routing profile id so its caller can continue with the function\'s computed outcome. + return target, profile, selector.name, routing_profile_id, pin_id + # What: compute target from targets and selector and 0; why: return target self catalog get target selector name routing profile id later reads target, so _resolve_request_locked must retain the computed value under that name. + target = selector.targets[0] + # What: return target and name and routing profile id and pin id from _resolve_request_locked; why: _resolve_request_locked exposes target and name and routing profile id and pin id so its caller can continue with the function\'s computed outcome. + return target, self._catalog.get(target), selector.name, routing_profile_id, pin_id + + # What: define has_routable_id around name; why: its direct callers call has_routable_id for has routable id and rely on this exact input and result contract. + def has_routable_id(self, name: str) -> bool: + """Whether *name* resolves under the current runtime profile snapshot.""" + # What: document whether name resolves under the current in the has_routable_id docstring; why: introspection and maintainers read this exact docstring fragment to understand has routable id behavior without executing it. + # What: enter the cond managed context before try; why: has_routable_id releases this resource or lock after try on both success and failure paths. + with self._cond: + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator.has_routable_id routes failures to catalog error while preserving cleanup and success flow. + try: + # What: call self._resolve_request_locked with name; why: has_routable_id invokes self._resolve_request_locked while performing except catalog error; the call advances that operation through its result or side effect. + self._resolve_request_locked(name) + # What: handle catalog error by return false; why: RoutingCoordinator.has_routable_id converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError: + # What: return false from has_routable_id; why: has_routable_id exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: return true from has_routable_id; why: has_routable_id exposes true so its caller can continue with the function\'s computed outcome. + return True + + # What: define set_active_routing_profile around name; why: its direct callers call set_active_routing_profile for set active routing profile and rely on this exact input and result contract. + def set_active_routing_profile(self, name: str | None) -> str | None: + """Atomically activate one pin map, or clear runtime pinning with ``None``.""" + # What: document atomically activate one pin map or in the set_active_routing_profile docstring; why: introspection and maintainers read this exact docstring fragment to understand set active routing profile behavior without executing it. + # What: enter the cond managed context before if name is not and self catalog routing profile; why: set_active_routing_profile releases this resource or lock after if name is not and self catalog routing profile on both success and failure paths. + with self._cond: + # What: gate on name and routing profile and catalog before routing error and name; why: set_active_routing_profile admits routing error and name only for this predicate and excludes the opposite state. + if name is not None and self._catalog.routing_profile(name) is None: + # What: raise RoutingError for the caller; why: RoutingCoordinator.set_active_routing_profile stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: supply status code to RoutingError; why: set_active_routing_profile binds this 404 value to RoutingError's status code input. + "unknown_profile", f"routing profile {name!r} not found", status_code=404 + # What: complete the RoutingError call with status code; why: RoutingCoordinator.set_active_routing_profile groups the supplied clauses as one RoutingError call before its value is consumed. + ) + # What: compute active routing profile from name; why: return self active routing profile later reads active routing profile, so set_active_routing_profile must retain the computed value under that name. + self._active_routing_profile = name + # What: call self._cond.notify_all with the declared inputs; why: set_active_routing_profile invokes self._cond.notify_all while performing return self active routing profile; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: return active routing profile from set_active_routing_profile; why: set_active_routing_profile exposes active routing profile so its caller can continue with the function\'s computed outcome. + return self._active_routing_profile + + # What: define resolve_upstream_path around path; why: its direct callers call resolve_upstream_path for resolve upstream path and rely on this exact input and result contract. + def resolve_upstream_path( + # What: declare the self input for resolve_upstream_path; why: resolve_upstream_path consumes self during with self cond, so callers must bind it with the other signature inputs. + self, path: str + # What: complete the enclosing predicate collection with str and str and model profile and str; why: RoutingCoordinator.resolve_upstream_path groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. + ) -> tuple[str, str, ModelProfile, str]: + """Apply the active profile's longest pin before concrete upstream lookup.""" + # What: document apply the active profile s longest in the resolve_upstream_path docstring; why: introspection and maintainers read this exact docstring fragment to understand resolve upstream path behavior without executing it. + # What: enter the cond managed context before normalized path strip; why: resolve_upstream_path releases this resource or lock after normalized path strip on both success and failure paths. + with self._cond: + # What: compute normalized from strip and path and value; why: rewritten normalized later reads normalized, so resolve_upstream_path must retain the computed value under that name. + normalized = path.strip("/") + # What: compute source id from the named fixture input; why: if source id is or len pin later reads source id, so resolve_upstream_path must retain the computed value under that name. + source_id = None + # What: compute rewritten from normalized; why: rewritten later reads rewritten, so resolve_upstream_path must retain the computed value under that name. + rewritten = normalized + # What: gate on active routing profile before routing profile and active routing profile and catalog; why: resolve_upstream_path admits routing profile and active routing profile and catalog only for this predicate and excludes the opposite state. + if self._active_routing_profile is not None: + # What: compute routing profile from routing profile and active routing profile and catalog; why: if routing profile is not later reads routing profile, so resolve_upstream_path must retain the computed value under that name. + routing_profile = self._catalog.routing_profile(self._active_routing_profile) + # What: gate on routing profile before pins and pin and target and routing profile and normalized; why: resolve_upstream_path admits pins and pin and target and routing profile and normalized only for this predicate and excludes the opposite state. + if routing_profile is not None: + # What: iterate across pins and routing profile to perform normalized and pin and startswith and source id and target; why: resolve_upstream_path repeats the body only while or for the loop header admits an iteration. + for pin, target in routing_profile.pins: + # What: gate on normalized and pin and startswith before source id and pin and target and rewritten and len; why: resolve_upstream_path admits source id and pin and target and rewritten and len only for this predicate and excludes the opposite state. + if (normalized == pin or normalized.startswith(pin + "/")) and ( + source_id is None or len(pin) > len(source_id) + ): + # What: compute source id from pin; why: if source id is not and not later reads source id, so resolve_upstream_path must retain the computed value under that name. + source_id = pin + # What: gate on target before rewritten; why: resolve_upstream_path admits rewritten only for this predicate and excludes the opposite state. + if target is None: + # What: compute rewritten from value; why: rewritten target normalized len pin later reads rewritten, so resolve_upstream_path must retain the computed value under that name. + rewritten = "" + # What: select the remaining branch that performs rewritten target normalized len pin; why: resolve_upstream_path covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: compute rewritten from target and normalized and len and pin; why: if source id is not and not later reads rewritten, so resolve_upstream_path must retain the computed value under that name. + rewritten = target + normalized[len(pin):] + # What: gate on source id and rewritten before catalog error and source id; why: resolve_upstream_path admits catalog error and source id only for this predicate and excludes the opposite state. + if source_id is not None and not rewritten: + # What: raise CatalogError for the caller; why: RoutingCoordinator.resolve_upstream_path stops this rejected path before it can mutate state, dispatch work, or report success. + raise CatalogError( + # What: apply the f upstream model id source id r portion of the enclosing predicate; why: this clause remains in resolve_upstream_path\'s enclosing expression so its grouping and evaluation order stay intact. + f"upstream model ID {source_id!r} is disabled by the active routing profile" + # What: complete the CatalogError call with source id; why: RoutingCoordinator.resolve_upstream_path groups the supplied clauses as one CatalogError call before its value is consumed. + ) + # What: compute routed id and profile and remaining from resolve upstream path and rewritten and catalog; why: return source id or routed id routed id profile later reads routed id and profile and remaining, so resolve_upstream_path must retain the computed value under that name. + routed_id, profile, remaining = self._catalog.resolve_upstream_path(rewritten) + # What: return routed id and profile and remaining and source id from resolve_upstream_path; why: resolve_upstream_path exposes routed id and profile and remaining and source id so its caller can continue with the function\'s computed outcome. + return source_id or routed_id, routed_id, profile, remaining + + # What: define begin_manual_lifecycle and its declared inputs; why: callers use begin_manual_lifecycle to perform the behavior named by this helper without duplicating its boundary checks. + def begin_manual_lifecycle(self, *, preempt_manual: bool = False) -> object: + """Reserve the lifecycle barrier for one legacy engine operation.""" + # What: document reserve the lifecycle barrier for one in the begin_manual_lifecycle docstring; why: introspection and maintainers read this exact docstring fragment to understand begin manual lifecycle behavior without executing it. + # What: enter the cond managed context before if self shutdown requested; why: begin_manual_lifecycle releases this resource or lock after if self shutdown requested on both success and failure paths. + with self._cond: + # What: gate on shutdown requested before routing error; why: begin_manual_lifecycle admits routing error only for this predicate and excludes the opposite state. + if self._shutdown_requested: + # What: raise RoutingError for the caller; why: RoutingCoordinator.begin_manual_lifecycle stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: supply status code to RoutingError; why: begin_manual_lifecycle binds this 503 value to RoutingError's status code input. + "router_shutting_down", "router shutdown is in progress", status_code=503 + # What: complete the RoutingError call with status code; why: RoutingCoordinator.begin_manual_lifecycle groups the supplied clauses as one RoutingError call before its value is consumed. + ) + # What: compute manual owned from manual lifecycle owner; why: or self pending and not manual owned later reads manual owned, so begin_manual_lifecycle must retain the computed value under that name. + manual_owned = self._manual_lifecycle_owner is not None + # What: compute routed owned from bool and leases and active name and pending; why: if routed owned or self switching and not later reads routed owned, so begin_manual_lifecycle must retain the computed value under that name. + routed_owned = bool( + # What: apply the self active name is not or self leases portion of routed owned; why: begin_manual_lifecycle uses this clause to evaluate routed owned as one grouped value. + self._active_name is not None or self._leases + # What: apply the or self pending and not manual owned portion of routed owned; why: begin_manual_lifecycle uses this clause to evaluate routed owned as one grouped value. + or (self._pending and not manual_owned) + # What: complete the bool call with leases; why: RoutingCoordinator.begin_manual_lifecycle groups the supplied clauses as one bool call before its value is consumed. + ) + # What: gate on routed owned and switching and preempt manual and manual owned before routing error; why: begin_manual_lifecycle admits routing error only for this predicate and excludes the opposite state. + if (routed_owned or (self._switching and not (preempt_manual and manual_owned))): + # What: raise RoutingError for the caller; why: RoutingCoordinator.begin_manual_lifecycle stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: apply the router owned portion of the enclosing predicate; why: this clause remains in begin_manual_lifecycle\'s enclosing expression so its grouping and evaluation order stay intact. + "router_owned", + # What: apply the router owns or is admitting an portion of the enclosing predicate; why: this clause remains in begin_manual_lifecycle\'s enclosing expression so its grouping and evaluation order stay intact. + "router owns or is admitting an engine; use router controls or wait", + # What: supply status code to RoutingError; why: begin_manual_lifecycle binds this 409 value to RoutingError's status code input. + status_code=409, + # What: complete the RoutingError call with status code; why: RoutingCoordinator.begin_manual_lifecycle groups the supplied clauses as one RoutingError call before its value is consumed. + ) + # What: compute owner from object; why: self manual lifecycle tokens add owner later reads owner, so begin_manual_lifecycle must retain the computed value under that name. + owner = object() + # What: call self._manual_lifecycle_tokens.add with owner; why: begin_manual_lifecycle invokes self._manual_lifecycle_tokens.add while performing self manual lifecycle owner owner; the call advances that operation through its result or side effect. + self._manual_lifecycle_tokens.add(owner) + # What: compute manual lifecycle owner from owner; why: the enclosing return or state update later reads manual lifecycle owner, so begin_manual_lifecycle must retain the computed value under that name. + self._manual_lifecycle_owner = owner + # What: compute switching from true; why: the enclosing return or state update later reads switching, so begin_manual_lifecycle must retain the computed value under that name. + self._switching = True + # What: return owner from begin_manual_lifecycle; why: begin_manual_lifecycle exposes owner so its caller can continue with the function\'s computed outcome. + return owner + + # What: define end_manual_lifecycle around owner; why: its direct callers call end_manual_lifecycle for end manual lifecycle and rely on this exact input and result contract. + def end_manual_lifecycle(self, owner: object) -> None: + """Release a matching legacy lifecycle reservation.""" + # What: document release a matching legacy lifecycle reservation in the end_manual_lifecycle docstring; why: introspection and maintainers read this exact docstring fragment to understand end manual lifecycle behavior without executing it. + # What: enter the cond managed context before if owner not in self manual lifecycle tokens; why: end_manual_lifecycle releases this resource or lock after if owner not in self manual lifecycle tokens on both success and failure paths. + with self._cond: + # What: gate on owner and manual lifecycle tokens before value error; why: end_manual_lifecycle admits value error only for this predicate and excludes the opposite state. + if owner not in self._manual_lifecycle_tokens: + # What: raise ValueError for the caller; why: RoutingCoordinator.end_manual_lifecycle stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("manual lifecycle reservation is not owned by caller") + # What: call self._manual_lifecycle_tokens.remove with owner; why: end_manual_lifecycle invokes self._manual_lifecycle_tokens.remove while performing if self manual lifecycle owner is owner; the call advances that operation through its result or side effect. + self._manual_lifecycle_tokens.remove(owner) + # What: gate on manual lifecycle owner and owner before manual lifecycle owner; why: end_manual_lifecycle admits manual lifecycle owner only for this predicate and excludes the opposite state. + if self._manual_lifecycle_owner is owner: + # What: compute manual lifecycle owner from the named fixture input; why: the enclosing return or state update later reads manual lifecycle owner, so end_manual_lifecycle must retain the computed value under that name. + self._manual_lifecycle_owner = None + # What: compute switching from false; why: the enclosing return or state update later reads switching, so end_manual_lifecycle must retain the computed value under that name. + self._switching = False + # What: call self._cond.notify_all with the declared inputs; why: end_manual_lifecycle invokes self._cond.notify_all while performing the enclosing return; the call advances that operation through its result or side effect. + self._cond.notify_all() + + # What: define release around lease; why: its direct callers call release for release and rely on this exact input and result contract. + def release(self, lease: RouteLease) -> None: + # What: enter the cond managed context before if lease router is not; why: release releases this resource or lock after if lease router is not on both success and failure paths. + with self._cond: + # What: gate on router and lease before value error; why: release admits value error only for this predicate and excludes the opposite state. + if lease.router is not self: + # What: raise ValueError for the caller; why: RoutingCoordinator.release stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("lease belongs to a different routing coordinator") + # What: gate on released and lease and leases before value error; why: release admits value error only for this predicate and excludes the opposite state. + if lease._released or self._leases <= 0: + # What: raise ValueError for the caller; why: RoutingCoordinator.release stops this rejected path before it can mutate state, dispatch work, or report success. + raise ValueError("routing lease was already released") + # What: compute released from true; why: the enclosing return or state update later reads released, so release must retain the computed value under that name. + lease._released = True + # What: compute leases from 1; why: if self leases later reads leases, so release must retain the computed value under that name. + self._leases -= 1 + # What: call self._drop_concurrency_reservation_locked with profile and lease; why: release invokes self._drop_concurrency_reservation_locked while performing if self leases; the call advances that operation through its result or side effect. + self._drop_concurrency_reservation_locked(lease.profile) + # What: gate on leases before schedule idle eviction; why: release admits schedule idle eviction only for this predicate and excludes the opposite state. + if self._leases == 0: + # What: call self._schedule_idle_eviction with the declared inputs; why: release invokes self._schedule_idle_eviction while performing self cond notify all; the call advances that operation through its result or side effect. + self._schedule_idle_eviction() + # What: call self._cond.notify_all with the declared inputs; why: release invokes self._cond.notify_all while performing the enclosing return; the call advances that operation through its result or side effect. + self._cond.notify_all() + + # What: define status around the current object state; why: its direct callers call status for status and rely on this exact input and result contract. + def status(self) -> dict: + # What: enter the cond managed context before return self status locked; why: status releases this resource or lock after return self status locked on both success and failure paths. + with self._cond: + # What: return status locked from status; why: status exposes status locked so its caller can continue with the function\'s computed outcome. + return self._status_locked() + + # What: define _status_locked around the current object state; why: its direct callers call _status_locked for status locked and rely on this exact input and result contract. + def _status_locked(self) -> dict: + # What: compute group from active name and group for and catalog; why: active group group name if group else later reads group, so _status_locked must retain the computed value under that name. + group = self._catalog.group_for(self._active_name) if self._active_name else None + # What: compute active identity matches from active matches engine locked; why: resident profiles self active name if active identity matches else later reads active identity matches, so _status_locked must retain the computed value under that name. + active_identity_matches = self._active_matches_engine_locked() + # What: return active name and active routing profile and activating name and active identity matches from _status_locked; why: _status_locked exposes active name and active routing profile and activating name and active identity matches so its caller can continue with the function\'s computed outcome. + return { + # What: map the active profile field as active name; why: RoutingCoordinator._status_locked carries active profile into "activeProfile": self._active_name. + "activeProfile": self._active_name, + # What: map the active routing profile field as active routing profile; why: RoutingCoordinator._status_locked carries active routing profile into "activeRoutingProfile": self._active_routing_profile. + "activeRoutingProfile": self._active_routing_profile, + # What: map the activating profile field as activating name; why: RoutingCoordinator._status_locked carries activating profile into "activatingProfile": self._activating_name. + "activatingProfile": self._activating_name, + # What: map the active group field as group and name; why: RoutingCoordinator._status_locked carries active group into "activeGroup": group.name if group else None. + "activeGroup": group.name if group else None, + # What: map the resident profiles field as active identity matches and active name; why: RoutingCoordinator._status_locked carries resident profiles into "residentProfiles": [self._active_name] if active_identity_matches else. + "residentProfiles": [self._active_name] if active_identity_matches else [], + # What: map the active identity matches engine field as active identity matches; why: RoutingCoordinator._status_locked carries active identity matches engine into "activeIdentityMatchesEngine": active_identity_matches. + "activeIdentityMatchesEngine": active_identity_matches, + # What: map the persistent field as bool and active identity matches and group and persistent; why: RoutingCoordinator._status_locked carries persistent into "persistent": bool(active_identity_matches and group and group.persisten. + "persistent": bool(active_identity_matches and group and group.persistent), + # What: map the max resident models field as 1; why: RoutingCoordinator._status_locked carries max resident models into "capacity": {"maxResidentModels": 1, "availableResidentSlots": 0 if self. + "capacity": {"maxResidentModels": 1, "availableResidentSlots": 0 if self._active_name else 1}, + # What: map the active requests field as leases; why: RoutingCoordinator._status_locked carries active requests into "activeRequests": self._leases. + "activeRequests": self._leases, + # What: map the reserved requests field as reservations; why: RoutingCoordinator._status_locked carries reserved requests into "reservedRequests": self._reservations. + "reservedRequests": self._reservations, + # What: map the shutting down field as shutdown requested; why: RoutingCoordinator._status_locked carries shutting down into "shuttingDown": self._shutdown_requested. + "shuttingDown": self._shutdown_requested, + # What: map the switching field as switching; why: RoutingCoordinator._status_locked carries switching into "switching": self._switching. + "switching": self._switching, + # What: map the queued requests field as len and pending; why: RoutingCoordinator._status_locked carries queued requests into "queuedRequests": len(self._pending). + "queuedRequests": len(self._pending), + # What: map the idle eviction scheduled field as idle timer; why: RoutingCoordinator._status_locked carries idle eviction scheduled into "idleEvictionScheduled": self._idle_timer is not None. + "idleEvictionScheduled": self._idle_timer is not None, + # What: map the evictions field as evictions; why: RoutingCoordinator._status_locked carries evictions into "evictions": self._evictions. + "evictions": self._evictions, + # What: map the admissions field as admissions; why: RoutingCoordinator._status_locked carries admissions into "admissions": self._admissions. + "admissions": self._admissions, + # What: map the activations field as activations; why: RoutingCoordinator._status_locked carries activations into "activations": self._activations. + "activations": self._activations, + # What: map the activation failures field as activation failures; why: RoutingCoordinator._status_locked carries activation failures into "activationFailures": self._activation_failures. + "activationFailures": self._activation_failures, + # What: map the cancellations field as cancellations; why: RoutingCoordinator._status_locked carries cancellations into "cancellations": self._cancellations. + "cancellations": self._cancellations, + # What: map the terminal streams field as terminal streams; why: RoutingCoordinator._status_locked carries terminal streams into "terminalStreams": self._terminal_streams. + "terminalStreams": self._terminal_streams, + # What: map the last ttft ms field as last ttft ms; why: RoutingCoordinator._status_locked carries last ttft ms into "lastTtftMs": self._last_ttft_ms. + "lastTtftMs": self._last_ttft_ms, + # What: map the last duration ms field as last duration ms; why: RoutingCoordinator._status_locked carries last duration ms into "lastDurationMs": self._last_duration_ms. + "lastDurationMs": self._last_duration_ms, + # What: map the last activation ms field as last activation ms; why: RoutingCoordinator._status_locked carries last activation ms into "lastActivationMs": self._last_activation_ms. + "lastActivationMs": self._last_activation_ms, + # What: map the last queue wait ms field as last queue wait ms; why: RoutingCoordinator._status_locked carries last queue wait ms into "lastQueueWaitMs": self._last_queue_wait_ms. + "lastQueueWaitMs": self._last_queue_wait_ms, + # What: map the last response bytes field as last response bytes; why: RoutingCoordinator._status_locked carries last response bytes into "lastResponseBytes": self._last_response_bytes. + "lastResponseBytes": self._last_response_bytes, + # What: map the last proxy bytes per second field as last proxy bytes per second; why: RoutingCoordinator._status_locked carries last proxy bytes per second into "lastProxyBytesPerSecond": self._last_proxy_bytes_per_second. + "lastProxyBytesPerSecond": self._last_proxy_bytes_per_second, + # What: map the scheduler field as scheduler and settings and catalog; why: RoutingCoordinator._status_locked carries scheduler into "scheduler": self._catalog.settings.scheduler. + "scheduler": self._catalog.settings.scheduler, + # What: map the global concurrency limit field as global concurrency limit and settings and catalog; why: RoutingCoordinator._status_locked carries global concurrency limit into "globalConcurrencyLimit": self._catalog.settings.global_concurrency_limi. + "globalConcurrencyLimit": self._catalog.settings.global_concurrency_limit, + # What: map the default profile concurrency limit field as default profile concurrency limit; why: RoutingCoordinator._status_locked carries default profile concurrency limit into "defaultProfileConcurrencyLimit": DEFAULT_PROFILE_CONCURRENCY_LIMIT. + "defaultProfileConcurrencyLimit": DEFAULT_PROFILE_CONCURRENCY_LIMIT, + # What: complete the enclosing predicate mapping with active profile and active routing profile and activating profile and active group and resident profiles; why: RoutingCoordinator._status_locked groups the supplied clauses as one enclosing predicate mapping mapping before its value is consumed. + } + + # What: expose catalog as a read-only computed property; why: callers read catalog through attribute access while its getter retains control of the derived value. + @property + # What: define catalog around the current object state; why: the registered API client call catalog for catalog and rely on this exact input and result contract. + def catalog(self) -> ModelCatalog: + # What: enter the cond managed context before return self catalog; why: catalog releases this resource or lock after return self catalog on both success and failure paths. + with self._cond: + # What: return catalog from catalog; why: catalog exposes catalog so its caller can continue with the function\'s computed outcome. + return self._catalog + + # What: define control_plane_snapshot around the current object state; why: its direct callers call control_plane_snapshot for control plane snapshot and rely on this exact input and result contract. + def control_plane_snapshot(self) -> tuple[ModelCatalog, dict]: + """Return one catalog and routing-state snapshot for control responses.""" + # What: document return one catalog and routing state snapshot in the control_plane_snapshot docstring; why: introspection and maintainers read this exact docstring fragment to understand control plane snapshot behavior without executing it. + # What: enter the cond managed context before return self catalog self status locked; why: control_plane_snapshot releases this resource or lock after return self catalog self status locked on both success and failure paths. + with self._cond: + # What: return catalog and status locked from control_plane_snapshot; why: control_plane_snapshot exposes catalog and status locked so its caller can continue with the function\'s computed outcome. + return self._catalog, self._status_locked() + + # What: define model_listing_snapshot around the current object state; why: its direct callers call model_listing_snapshot for model listing snapshot and rely on this exact input and result contract. + def model_listing_snapshot(self) -> tuple[ModelCatalog, frozenset[str]]: + """Return one atomic public-catalog and loaded/starting identity snapshot.""" + # What: document return one atomic public catalog and loaded in the model_listing_snapshot docstring; why: introspection and maintainers read this exact docstring fragment to understand model listing snapshot behavior without executing it. + # What: enter the cond managed context before loaded set str set; why: model_listing_snapshot releases this resource or lock after loaded set str set on both success and failure paths. + with self._cond: + # What: compute loaded from set; why: loaded add self active name later reads loaded, so model_listing_snapshot must retain the computed value under that name. + loaded: set[str] = set() + # What: gate on active name and active matches engine locked before add and active name and loaded; why: model_listing_snapshot admits add and active name and loaded only for this predicate and excludes the opposite state. + if self._active_name is not None and self._active_matches_engine_locked(): + # What: call loaded.add with active name; why: model_listing_snapshot invokes loaded.add while performing if self activating name is not and self activating port; the call advances that operation through its result or side effect. + loaded.add(self._active_name) + # What: gate on activating name and activating port before profile and get and activating name and catalog; why: model_listing_snapshot admits profile and get and activating name and catalog only for this predicate and excludes the opposite state. + if self._activating_name is not None and self._activating_port is not None: + # What: compute profile from get and activating name and catalog; why: if self engine matches profile self activating port later reads profile, so model_listing_snapshot must retain the computed value under that name. + profile = self._catalog.get(self._activating_name) + # What: gate on engine matches and profile and activating port before add and activating name and loaded; why: model_listing_snapshot admits add and activating name and loaded only for this predicate and excludes the opposite state. + if self._engine_matches(profile, self._activating_port): + # What: call loaded.add with activating name; why: model_listing_snapshot invokes loaded.add while performing return self catalog frozenset loaded; the call advances that operation through its result or side effect. + loaded.add(self._activating_name) + # What: return catalog and frozenset and loaded from model_listing_snapshot; why: model_listing_snapshot exposes catalog and frozenset and loaded so its caller can continue with the function\'s computed outcome. + return self._catalog, frozenset(loaded) + + # What: define public_model_listing_snapshot around the current object state; why: its direct callers call public_model_listing_snapshot for public model listing snapshot and rely on this exact input and result contract. + def public_model_listing_snapshot( + # What: declare the self input for public_model_listing_snapshot; why: public_model_listing_snapshot consumes self during with self cond, so callers must bind it with the other signature inputs. + self, + # What: complete the enclosing predicate collection with model catalog and frozenset and str and str; why: RoutingCoordinator.public_model_listing_snapshot groups the supplied clauses as one enclosing predicate collection collection before its value is consumed. + ) -> tuple[ModelCatalog, frozenset[str], str | None]: + """Include the active runtime pin map in the same catalog/residency snapshot.""" + # What: document include the active runtime pin map in the public_model_listing_snapshot docstring; why: introspection and maintainers read this exact docstring fragment to understand public model listing snapshot behavior without executing it. + # What: enter the cond managed context before loaded set str set; why: public_model_listing_snapshot releases this resource or lock after loaded set str set on both success and failure paths. + with self._cond: + # What: compute loaded from set; why: loaded add self active name later reads loaded, so public_model_listing_snapshot must retain the computed value under that name. + loaded: set[str] = set() + # What: gate on active name and active matches engine locked before add and active name and loaded; why: public_model_listing_snapshot admits add and active name and loaded only for this predicate and excludes the opposite state. + if self._active_name is not None and self._active_matches_engine_locked(): + # What: call loaded.add with active name; why: public_model_listing_snapshot invokes loaded.add while performing if self activating name is not and self activating port; the call advances that operation through its result or side effect. + loaded.add(self._active_name) + # What: gate on activating name and activating port before profile and get and activating name and catalog; why: public_model_listing_snapshot admits profile and get and activating name and catalog only for this predicate and excludes the opposite state. + if self._activating_name is not None and self._activating_port is not None: + # What: compute profile from get and activating name and catalog; why: if self engine matches profile self activating port later reads profile, so public_model_listing_snapshot must retain the computed value under that name. + profile = self._catalog.get(self._activating_name) + # What: gate on engine matches and profile and activating port before add and activating name and loaded; why: public_model_listing_snapshot admits add and activating name and loaded only for this predicate and excludes the opposite state. + if self._engine_matches(profile, self._activating_port): + # What: call loaded.add with activating name; why: public_model_listing_snapshot invokes loaded.add while performing return self catalog frozenset loaded self active routing profile; the call advances that operation through its result or side effect. + loaded.add(self._activating_name) + # What: return catalog and active routing profile and frozenset and loaded from public_model_listing_snapshot; why: public_model_listing_snapshot exposes catalog and active routing profile and frozenset and loaded so its caller can continue with the function\'s computed outcome. + return self._catalog, frozenset(loaded), self._active_routing_profile + + # What: define active_matches_engine around the current object state; why: its direct callers call active_matches_engine for active matches engine and rely on this exact input and result contract. + def active_matches_engine(self) -> bool: + """Whether the manager still owns the exact resident routed profile. + + A listening child alone is not a readiness signal: an out-of-band or + stale child must not make the stable router URL appear healthy for the + alias recorded by the coordinator. + """ + # What: document whether the manager still owns the in the active_matches_engine docstring; why: introspection and maintainers read this exact docstring fragment to understand active matches engine behavior without executing it. + # What: document a listening child alone is not in the active_matches_engine docstring; why: introspection and maintainers read this exact docstring fragment to understand active matches engine behavior without executing it. + # What: document stale child must not make the in the active_matches_engine docstring; why: introspection and maintainers read this exact docstring fragment to understand active matches engine behavior without executing it. + # What: document alias recorded by the coordinator in the active_matches_engine docstring; why: introspection and maintainers read this exact docstring fragment to understand active matches engine behavior without executing it. + # What: preserve the paragraph boundary in the the active_matches_engine docstring; why: introspection and maintainers read this paragraph break to understand active matches engine behavior without executing it. + # What: enter the cond managed context before return self active matches engine locked; why: active_matches_engine releases this resource or lock after return self active matches engine locked on both success and failure paths. + with self._cond: + # What: return active matches engine locked from active_matches_engine; why: active_matches_engine exposes active matches engine locked so its caller can continue with the function\'s computed outcome. + return self._active_matches_engine_locked() + + # What: define profile_is_resident around model id; why: its direct callers call profile_is_resident for profile is resident and rely on this exact input and result contract. + def profile_is_resident(self, model_id: str) -> bool: + """Whether *model_id* resolves to the exact readiness-gated resident. + + This lifecycle-state query intentionally does not perform network I/O. + It lets direct static-asset requests refuse a cold activation while + using the same exact identity check as ordinary warm admission. + """ + # What: document whether model id resolves to the exact in the profile_is_resident docstring; why: introspection and maintainers read this exact docstring fragment to understand profile is resident behavior without executing it. + # What: document this lifecycle state query intentionally does not in the profile_is_resident docstring; why: introspection and maintainers read this exact docstring fragment to understand profile is resident behavior without executing it. + # What: document it lets direct static asset requests refuse in the profile_is_resident docstring; why: introspection and maintainers read this exact docstring fragment to understand profile is resident behavior without executing it. + # What: document using the same exact identity check in the profile_is_resident docstring; why: introspection and maintainers read this exact docstring fragment to understand profile is resident behavior without executing it. + # What: preserve the paragraph boundary in the the profile_is_resident docstring; why: introspection and maintainers read this paragraph break to understand profile is resident behavior without executing it. + # What: enter the cond managed context before try; why: profile_is_resident releases this resource or lock after try on both success and failure paths. + with self._cond: + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator.profile_is_resident routes failures to catalog error while preserving cleanup and success flow. + try: + # What: compute profile from get and model id and catalog; why: and self active profile ready locked profile later reads profile, so profile_is_resident must retain the computed value under that name. + profile = self._catalog.get(model_id) + # What: handle catalog error by return false; why: RoutingCoordinator.profile_is_resident converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError: + # What: return false from profile_is_resident; why: profile_is_resident exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: return shutdown requested and switching and active profile ready locked and profile from profile_is_resident; why: profile_is_resident exposes shutdown requested and switching and active profile ready locked and profile so its caller can continue with the function\'s computed outcome. + return ( + # What: apply the not self shutdown requested portion of the enclosing predicate; why: this clause remains in profile_is_resident\'s enclosing expression so its grouping and evaluation order stay intact. + not self._shutdown_requested + # What: apply the and not self switching portion of the enclosing predicate; why: this clause remains in profile_is_resident\'s enclosing expression so its grouping and evaluation order stay intact. + and not self._switching + # What: call self._active_profile_ready_locked with profile; why: profile_is_resident consumes the self._active_profile_ready_locked return value while evaluating and self._active_profile_ready_locked(profile). + and self._active_profile_ready_locked(profile) + # What: complete the profile_is_resident signature with self and model id; why: RoutingCoordinator.profile_is_resident groups the supplied clauses as one profile_is_resident signature before its value is consumed. + ) + + # What: define is_ready around probe; why: its direct callers call is_ready for is ready and rely on this exact input and result contract. + def is_ready(self, probe=None) -> bool: + """Atomically verify resident identity and fresh engine readiness. + + Holding the admission condition across the bounded loopback probe keeps + a conflicting swap from committing between an identity snapshot and a + stale successful health response. + """ + # What: document atomically verify resident identity and fresh in the is_ready docstring; why: introspection and maintainers read this exact docstring fragment to understand is ready behavior without executing it. + # What: document holding the admission condition across the in the is_ready docstring; why: introspection and maintainers read this exact docstring fragment to understand is ready behavior without executing it. + # What: document a conflicting swap from committing around in the is_ready docstring; why: introspection and maintainers read this exact docstring fragment to understand is ready behavior without executing it. + # What: document stale successful health response in the is_ready docstring; why: introspection and maintainers read this exact docstring fragment to understand is ready behavior without executing it. + # What: preserve the paragraph boundary in the the is_ready docstring; why: introspection and maintainers read this paragraph break to understand is ready behavior without executing it. + # What: enter the cond managed context before if self shutdown requested or self switching or not; why: is_ready releases this resource or lock after if self shutdown requested or self switching or not on both success and failure paths. + with self._cond: + # What: gate on shutdown requested and switching and active matches engine locked before the computed value; why: is_ready admits the computed value only for this predicate and excludes the opposite state. + if self._shutdown_requested or self._switching or not self._active_matches_engine_locked(): + # What: return false from is_ready; why: is_ready exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: compute state from status and manager; why: port state get port later reads state, so is_ready must retain the computed value under that name. + state = self._manager.status() + # What: compute port from get and state and port; why: if not isinstance port int or later reads port, so is_ready must retain the computed value under that name. + port = state.get("port") + # What: gate on port and isinstance and int before the computed value; why: is_ready admits the computed value only for this predicate and excludes the opposite state. + if not isinstance(port, int) or port <= 0: + # What: return false from is_ready; why: is_ready exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: compute profile from get and active name and catalog; why: if profile check endpoint default check endpoint later reads profile, so is_ready must retain the computed value under that name. + profile = self._catalog.get(self._active_name) + # What: compute active probe from probe and probe; why: active probe fresh health port later reads active probe, so is_ready must retain the computed value under that name. + active_probe = probe or self._probe + # What: compute health from check endpoint and default check endpoint and fresh health and port; why: health get reachable later reads health, so is_ready must retain the computed value under that name. + health = ( + # What: call active_probe.fresh_health with port; why: is_ready invokes active_probe.fresh_health while performing if profile check endpoint default check endpoint; the call advances that operation through its result or side effect. + active_probe.fresh_health(port) + # What: apply the if profile check endpoint default check endpoint portion of health; why: is_ready uses this clause to evaluate health as one grouped value. + if profile.check_endpoint == DEFAULT_CHECK_ENDPOINT + # What: call active_probe.fresh_readiness with port and check endpoint and profile; why: is_ready consumes the active_probe.fresh_readiness return value while evaluating else active_probe.fresh_readiness(port, profile.check_endpoint). + else active_probe.fresh_readiness(port, profile.check_endpoint) + # What: complete the health expression with health active probe fresh health port if profile check endpoint equals default check endpoint else active probe fresh re; why: RoutingCoordinator.is_ready groups the supplied clauses as one health expression before its value is consumed. + ) + # What: gate on shutdown requested and switching and active matches engine locked before the computed value; why: is_ready admits the computed value only for this predicate and excludes the opposite state. + if self._shutdown_requested or self._switching or not self._active_matches_engine_locked(): + # What: return false from is_ready; why: is_ready exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: return bool and get and health and check endpoint from is_ready; why: is_ready exposes bool and get and health and check endpoint so its caller can continue with the function\'s computed outcome. + return bool( + # What: call health.get with reachable; why: is_ready invokes health.get while performing and; the call advances that operation through its result or side effect. + health.get("reachable") + # What: apply the and portion of the enclosing predicate; why: this clause remains in is_ready\'s enclosing expression so its grouping and evaluation order stay intact. + and ( + # What: apply the profile check endpoint default check endpoint portion of the enclosing predicate; why: this clause remains in is_ready\'s enclosing expression so its grouping and evaluation order stay intact. + profile.check_endpoint != DEFAULT_CHECK_ENDPOINT + # What: apply the or portion of the enclosing predicate; why: this clause remains in is_ready\'s enclosing expression so its grouping and evaluation order stay intact. + or ( + # What: call health.get with status; why: is_ready invokes health.get while performing and health get maintenance serving serving; the call advances that operation through its result or side effect. + health.get("status") == "ok" + # What: call health.get with maintenance and serving; why: is_ready consumes the health.get return value while evaluating and health.get("maintenance", "serving") == "serving". + and health.get("maintenance", "serving") == "serving" + # What: complete the bool call with get; why: RoutingCoordinator.is_ready groups the supplied clauses as one bool call before its value is consumed. + ) + # What: complete the bool call with get; why: RoutingCoordinator.is_ready groups the supplied clauses as one bool call before its value is consumed. + ) + # What: complete the bool call with get; why: RoutingCoordinator.is_ready groups the supplied clauses as one bool call before its value is consumed. + ) + + # What: expose upstream_timeout_s as a read-only computed property; why: callers read upstream_timeout_s through attribute access while its getter retains control of the derived value. + @property + # What: define upstream_timeout_s around the current object state; why: the registered API client call upstream_timeout_s for upstream timeout s and rely on this exact input and result contract. + def upstream_timeout_s(self) -> float: + # What: enter the cond managed context before return self catalog settings upstream timeout s; why: upstream_timeout_s releases this resource or lock after return self catalog settings upstream timeout s on both success and failure paths. + with self._cond: + # What: return upstream timeout s and settings and catalog from upstream_timeout_s; why: upstream_timeout_s exposes upstream timeout s and settings and catalog so its caller can continue with the function\'s computed outcome. + return self._catalog.settings.upstream_timeout_s + + # What: define replace_catalog around catalog; why: its direct callers call replace_catalog for replace catalog and rely on this exact input and result contract. + def replace_catalog(self, catalog: ModelCatalog) -> None: + """Atomically install a validated catalog without changing a live engine. + + Removing or redefining the active profile is refused. The operator can + explicitly unload first, which keeps configuration reload from silently + changing the ownership contract of an existing engine. + """ + # What: document atomically install a validated catalog without in the replace_catalog docstring; why: introspection and maintainers read this exact docstring fragment to understand replace catalog behavior without executing it. + # What: document removing or redefining the active profile in the replace_catalog docstring; why: introspection and maintainers read this exact docstring fragment to understand replace catalog behavior without executing it. + # What: document explicitly unload first which keeps configuration in the replace_catalog docstring; why: introspection and maintainers read this exact docstring fragment to understand replace catalog behavior without executing it. + # What: document changing the ownership contract of an in the replace_catalog docstring; why: introspection and maintainers read this exact docstring fragment to understand replace catalog behavior without executing it. + # What: preserve the paragraph boundary in the the replace_catalog docstring; why: introspection and maintainers read this paragraph break to understand replace catalog behavior without executing it. + # What: enter the cond managed context before if; why: replace_catalog releases this resource or lock after if on both success and failure paths. + with self._cond: + # What: gate on shutdown requested and switching and pending and manual lifecycle tokens before routing error; why: replace_catalog admits routing error only for this predicate and excludes the opposite state. + if ( + # What: apply the self shutdown requested portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + self._shutdown_requested + # What: apply the or self switching portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + or self._switching + # What: apply the or self pending portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + or self._pending + # What: apply the or self manual lifecycle tokens portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + or self._manual_lifecycle_tokens + # What: complete the enclosing predicate with if self shutdown requested or self switching or self pending or self manual lifecycle tokens raise; why: RoutingCoordinator.replace_catalog groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RoutingError for the caller; why: RoutingCoordinator.replace_catalog stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: apply the reload conflict portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + "reload_conflict", + # What: apply the cannot reload while admission or lifecycle portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + "cannot reload while admission or lifecycle work is in progress", + # What: supply status code to RoutingError; why: replace_catalog binds this 409 value to RoutingError's status code input. + status_code=409, + # What: complete the RoutingError call with status code; why: RoutingCoordinator.replace_catalog groups the supplied clauses as one RoutingError call before its value is consumed. + ) + # What: gate on active name before replacement and current and catalog error and get and active name; why: replace_catalog admits replacement and current and catalog error and get and active name only for this predicate and excludes the opposite state. + if self._active_name is not None: + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator.replace_catalog routes failures to catalog error while preserving cleanup and success flow. + try: + # What: compute replacement from get and active name and catalog; why: replacement ttl s if replacement ttl s is not else later reads replacement, so replace_catalog must retain the computed value under that name. + replacement = catalog.get(self._active_name) + # What: compute current from get and active name and catalog; why: current ttl s if current ttl s is not else later reads current, so replace_catalog must retain the computed value under that name. + current = self._catalog.get(self._active_name) + # What: handle catalog error by raise routing error; why: RoutingCoordinator.replace_catalog converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError as exc: + # What: raise RoutingError for the caller; why: RoutingCoordinator.replace_catalog stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: apply the reload conflict portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + "reload_conflict", + # What: apply the cannot remove the active profile until portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + "cannot remove the active profile until it is unloaded", + # What: supply status code to RoutingError; why: replace_catalog binds this 409 value to RoutingError's status code input. + status_code=409, + # What: apply the from exc portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + ) from exc + # What: compute current group from group for and active name and catalog; why: or replacement group current group later reads current group, so replace_catalog must retain the computed value under that name. + current_group = self._catalog.group_for(self._active_name) + # What: compute replacement group from group for and active name and catalog; why: or replacement group current group later reads replacement group, so replace_catalog must retain the computed value under that name. + replacement_group = catalog.group_for(self._active_name) + # What: compute current ttl from ttl s and default ttl s and current and settings; why: or replacement ttl current ttl later reads current ttl, so replace_catalog must retain the computed value under that name. + current_ttl = ( + # What: apply the current ttl s if current ttl s is not else portion of current ttl; why: replace_catalog uses this clause to evaluate current ttl as one grouped value. + current.ttl_s if current.ttl_s is not None else self._catalog.settings.default_ttl_s + # What: complete the current_ttl expression with current ttl current ttl s if current ttl s is not else self catalog settings default ttl s; why: RoutingCoordinator.replace_catalog groups the supplied clauses as one current_ttl expression before its value is consumed. + ) + # What: compute replacement ttl from ttl s and default ttl s and replacement and settings; why: or replacement ttl current ttl later reads replacement ttl, so replace_catalog must retain the computed value under that name. + replacement_ttl = ( + # What: apply the replacement ttl s if replacement ttl s is not else portion of replacement ttl; why: replace_catalog uses this clause to evaluate replacement ttl as one grouped value. + replacement.ttl_s if replacement.ttl_s is not None else catalog.settings.default_ttl_s + # What: complete the replacement_ttl expression with replacement ttl replacement ttl s if replacement ttl s is not else catalog settings default ttl s; why: RoutingCoordinator.replace_catalog groups the supplied clauses as one replacement_ttl expression before its value is consumed. + ) + # What: compute current unload timeout from unload timeout s and current and settings and catalog; why: or replacement unload timeout current unload timeout later reads current unload timeout, so replace_catalog must retain the computed value under that name. + current_unload_timeout = ( + # What: apply the current unload timeout s portion of current unload timeout; why: replace_catalog uses this clause to evaluate current unload timeout as one grouped value. + current.unload_timeout_s + # What: apply the if current unload timeout s is not portion of current unload timeout; why: replace_catalog uses this clause to evaluate current unload timeout as one grouped value. + if current.unload_timeout_s is not None + # What: apply the else self catalog settings unload timeout s portion of current unload timeout; why: replace_catalog uses this clause to evaluate current unload timeout as one grouped value. + else self._catalog.settings.unload_timeout_s + # What: execute the grouped source fragment; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + ) + # What: compute replacement unload timeout from unload timeout s and replacement and settings and catalog; why: or replacement unload timeout current unload timeout later reads replacement unload timeout, so replace_catalog must retain the computed value under that name. + replacement_unload_timeout = ( + # What: apply the replacement unload timeout s portion of replacement unload timeout; why: replace_catalog uses this clause to evaluate replacement unload timeout as one grouped value. + replacement.unload_timeout_s + # What: apply the if replacement unload timeout s is not portion of replacement unload timeout; why: replace_catalog uses this clause to evaluate replacement unload timeout as one grouped value. + if replacement.unload_timeout_s is not None + # What: apply the else catalog settings unload timeout s portion of replacement unload timeout; why: replace_catalog uses this clause to evaluate replacement unload timeout as one grouped value. + else catalog.settings.unload_timeout_s + # What: execute the grouped source fragment; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + ) + # What: gate on replacement and current and replacement group and current group and replacement ttl before routing error; why: replace_catalog admits routing error only for this predicate and excludes the opposite state. + if ( + # What: apply the replacement current portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + replacement != current + # What: apply the or replacement group current group portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + or replacement_group != current_group + # What: apply the or replacement ttl current ttl portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + or replacement_ttl != current_ttl + # What: apply the or replacement unload timeout current unload timeout portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + or replacement_unload_timeout != current_unload_timeout + # What: complete the enclosing predicate with if replacement differs from current or replacement group differs from; why: RoutingCoordinator.replace_catalog groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ): + # What: raise RoutingError for the caller; why: RoutingCoordinator.replace_catalog stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: apply the reload conflict portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + "reload_conflict", + # What: apply the cannot redefine the active profile until portion of the enclosing predicate; why: this clause remains in replace_catalog\'s enclosing expression so its grouping and evaluation order stay intact. + "cannot redefine the active profile until it is unloaded", + # What: supply status code to RoutingError; why: replace_catalog binds this 409 value to RoutingError's status code input. + status_code=409, + # What: complete the RoutingError call with status code; why: RoutingCoordinator.replace_catalog groups the supplied clauses as one RoutingError call before its value is consumed. + ) + # What: compute catalog from catalog; why: the enclosing return or state update later reads catalog, so replace_catalog must retain the computed value under that name. + self._catalog = catalog + # Match the pinned runtime contract: config reload starts with no + # active routing profile rather than silently carrying pin state + # into a potentially different profile definition. + # What: compute active routing profile from the named fixture input; why: the enclosing return or state update later reads active routing profile, so replace_catalog must retain the computed value under that name. + self._active_routing_profile = None + # What: call self._cond.notify_all with the declared inputs; why: replace_catalog invokes self._cond.notify_all while performing the enclosing return; the call advances that operation through its result or side effect. + self._cond.notify_all() + + # What: define prometheus around the current object state; why: its direct callers call prometheus for prometheus and rely on this exact input and result contract. + def prometheus(self) -> str: + """Render bounded router counters without importing a metrics package.""" + # What: document render bounded router counters without importing in the prometheus docstring; why: introspection and maintainers read this exact docstring fragment to understand prometheus behavior without executing it. + # What: compute status from status; why: active requests status active requests later reads status, so prometheus must retain the computed value under that name. + status = self.status() + # What: compute values from status and int and active requests and reserved requests and queued requests; why: for name value in values items later reads values, so prometheus must retain the computed value under that name. + values = { + # What: map the active requests field as status and active requests; why: RoutingCoordinator.prometheus carries active requests through values into for name value in values items. + "active_requests": status["activeRequests"], + # What: map the reserved requests field as status and reserved requests; why: RoutingCoordinator.prometheus carries reserved requests through values into for name value in values items. + "reserved_requests": status["reservedRequests"], + # What: map the queued requests field as status and queued requests; why: RoutingCoordinator.prometheus carries queued requests through values into for name value in values items. + "queued_requests": status["queuedRequests"], + # What: map the shutting down field as int and status and shutting down; why: RoutingCoordinator.prometheus carries shutting down through values into for name value in values items. + "shutting_down": int(status["shuttingDown"]), + # What: map the active identity matches engine field as int and status and active identity matches engine; why: RoutingCoordinator.prometheus carries active identity matches engine through values into for name value in values items. + "active_identity_matches_engine": int(status["activeIdentityMatchesEngine"]), + # What: map the admissions total field as status and admissions; why: RoutingCoordinator.prometheus carries admissions total through values into for name value in values items. + "admissions_total": status["admissions"], + # What: map the activations total field as status and activations; why: RoutingCoordinator.prometheus carries activations total through values into for name value in values items. + "activations_total": status["activations"], + # What: map the activation failures total field as status and activation failures; why: RoutingCoordinator.prometheus carries activation failures total through values into for name value in values items. + "activation_failures_total": status["activationFailures"], + # What: map the cancellations total field as status and cancellations; why: RoutingCoordinator.prometheus carries cancellations total through values into for name value in values items. + "cancellations_total": status["cancellations"], + # What: map the terminal streams total field as status and terminal streams; why: RoutingCoordinator.prometheus carries terminal streams total through values into for name value in values items. + "terminal_streams_total": status["terminalStreams"], + # What: map the evictions total field as status and evictions; why: RoutingCoordinator.prometheus carries evictions total through values into for name value in values items. + "evictions_total": status["evictions"], + # What: complete the values mapping with active requests and reserved requests and queued requests and shutting down and active identity matches engine; why: RoutingCoordinator.prometheus groups the supplied clauses as one values mapping before its value is consumed. + } + # What: initialize lines as an empty runtime accumulator; why: RoutingCoordinator.prometheus appends or maps entries into it during lines extend f type metric metric type f metric before consuming the aggregate. + lines = [] + # What: iterate across items and values to perform metric and name; why: prometheus repeats the body only while or for the loop header admits an iteration. + for name, value in values.items(): + # What: compute metric from name and freetoken swap; why: lines extend f type metric metric type f later reads metric, so prometheus must retain the computed value under that name. + metric = f"freetoken_swap_{name}" + # What: compute metric type from endswith and name and counter and gauge and total; why: lines extend f type metric metric type f later reads metric type, so prometheus must retain the computed value under that name. + metric_type = "counter" if name.endswith("_total") else "gauge" + # What: preserve the exact lines extend f type metric metric type f literal fragment; why: prometheus passes this fragment verbatim through lines.extend((f"# TYPE {metric} {metric_type}", f"{metric} {value}")), because changing it would alter a protocol payload, serialized fixture, or public message. + lines.extend((f"# TYPE {metric} {metric_type}", f"{metric} {value}")) + # What: iterate across status to perform value and metric and extend and name and lines; why: prometheus repeats the body only while or for the loop header admits an iteration. + for name, value in ( + # What: apply the last ttft ms status last ttft ms portion of the enclosing predicate; why: this clause remains in prometheus\'s enclosing expression so its grouping and evaluation order stay intact. + ("last_ttft_ms", status["lastTtftMs"]), + # What: apply the last duration ms status last duration ms portion of the enclosing predicate; why: this clause remains in prometheus\'s enclosing expression so its grouping and evaluation order stay intact. + ("last_duration_ms", status["lastDurationMs"]), + # What: apply the last activation ms status last activation ms portion of the enclosing predicate; why: this clause remains in prometheus\'s enclosing expression so its grouping and evaluation order stay intact. + ("last_activation_ms", status["lastActivationMs"]), + # What: apply the last queue wait ms status last queue wait ms portion of the enclosing predicate; why: this clause remains in prometheus\'s enclosing expression so its grouping and evaluation order stay intact. + ("last_queue_wait_ms", status["lastQueueWaitMs"]), + # What: apply the last response bytes status last response bytes portion of the enclosing predicate; why: this clause remains in prometheus\'s enclosing expression so its grouping and evaluation order stay intact. + ("last_response_bytes", status["lastResponseBytes"]), + # What: apply the last proxy bytes per second status last proxy bytes per second portion of the enclosing predicate; why: this clause remains in prometheus\'s enclosing expression so its grouping and evaluation order stay intact. + ("last_proxy_bytes_per_second", status["lastProxyBytesPerSecond"]), + # What: execute the grouped source fragment; why: the enclosing symbol requires this operation for its concrete qualification or routing path. + ): + # What: gate on value before metric and name; why: prometheus admits metric and name only for this predicate and excludes the opposite state. + if value is not None: + # What: compute metric from name and freetoken swap; why: lines extend f type metric gauge f later reads metric, so prometheus must retain the computed value under that name. + metric = f"freetoken_swap_{name}" + # What: preserve the exact lines extend f type metric gauge f literal fragment; why: prometheus passes this fragment verbatim through lines.extend((f"# TYPE {metric} gauge", f"{metric} {value}")), because changing it would alter a protocol payload, serialized fixture, or public message. + lines.extend((f"# TYPE {metric} gauge", f"{metric} {value}")) + # What: return join and lines and value and value from prometheus; why: prometheus exposes join and lines and value and value so its caller can continue with the function\'s computed outcome. + return "\n".join(lines) + "\n" + + # What: define record_cancellation around the current object state; why: its direct callers call record_cancellation for record cancellation and rely on this exact input and result contract. + def record_cancellation(self) -> None: + # What: enter the cond managed context before self cancellations; why: record_cancellation releases this resource or lock after self cancellations on both success and failure paths. + with self._cond: + # What: compute cancellations from 1; why: the enclosing return or state update later reads cancellations, so record_cancellation must retain the computed value under that name. + self._cancellations += 1 + + # What: define record_stream around ttft s and duration s and response bytes and completed; why: its direct callers call record_stream for record stream and rely on this exact input and result contract. + def record_stream( + # What: declare the self input for record_stream; why: record_stream consumes self during with self cond, so callers must bind it with the other signature inputs. + self, *, ttft_s: float | None, duration_s: float, response_bytes: int, completed: bool = True + # What: complete the enclosing predicate with group delimiter; why: RoutingCoordinator.record_stream groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) -> None: + """Record transport timing without crediting a router-cancelled stream as complete.""" + # What: document record transport timing without crediting a in the record_stream docstring; why: introspection and maintainers read this exact docstring fragment to understand record stream behavior without executing it. + # What: enter the cond managed context before if completed; why: record_stream releases this resource or lock after if completed on both success and failure paths. + with self._cond: + # What: gate on completed before terminal streams; why: record_stream admits terminal streams only for this predicate and excludes the opposite state. + if completed: + # What: compute terminal streams from 1; why: the enclosing return or state update later reads terminal streams, so record_stream must retain the computed value under that name. + self._terminal_streams += 1 + # What: compute last ttft ms from ttft s and round and 3 and 1000; why: the enclosing return or state update later reads last ttft ms, so record_stream must retain the computed value under that name. + self._last_ttft_ms = round(ttft_s * 1000, 3) if ttft_s is not None else None + # What: compute last duration ms from round and duration s and 3 and 1000; why: the enclosing return or state update later reads last duration ms, so record_stream must retain the computed value under that name. + self._last_duration_ms = round(duration_s * 1000, 3) + # What: compute last response bytes from response bytes; why: the enclosing return or state update later reads last response bytes, so record_stream must retain the computed value under that name. + self._last_response_bytes = response_bytes + # What: compute last proxy bytes per second from duration s and round and response bytes and 0 and 3; why: the enclosing return or state update later reads last proxy bytes per second, so record_stream must retain the computed value under that name. + self._last_proxy_bytes_per_second = round(response_bytes / duration_s, 3) if duration_s > 0 else None + + # What: define evict_idle around name; why: its direct callers call evict_idle for evict idle and rely on this exact input and result contract. + def evict_idle(self, name: str | None = None) -> bool: + """Unload a truly idle matching engine, preserving lifecycle accounting. + + The timer calls this method, and tests may call it directly. A stale + timer cannot unload a newer profile because identity is checked under + admission before entering the manager lifecycle transaction. + """ + # What: document unload a truly idle matching engine in the evict_idle docstring; why: introspection and maintainers read this exact docstring fragment to understand evict idle behavior without executing it. + # What: document the timer calls this method and in the evict_idle docstring; why: introspection and maintainers read this exact docstring fragment to understand evict idle behavior without executing it. + # What: document timer cannot unload a newer profile in the evict_idle docstring; why: introspection and maintainers read this exact docstring fragment to understand evict idle behavior without executing it. + # What: document admission before entering the manager lifecycle in the evict_idle docstring; why: introspection and maintainers read this exact docstring fragment to understand evict idle behavior without executing it. + # What: preserve the paragraph boundary in the the evict_idle docstring; why: introspection and maintainers read this paragraph break to understand evict idle behavior without executing it. + # What: enter the cond managed context before if self shutdown requested; why: evict_idle releases this resource or lock after if self shutdown requested on both success and failure paths. + with self._cond: + # What: gate on shutdown requested before the computed value; why: evict_idle admits the computed value only for this predicate and excludes the opposite state. + if self._shutdown_requested: + # What: return false from evict_idle; why: evict_idle exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: gate on name before name and catalog error and get and catalog; why: evict_idle admits name and catalog error and get and catalog only for this predicate and excludes the opposite state. + if name is not None: + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator.evict_idle routes failures to catalog error while preserving cleanup and success flow. + try: + # What: compute name from name and get and catalog; why: if name is not and active later reads name, so evict_idle must retain the computed value under that name. + name = self._catalog.get(name).name + # What: handle catalog error by return false; why: RoutingCoordinator.evict_idle converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError: + # What: return false from evict_idle; why: evict_idle exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: compute active from active name; why: if name is not and active later reads active, so evict_idle must retain the computed value under that name. + active = self._active_name + # What: gate on name and active before the computed value; why: evict_idle admits the computed value only for this predicate and excludes the opposite state. + if name is not None and active != name: + # What: return false from evict_idle; why: evict_idle exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: gate on leases and switching and active before the computed value; why: evict_idle admits the computed value only for this predicate and excludes the opposite state. + if active is None or self._leases or self._switching: + # What: return false from evict_idle; why: evict_idle exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: compute profile from get and active and catalog; why: port self port for profile later reads profile, so evict_idle must retain the computed value under that name. + profile = self._catalog.get(active) + # What: compute port from port for and profile; why: if not self matches active profile port later reads port, so evict_idle must retain the computed value under that name. + port = self._port_for(profile) + # What: gate on matches active and profile and port before active name; why: evict_idle admits active name only for this predicate and excludes the opposite state. + if not self._matches_active(profile, port): + # What: compute active name from the named fixture input; why: self active name later reads active name, so evict_idle must retain the computed value under that name. + self._active_name = None + # What: compute idle timer from the named fixture input; why: self idle timer later reads idle timer, so evict_idle must retain the computed value under that name. + self._idle_timer = None + # What: call self._cond.notify_all with the declared inputs; why: evict_idle invokes self._cond.notify_all while performing return; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: return false from evict_idle; why: evict_idle exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: compute switching from true; why: self switching later reads switching, so evict_idle must retain the computed value under that name. + self._switching = True + # What: compute idle timer from the named fixture input; why: the enclosing return or state update later reads idle timer, so evict_idle must retain the computed value under that name. + self._idle_timer = None + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator.evict_idle routes failures to exception while preserving cleanup and success flow. + try: + # What: compute timeout from unload timeout s and profile and settings and catalog; why: self manager stop timeout timeout later reads timeout, so evict_idle must retain the computed value under that name. + timeout = profile.unload_timeout_s or self._catalog.settings.unload_timeout_s + # What: supply timeout to self._manager.stop; why: evict_idle binds this timeout value to self._manager.stop's timeout input. + self._manager.stop(timeout=timeout) + # What: handle exception by with self cond; why: RoutingCoordinator.evict_idle converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception: + # What: enter the cond managed context before self switching; why: evict_idle releases this resource or lock after self switching on both success and failure paths. + with self._cond: + # What: compute switching from false; why: self switching later reads switching, so evict_idle must retain the computed value under that name. + self._switching = False + # What: call self._cond.notify_all with the declared inputs; why: evict_idle invokes self._cond.notify_all while performing raise; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: re-propagate the active failure to the caller; why: RoutingCoordinator.evict_idle stops this rejected path before it can mutate state, dispatch work, or report success. + raise + # What: enter the cond managed context before self active name; why: evict_idle releases this resource or lock after self active name on both success and failure paths. + with self._cond: + # What: compute active name from the named fixture input; why: the enclosing return or state update later reads active name, so evict_idle must retain the computed value under that name. + self._active_name = None + # What: compute switching from false; why: the enclosing return or state update later reads switching, so evict_idle must retain the computed value under that name. + self._switching = False + # What: compute evictions from 1; why: the enclosing return or state update later reads evictions, so evict_idle must retain the computed value under that name. + self._evictions += 1 + # What: call self._cond.notify_all with the declared inputs; why: evict_idle invokes self._cond.notify_all while performing return; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: return true from evict_idle; why: evict_idle exposes true so its caller can continue with the function\'s computed outcome. + return True + + # What: define begin_shutdown around the current object state; why: its direct callers call begin_shutdown for begin shutdown and rely on this exact input and result contract. + def begin_shutdown(self) -> object: + """Close admission immediately, before executor-side lifecycle work can queue.""" + # What: document close admission immediately before executor side lifecycle in the begin_shutdown docstring; why: introspection and maintainers read this exact docstring fragment to understand begin shutdown behavior without executing it. + # What: enter the cond managed context before if self shutdown requested; why: begin_shutdown releases this resource or lock after if self shutdown requested on both success and failure paths. + with self._cond: + # What: gate on shutdown requested before routing error; why: begin_shutdown admits routing error only for this predicate and excludes the opposite state. + if self._shutdown_requested: + # What: raise RoutingError for the caller; why: RoutingCoordinator.begin_shutdown stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: supply status code to RoutingError; why: begin_shutdown binds this 409 value to RoutingError's status code input. + "router_shutting_down", "router shutdown is already in progress", status_code=409 + # What: complete the RoutingError call with status code; why: RoutingCoordinator.begin_shutdown groups the supplied clauses as one RoutingError call before its value is consumed. + ) + # What: compute owner from object; why: self shutdown owner owner later reads owner, so begin_shutdown must retain the computed value under that name. + owner = object() + # What: compute shutdown requested from true; why: the enclosing return or state update later reads shutdown requested, so begin_shutdown must retain the computed value under that name. + self._shutdown_requested = True + # What: compute shutdown owner from owner; why: the enclosing return or state update later reads shutdown owner, so begin_shutdown must retain the computed value under that name. + self._shutdown_owner = owner + # What: call self._cancel_idle_timer with the declared inputs; why: begin_shutdown invokes self._cancel_idle_timer while performing self cond notify all; the call advances that operation through its result or side effect. + self._cancel_idle_timer() + # What: call self._cond.notify_all with the declared inputs; why: begin_shutdown invokes self._cond.notify_all while performing return owner; the call advances that operation through its result or side effect. + self._cond.notify_all() + # What: return owner from begin_shutdown; why: begin_shutdown exposes owner so its caller can continue with the function\'s computed outcome. + return owner + + def finish_shutdown( + self, + owner: object, + timeout: float | None = None, + force: bool = False, + drain_timeout: float | None = None, + ) -> dict: + """Drain ownership within a bound, then permanently stop the managed child.""" + # Use the manager timeout as the default drain bound so one caller setting controls the transaction. + effective_drain_timeout = timeout if drain_timeout is None else drain_timeout + # Stop the child only after leases drain or an explicitly forced deadline expires. + return self._finish_exit( + owner, + lambda: self._manager.shutdown(timeout, force), + drain_timeout=effective_drain_timeout, + force_drain=force, + ) + + def finish_detach( + self, + owner: object, + drain_timeout: float | None = None, + force_drain: bool = False, + ) -> None: + """Drain ownership within a bound, then persist the child for re-adoption.""" + # Detach only after ownership drains unless an OS-exit caller explicitly requests deadline forcing. + self._finish_exit( + owner, + self._manager.detach, + drain_timeout=drain_timeout, + force_drain=force_drain, + ) + + def _finish_exit( + self, + owner: object, + action: Callable[[], object], + *, + drain_timeout: float | None = None, + force_drain: bool = False, + ): + # Convert the relative limit once so spurious condition wakeups cannot extend shutdown forever. + deadline = None if drain_timeout is None else time.monotonic() + max(0.0, drain_timeout) + # Hold the router condition while checking ownership and waiting for active lifecycle users. + with self._cond: + # Reject callers that do not own the reservation to preserve single-owner lifecycle safety. + if self._shutdown_owner is not owner: + raise ValueError("shutdown reservation is not owned by caller") + # Wait until leases, switches, and manual tokens release or the bounded drain expires. + while self._leases or self._switching or self._manual_lifecycle_tokens: + # Wait without a limit only for legacy callers that intentionally omitted a timeout. + if deadline is None: + self._cond.wait() + continue + # Recompute remaining time after every wakeup so total wait stays bounded. + remaining = deadline - time.monotonic() + # Handle an expired drain according to the caller's explicit force policy. + if remaining <= 0: + # Continue only when the caller authorized forced operating-system-exit cleanup. + if force_drain: + break + # Roll back the shutdown latch because a non-forced timeout leaves the router operational. + self._shutdown_requested = False + # Release the reservation so another lifecycle request can retry after leases complete. + self._shutdown_owner = None + # Restore idle eviction because the aborted shutdown no longer owns lifecycle. + self._schedule_idle_eviction() + # Wake blocked threads so they can observe the restored state. + self._cond.notify_all() + # Surface a concrete timeout instead of hanging beyond the requested deadline. + raise TimeoutError("timed out waiting for router ownership to drain") + # Sleep only for the remaining bound so repeated wakeups cannot reset the deadline. + self._cond.wait(remaining) + # Reserve the transition while the terminal manager action runs outside the condition lock. + self._switching = True + # Run the child stop or detach action while preserving rollback on manager failure. + try: + result = action() + except Exception: + # Restore routing state because the manager action did not complete its terminal transition. + with self._cond: + # Clear the shutdown latch so callers can retry or use the surviving child. + self._shutdown_requested = False + # Release the failed action's ownership reservation. + self._shutdown_owner = None + # Clear the transition flag so future routing work is not permanently blocked. + self._switching = False + # Restore the idle policy that was suspended when shutdown began. + self._schedule_idle_eviction() + # Wake all blocked participants so they observe the rolled-back state. + self._cond.notify_all() + # Preserve the manager exception because it carries the actionable lifecycle failure. + raise + # Commit the terminal router state after the child action succeeds. + with self._cond: + # Clear active identity because the daemon no longer owns a routable child lifecycle. + self._active_name = None + # Release the shutdown reservation because its terminal action completed. + self._shutdown_owner = None + # Clear the transition flag so status accurately reports the completed exit. + self._switching = False + # Wake blocked participants so repeated exit hooks can return idempotently. + self._cond.notify_all() + # Return the manager result to preserve the public shutdown response contract. + return result + + # What: define shutdown around timeout and force; why: its direct callers call shutdown for shutdown and rely on this exact input and result contract. + def shutdown(self, timeout: float | None = None, force: bool = False) -> dict: + """Synchronous convenience wrapper for a complete shutdown transaction.""" + # What: document synchronous convenience wrapper for a complete in the shutdown docstring; why: introspection and maintainers read this exact docstring fragment to understand shutdown behavior without executing it. + # What: return finish shutdown and timeout and force and begin shutdown from shutdown; why: shutdown exposes finish shutdown and timeout and force and begin shutdown so its caller can continue with the function\'s computed outcome. + return self.finish_shutdown(self.begin_shutdown(), timeout, force) + + # What: define coordinated_exit around stop child; why: its direct callers call coordinated_exit for coordinated exit and rely on this exact input and result contract. + def coordinated_exit(self, *, stop_child: bool) -> object | None: + """Idempotently quiesce for an OS/lifespan exit using the configured child policy.""" + # What: document idempotently quiesce for an os lifespan in the coordinated_exit docstring; why: introspection and maintainers read this exact docstring fragment to understand coordinated exit behavior without executing it. + # What: iterate across the computed value to perform owner and routing error and begin shutdown and cond and shutdown requested; why: coordinated_exit repeats the body only while or for the loop header admits an iteration. + while True: + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator.coordinated_exit routes failures to routing error while preserving cleanup and success flow. + try: + # What: compute owner from begin shutdown; why: return self finish shutdown owner later reads owner, so coordinated_exit must retain the computed value under that name. + owner = self.begin_shutdown() + # What: apply the break portion of the enclosing predicate; why: this clause remains in coordinated_exit\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: handle routing error by with self cond; why: RoutingCoordinator.coordinated_exit converts that failure into this concrete recovery, response, or cleanup behavior. + except RoutingError: + # What: enter the cond managed context before while self shutdown owner is not; why: coordinated_exit releases this resource or lock after while self shutdown owner is not on both success and failure paths. + with self._cond: + # What: iterate across shutdown owner to perform wait and cond; why: coordinated_exit repeats the body only while or for the loop header admits an iteration. + while self._shutdown_owner is not None: + # What: call self._cond.wait with the declared inputs; why: coordinated_exit invokes self._cond.wait while performing if self shutdown requested; the call advances that operation through its result or side effect. + self._cond.wait() + # What: gate on shutdown requested before the computed value; why: coordinated_exit admits the computed value only for this predicate and excludes the opposite state. + if self._shutdown_requested: + # What: return no value from coordinated_exit; why: coordinated_exit returns no value to callers that depend on its completed result. + return None + # What: gate on stop child before finish shutdown and owner; why: coordinated_exit admits finish shutdown and owner only for this predicate and excludes the opposite state. + if stop_child: + # What: return finish shutdown and owner from coordinated_exit; why: coordinated_exit exposes finish shutdown and owner so its caller can continue with the function\'s computed outcome. + # Bound lease drain to 30 seconds and child termination to 120 seconds before systemd escalates. + return self.finish_shutdown(owner, timeout=120.0, force=True, drain_timeout=30.0) + # What: return finish detach and owner from coordinated_exit; why: coordinated_exit exposes finish detach and owner so its caller can continue with the function\'s computed outcome. + # Force detach after a bounded drain because operating-system exit cannot wait indefinitely. + return self.finish_detach(owner, drain_timeout=30.0, force_drain=True) + + # What: apply staticmethod behavior to _new_timer; why: Python attaches this named decorator's registration or descriptor semantics to _new_timer. + @staticmethod + # What: define _new_timer around delay and callback; why: the registered API client call _new_timer for new timer and rely on this exact input and result contract. + def _new_timer(delay: float, callback: Callable[[], None]): + # What: compute timer from timer and delay and callback and threading; why: timer daemon later reads timer, so _new_timer must retain the computed value under that name. + timer = threading.Timer(delay, callback) + # What: compute daemon from true; why: the enclosing return or state update later reads daemon, so _new_timer must retain the computed value under that name. + timer.daemon = True + # What: return timer from _new_timer; why: _new_timer exposes timer so its caller can continue with the function\'s computed outcome. + return timer + + # What: define _cancel_idle_timer around the current object state; why: its direct callers call _cancel_idle_timer for cancel idle timer and rely on this exact input and result contract. + def _cancel_idle_timer(self) -> None: + # What: gate on idle timer before cancel and idle timer; why: _cancel_idle_timer admits cancel and idle timer only for this predicate and excludes the opposite state. + if self._idle_timer is not None: + # What: call self._idle_timer.cancel with the declared inputs; why: _cancel_idle_timer invokes self._idle_timer.cancel while performing self idle timer; the call advances that operation through its result or side effect. + self._idle_timer.cancel() + # What: compute idle timer from the named fixture input; why: the enclosing return or state update later reads idle timer, so _cancel_idle_timer must retain the computed value under that name. + self._idle_timer = None + + # What: define _schedule_idle_eviction around the current object state; why: its direct callers call _schedule_idle_eviction for schedule idle eviction and rely on this exact input and result contract. + def _schedule_idle_eviction(self) -> None: + # What: gate on active name before the computed value; why: _schedule_idle_eviction admits the computed value only for this predicate and excludes the opposite state. + if self._active_name is None: + # What: return no value from _schedule_idle_eviction; why: _schedule_idle_eviction returns no value to callers that depend on its completed result. + return + # What: compute profile from get and active name and catalog; why: ttl profile ttl s if profile ttl s is not later reads profile, so _schedule_idle_eviction must retain the computed value under that name. + profile = self._catalog.get(self._active_name) + # What: compute ttl from ttl s and default ttl s and profile and settings; why: if ttl later reads ttl, so _schedule_idle_eviction must retain the computed value under that name. + ttl = profile.ttl_s if profile.ttl_s is not None else self._catalog.settings.default_ttl_s + # What: gate on ttl before the computed value; why: _schedule_idle_eviction admits the computed value only for this predicate and excludes the opposite state. + if ttl <= 0: + # What: return no value from _schedule_idle_eviction; why: _schedule_idle_eviction returns no value to callers that depend on its completed result. + return + # What: call self._cancel_idle_timer with the declared inputs; why: _schedule_idle_eviction invokes self._cancel_idle_timer while performing timer self timer factory ttl lambda self evict idle profile name; the call advances that operation through its result or side effect. + self._cancel_idle_timer() + # What: compute timer from timer factory and ttl and evict idle and name; why: self idle timer timer later reads timer, so _schedule_idle_eviction must retain the computed value under that name. + timer = self._timer_factory(ttl, lambda: self.evict_idle(profile.name)) + # What: compute idle timer from timer; why: the enclosing return or state update later reads idle timer, so _schedule_idle_eviction must retain the computed value under that name. + self._idle_timer = timer + # What: call timer.start with the declared inputs; why: _schedule_idle_eviction invokes timer.start while performing the enclosing return; the call advances that operation through its result or side effect. + timer.start() + + # What: define _capacity_block around target; why: its direct callers call _capacity_block for capacity block and rely on this exact input and result contract. + def _capacity_block(self, target: ModelProfile) -> str | None: + """Return a capacity-policy explanation, if a swap cannot be admitted.""" + # What: document return a capacity policy explanation if a in the _capacity_block docstring; why: introspection and maintainers read this exact docstring fragment to understand capacity block behavior without executing it. + # What: gate on active name and name and target before the computed value; why: _capacity_block admits the computed value only for this predicate and excludes the opposite state. + if self._active_name is None or self._active_name == target.name: + # What: return no value from _capacity_block; why: _capacity_block returns no value to callers that depend on its completed result. + return None + # What: compute active group from group for and active name and catalog; why: if active group is not and active group persistent later reads active group, so _capacity_block must retain the computed value under that name. + active_group = self._catalog.group_for(self._active_name) + # What: compute target group from group for and name and catalog and target; why: if target group is not and target group persistent later reads target group, so _capacity_block must retain the computed value under that name. + target_group = self._catalog.group_for(target.name) + # What: gate on persistent and active group before active name; why: _capacity_block admits active name only for this predicate and excludes the opposite state. + if active_group is not None and active_group.persistent: + # What: return active name and active and profile and is and persistent from _capacity_block; why: _capacity_block exposes active name and active and profile and is and persistent so its caller can continue with the function\'s computed outcome. + return ( + # What: preserve the exact f active profile self active name r is literal fragment; why: _capacity_block passes this fragment verbatim through f"active profile {self._active_name!r} is persistent and consumes the ", because changing it would alter a protocol payload, serialized fixture, or public messa. + # What: preserve the exact single resident model slot unload it before literal fragment; why: _capacity_block passes this fragment verbatim through f"active profile {self._active_name!r} is persistent and consumes the ", because changing it would alter a protocol payload, serialized fixture, or public. + f"active profile {self._active_name!r} is persistent and consumes the " + "single resident-model slot; unload it before selecting another profile" + # What: complete the enclosing predicate with return f active profile self active name r is persistent and; why: RoutingCoordinator._capacity_block groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) + # What: gate on persistent and target group before name and target; why: _capacity_block admits name and target only for this predicate and excludes the opposite state. + if target_group is not None and target_group.persistent: + # What: return name and target and profile and requires and a from _capacity_block; why: _capacity_block exposes name and target and profile and requires and a so its caller can continue with the function\'s computed outcome. + return ( + # What: preserve the exact f profile target name r requires a literal fragment; why: _capacity_block passes this fragment verbatim through f"profile {target.name!r} requires a persistent resident slot; unload th, because changing it would alter a protocol payload, serialized fixture, or public message. + # What: preserve the exact current profile before selecting it literal fragment; why: _capacity_block passes this fragment verbatim through f"profile {target.name!r} requires a persistent resident slot; unload th, because changing it would alter a protocol payload, serialized fixture, or public message. + f"profile {target.name!r} requires a persistent resident slot; unload the " + "current profile before selecting it" + # What: complete the enclosing predicate with return f profile target name r requires a persistent resident; why: RoutingCoordinator._capacity_block groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) + # What: return no value from _capacity_block; why: _capacity_block returns no value to callers that depend on its completed result. + return None + + # What: define _remove_pending_locked around ticket and cancellation; why: its direct callers call _remove_pending_locked for remove pending locked and rely on this exact input and result contract. + def _remove_pending_locked( + # What: declare the self input for _remove_pending_locked; why: _remove_pending_locked consumes self during self pending remove ticket, so callers must bind it with the other signature inputs. + self, + # What: declare the ticket input for _remove_pending_locked; why: _remove_pending_locked consumes ticket during self pending remove ticket, so callers must bind it with the other signature inputs. + ticket: tuple[int, int, str], + # What: declare the cancellation input for _remove_pending_locked; why: _remove_pending_locked consumes cancellation during pending self pending by cancellation get cancellation if cancellation is, so callers must bind it with the other signature inputs. + cancellation: threading.Event | None, + # What: complete the enclosing predicate with bool; why: RoutingCoordinator._remove_pending_locked groups the supplied clauses as one enclosing predicate expression before its value is consumed. + ) -> bool: + """Idempotently remove one ticket and its optional progress lookup.""" + # What: document idempotently remove one ticket and its in the _remove_pending_locked docstring; why: introspection and maintainers read this exact docstring fragment to understand remove pending locked behavior without executing it. + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator._remove_pending_locked routes failures to value error while preserving cleanup and success flow. + try: + # What: call self._pending.remove with ticket; why: _remove_pending_locked invokes self._pending.remove while performing except value error; the call advances that operation through its result or side effect. + self._pending.remove(ticket) + # What: handle value error by removed false; why: RoutingCoordinator._remove_pending_locked converts that failure into this concrete recovery, response, or cleanup behavior. + except ValueError: + # What: compute removed from false; why: removed later reads removed, so _remove_pending_locked must retain the computed value under that name. + removed = False + # What: select the remaining branch that performs removed; why: _remove_pending_locked covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: compute removed from true; why: return removed later reads removed, so _remove_pending_locked must retain the computed value under that name. + removed = True + # What: compute pending from cancellation and get and pending by cancellation; why: if pending is not and pending later reads pending, so _remove_pending_locked must retain the computed value under that name. + pending = self._pending_by_cancellation.get(cancellation) if cancellation is not None else None + # What: gate on pending and ticket before pop and cancellation and pending by cancellation; why: _remove_pending_locked admits pop and cancellation and pending by cancellation only for this predicate and excludes the opposite state. + if pending is not None and pending[0] == ticket: + # What: call self._pending_by_cancellation.pop with cancellation and the named fixture input; why: _remove_pending_locked invokes self._pending_by_cancellation.pop while performing return removed; the call advances that operation through its result or side effect. + self._pending_by_cancellation.pop(cancellation, None) + # What: return removed from _remove_pending_locked; why: _remove_pending_locked exposes removed so its caller can continue with the function\'s computed outcome. + return removed + + # What: define _active_profile_ready_locked around profile; why: its direct callers call _active_profile_ready_locked for active profile ready locked and rely on this exact input and result contract. + def _active_profile_ready_locked(self, profile: ModelProfile) -> bool: + """Whether *profile* is the exact readiness-gated resident engine.""" + # What: document whether profile is the exact readiness gated in the _active_profile_ready_locked docstring; why: introspection and maintainers read this exact docstring fragment to understand active profile ready locked behavior without executing it. + # What: gate on active name and name and profile before the computed value; why: _active_profile_ready_locked admits the computed value only for this predicate and excludes the opposite state. + if self._active_name != profile.name: + # What: return false from _active_profile_ready_locked; why: _active_profile_ready_locked exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: return engine matches and profile and port for from _active_profile_ready_locked; why: _active_profile_ready_locked exposes engine matches and profile and port for so its caller can continue with the function\'s computed outcome. + return self._engine_matches(profile, self._port_for(profile)) + + # What: define _reserve_concurrency_locked around profile; why: its direct callers call _reserve_concurrency_locked for reserve concurrency locked and rely on this exact input and result contract. + def _reserve_concurrency_locked(self, profile: ModelProfile) -> None: + """Reserve active/queued capacity or reject immediately like the pinned scheduler.""" + # What: document reserve active queued capacity or reject in the _reserve_concurrency_locked docstring; why: introspection and maintainers read this exact docstring fragment to understand reserve concurrency locked behavior without executing it. + # What: compute global limit from global concurrency limit and settings and catalog; why: if global limit and self reservations global limit or later reads global limit, so _reserve_concurrency_locked must retain the computed value under that name. + global_limit = self._catalog.settings.global_concurrency_limit + # What: compute profile limit from concurrency limit and default profile concurrency limit and profile; why: if global limit and self reservations global limit or later reads profile limit, so _reserve_concurrency_locked must retain the computed value under that name. + profile_limit = profile.concurrency_limit or DEFAULT_PROFILE_CONCURRENCY_LIMIT + # What: compute profile reserved from get and name and profile reservations and profile and 0; why: if global limit and self reservations global limit or later reads profile reserved, so _reserve_concurrency_locked must retain the computed value under that name. + profile_reserved = self._profile_reservations.get(profile.name, 0) + # What: gate on global limit and profile reserved and profile limit and reservations before routing error and name and profile; why: _reserve_concurrency_locked admits routing error and name and profile only for this predicate and excludes the opposite state. + if (global_limit and self._reservations >= global_limit) or profile_reserved >= profile_limit: + # What: raise RoutingError for the caller; why: RoutingCoordinator._reserve_concurrency_locked stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: apply the concurrency limit portion of the enclosing predicate; why: this clause remains in _reserve_concurrency_locked\'s enclosing expression so its grouping and evaluation order stay intact. + "concurrency_limit", + # What: apply the f concurrency limit reached for profile portion of the enclosing predicate; why: this clause remains in _reserve_concurrency_locked\'s enclosing expression so its grouping and evaluation order stay intact. + f"concurrency limit reached for profile {profile.name!r}", + # What: supply status code to RoutingError; why: _reserve_concurrency_locked binds this 429 value to RoutingError's status code input. + status_code=429, + # What: complete the RoutingError call with status code; why: RoutingCoordinator._reserve_concurrency_locked groups the supplied clauses as one RoutingError call before its value is consumed. + ) + # What: compute reservations from 1; why: the enclosing return or state update later reads reservations, so _reserve_concurrency_locked must retain the computed value under that name. + self._reservations += 1 + # What: compute profile reservations entry from profile reserved and 1; why: the enclosing return or state update later reads profile reservations entry, so _reserve_concurrency_locked must retain the computed value under that name. + self._profile_reservations[profile.name] = profile_reserved + 1 + + # What: define _drop_concurrency_reservation_locked around profile; why: its direct callers call _drop_concurrency_reservation_locked for drop concurrency reservation locked and rely on this exact input and result contract. + def _drop_concurrency_reservation_locked(self, profile: ModelProfile) -> None: + # What: compute count from get and name and profile reservations and profile and 0; why: if self reservations or count later reads count, so _drop_concurrency_reservation_locked must retain the computed value under that name. + count = self._profile_reservations.get(profile.name, 0) + # What: gate on reservations and count before runtime error; why: _drop_concurrency_reservation_locked admits runtime error only for this predicate and excludes the opposite state. + if self._reservations <= 0 or count <= 0: + # What: raise RuntimeError for the caller; why: RoutingCoordinator._drop_concurrency_reservation_locked stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("routing concurrency reservation underflow") + # What: compute reservations from 1; why: the enclosing return or state update later reads reservations, so _drop_concurrency_reservation_locked must retain the computed value under that name. + self._reservations -= 1 + # What: gate on count before pop and name and profile reservations and profile; why: _drop_concurrency_reservation_locked admits pop and name and profile reservations and profile only for this predicate and excludes the opposite state. + if count == 1: + # What: call self._profile_reservations.pop with name and profile; why: _drop_concurrency_reservation_locked invokes self._profile_reservations.pop while performing else; the call advances that operation through its result or side effect. + self._profile_reservations.pop(profile.name) + # What: select the remaining branch that performs self profile reservations profile name count; why: _drop_concurrency_reservation_locked covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: compute profile reservations entry from count and 1; why: the enclosing return or state update later reads profile reservations entry, so _drop_concurrency_reservation_locked must retain the computed value under that name. + self._profile_reservations[profile.name] = count - 1 + + # What: define _matches_active around profile and port; why: its direct callers call _matches_active for matches active and rely on this exact input and result contract. + def _matches_active(self, profile: ModelProfile, port: int) -> bool: + # What: return active name and name and engine matches and profile from _matches_active; why: _matches_active exposes active name and name and engine matches and profile so its caller can continue with the function\'s computed outcome. + return self._active_name == profile.name and self._engine_matches(profile, port) + + # What: define _engine_matches around profile and port; why: its direct callers call _engine_matches for engine matches and rely on this exact input and result contract. + def _engine_matches(self, profile: ModelProfile, port: int) -> bool: + # What: compute state from status and manager; why: state get running later reads state, so _engine_matches must retain the computed value under that name. + state = self._manager.status() + # What: return bool and get and model and port from _engine_matches; why: _engine_matches exposes bool and get and model and port so its caller can continue with the function\'s computed outcome. + return bool( + # What: call state.get with running; why: _engine_matches invokes state.get while performing and state get model profile model; the call advances that operation through its result or side effect. + state.get("running") + # What: call state.get with model; why: _engine_matches invokes state.get while performing and state get port port; the call advances that operation through its result or side effect. + and state.get("model") == profile.model + # What: call state.get with port; why: _engine_matches invokes state.get while performing and self manager serve args list profile args; the call advances that operation through its result or side effect. + and state.get("port") == port + # What: call self._manager.serve_args with the declared inputs; why: _engine_matches consumes the self._manager.serve_args return value while evaluating and self._manager.serve_args() == list(profile.args). + and self._manager.serve_args() == list(profile.args) + # What: complete the bool call with get; why: RoutingCoordinator._engine_matches groups the supplied clauses as one bool call before its value is consumed. + ) + + # What: define _active_matches_engine_locked around the current object state; why: its direct callers call _active_matches_engine_locked for active matches engine locked and rely on this exact input and result contract. + def _active_matches_engine_locked(self) -> bool: + """Internal exact-identity check; caller holds ``self._cond``.""" + # What: document internal exact identity check caller holds self cond in the _active_matches_engine_locked docstring; why: introspection and maintainers read this exact docstring fragment to understand active matches engine locked behavior without executing it. + # What: gate on active name before the computed value; why: _active_matches_engine_locked admits the computed value only for this predicate and excludes the opposite state. + if self._active_name is None: + # What: return false from _active_matches_engine_locked; why: _active_matches_engine_locked exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: establish the handler boundary for the protected operation; why: RoutingCoordinator._active_matches_engine_locked routes failures to catalog error while preserving cleanup and success flow. + try: + # What: compute profile from get and active name and catalog; why: return self matches active profile self port for profile later reads profile, so _active_matches_engine_locked must retain the computed value under that name. + profile = self._catalog.get(self._active_name) + # What: handle catalog error by return false; why: RoutingCoordinator._active_matches_engine_locked converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError: + # What: return false from _active_matches_engine_locked; why: _active_matches_engine_locked exposes false so its caller can continue with the function\'s computed outcome. + return False + # What: return matches active and profile and port for from _active_matches_engine_locked; why: _active_matches_engine_locked exposes matches active and profile and port for so its caller can continue with the function\'s computed outcome. + return self._matches_active(profile, self._port_for(profile)) + + # What: define _port_for around profile; why: its direct callers call _port_for for port for and rely on this exact input and result contract. + def _port_for(self, profile: ModelProfile) -> int: + """Resolve a profile's proxy/readiness target under router ownership.""" + # What: document resolve a profile s proxy readiness in the _port_for docstring; why: introspection and maintainers read this exact docstring fragment to understand port for behavior without executing it. + # What: gate on port and profile before default port; why: _port_for admits default port only for this predicate and excludes the opposite state. + if profile.port is None: + # What: return default port from _port_for; why: _port_for exposes default port so its caller can continue with the function\'s computed outcome. + return self._default_port + # What: gate on port and profile before port and profile; why: _port_for admits port and profile only for this predicate and excludes the opposite state. + if profile.port != 0: + # What: return port and profile from _port_for; why: _port_for exposes port and profile so its caller can continue with the function\'s computed outcome. + return profile.port + # What: compute state from status and manager; why: if self active name profile name and state get running later reads state, so _port_for must retain the computed value under that name. + state = self._manager.status() + # A dynamic profile retains its concrete port for its whole residency; + # a fresh activation gets a new kernel-selected one. + # What: gate on active name and name and get and isinstance and int before state; why: _port_for admits state only for this predicate and excludes the opposite state. + if (self._active_name == profile.name and state.get("running") + # What: call isinstance with get and state and port and int; why: _port_for invokes isinstance while performing return state port; the call advances that operation through its result or side effect. + and isinstance(state.get("port"), int) and state["port"] > 0): + # What: return state and port from _port_for; why: _port_for exposes state and port so its caller can continue with the function\'s computed outcome. + return state["port"] + # What: return port allocator from _port_for; why: _port_for exposes port allocator so its caller can continue with the function\'s computed outcome. + return self._port_allocator() + + # What: define _activate around profile and port; why: its direct callers call _activate for activate and rely on this exact input and result contract. + def _activate(self, profile: ModelProfile, port: int) -> int | None: + # What: compute state from status and manager; why: state get running later reads state, so _activate must retain the computed value under that name. + state = self._manager.status() + # What: compute exact from get and model and port and state; why: if exact later reads exact, so _activate must retain the computed value under that name. + exact = ( + # What: call state.get with running; why: _activate invokes state.get while performing and state get model profile model; the call advances that operation through its result or side effect. + state.get("running") + # What: call state.get with model; why: _activate invokes state.get while performing and state get port port; the call advances that operation through its result or side effect. + and state.get("model") == profile.model + # What: call state.get with port; why: _activate invokes state.get while performing and self manager serve args list profile args; the call advances that operation through its result or side effect. + and state.get("port") == port + # What: call self._manager.serve_args with the declared inputs; why: _activate consumes the self._manager.serve_args return value while evaluating and self._manager.serve_args() == list(profile.args). + and self._manager.serve_args() == list(profile.args) + # What: complete the exact expression with exact state get running and state get model equals profile model and; why: RoutingCoordinator._activate groups the supplied clauses as one exact expression before its value is consumed. + ) + # What: compute ticket from the named fixture input; why: result ticket self manager switch for readiness later reads ticket, so _activate must retain the computed value under that name. + ticket = None + # What: gate on exact before result and get and state; why: _activate admits result and get and state only for this predicate and excludes the opposite state. + if exact: + # What: map the pid field as get and state and pid; why: RoutingCoordinator._activate carries pid through result into result ticket self manager switch for readiness. + result = {"pid": state.get("pid"), "idempotent": True} + # What: gate on get and state before cond and activations; why: _activate admits cond and activations only for this predicate and excludes the opposite state. + elif state.get("running"): + # What: enter the cond managed context before self activations; why: _activate releases this resource or lock after self activations on both success and failure paths. + with self._cond: + # What: compute activations from 1; why: self activations later reads activations, so _activate must retain the computed value under that name. + self._activations += 1 + # What: compute result and ticket from switch for readiness and model and port and manager; why: result self manager start profile model port list profile args later reads result and ticket, so _activate must retain the computed value under that name. + result, ticket = self._manager.switch_for_readiness( + # What: call list with args and profile; why: _activate consumes the list return value while evaluating profile.model, port, list(profile.args). + profile.model, port, list(profile.args) + # What: complete the self._manager.switch_for_readiness call with model and port and list; why: RoutingCoordinator._activate groups the supplied clauses as one self._manager.switch_for_readiness call before its value is consumed. + ) + # What: select the remaining branch that performs with self cond; why: _activate covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: enter the cond managed context before self activations; why: _activate releases this resource or lock after self activations on both success and failure paths. + with self._cond: + # What: compute activations from 1; why: the enclosing return or state update later reads activations, so _activate must retain the computed value under that name. + self._activations += 1 + # What: compute result from start and model and port and manager; why: pid result get pid later reads result, so _activate must retain the computed value under that name. + result = self._manager.start(profile.model, port, list(profile.args)) + # What: compute readiness args from port and ready timeout s and get and profile; why: readiness args path profile check endpoint later reads readiness args, so _activate must retain the computed value under that name. + readiness_args = { + # What: map the pid field as get and result and pid; why: RoutingCoordinator._activate carries pid through readiness args into readiness args path profile check endpoint. + "pid": result.get("pid"), + # What: map the port field as port; why: RoutingCoordinator._activate carries port through readiness args into readiness args path profile check endpoint. + "port": port, + # What: map the timeout s field as ready timeout s and profile; why: RoutingCoordinator._activate carries timeout s through readiness args into readiness args path profile check endpoint. + "timeout_s": profile.ready_timeout_s, + # What: complete the readiness_args mapping with pid and port and timeout s; why: RoutingCoordinator._activate groups the supplied clauses as one readiness_args mapping before its value is consumed. + } + # What: gate on check endpoint and default check endpoint and profile before check endpoint and readiness args and profile; why: _activate admits check endpoint and readiness args and profile only for this predicate and excludes the opposite state. + if profile.check_endpoint != DEFAULT_CHECK_ENDPOINT: + # What: compute readiness args entry from check endpoint and profile; why: readiness self ready fn self manager self probe readiness args later reads readiness args entry, so _activate must retain the computed value under that name. + readiness_args["path"] = profile.check_endpoint + # What: compute readiness from ready fn and manager and probe and readiness args; why: if readiness get ready later reads readiness, so _activate must retain the computed value under that name. + readiness = self._ready_fn(self._manager, self._probe, **readiness_args) + # What: gate on get and readiness before get and result; why: _activate admits get and result only for this predicate and excludes the opposite state. + if readiness.get("ready"): + # What: return get and result and pid from _activate; why: _activate exposes get and result and pid so its caller can continue with the function\'s computed outcome. + return result.get("pid") + # What: compute recovery from the named fixture input; why: recovery self manager recover switch ticket later reads recovery, so _activate must retain the computed value under that name. + recovery = None + # What: gate on ticket before recovery and recover switch and ticket and manager; why: _activate admits recovery and recover switch and ticket and manager only for this predicate and excludes the opposite state. + if ticket is not None: + # What: compute recovery from recover switch and ticket and manager; why: recovery recovery later reads recovery, so _activate must retain the computed value under that name. + recovery = self._manager.recover_switch(ticket) + # What: compute reason from get and readiness and reason and not ready; why: f profile profile name r is not later reads reason, so _activate must retain the computed value under that name. + reason = readiness.get("reason", "not-ready") + # What: raise RoutingError for the caller; why: RoutingCoordinator._activate stops this rejected path before it can mutate state, dispatch work, or report success. + raise RoutingError( + # What: apply the engine not ready portion of the enclosing predicate; why: this clause remains in _activate\'s enclosing expression so its grouping and evaluation order stay intact. + "engine_not_ready", + # What: apply the f profile profile name r is not portion of the enclosing predicate; why: this clause remains in _activate\'s enclosing expression so its grouping and evaluation order stay intact. + f"profile {profile.name!r} is not ready: {reason}", + # What: supply recovery to RoutingError; why: _activate binds this recovery value to RoutingError's recovery input. + recovery=recovery, + # What: complete the RoutingError call with recovery; why: RoutingCoordinator._activate groups the supplied clauses as one RoutingError call before its value is consumed. + ) diff --git a/python/freetoken/daemon/serve_manager.py b/python/freetoken/daemon/serve_manager.py index 681bdef497..b42d4ff288 100644 --- a/python/freetoken/daemon/serve_manager.py +++ b/python/freetoken/daemon/serve_manager.py @@ -40,12 +40,39 @@ class Conflict(RuntimeError): """A different serve (model/port/args) is already running; the client should switch().""" +# What: define SwitchLaunchError as the owner of __init__; why: daemon callers use this class boundary so those methods share one switch launch error state invariant. +class SwitchLaunchError(RuntimeError): + """Replacement failed; rollback describes launch recovery, not readiness.""" +# What: document replacement failed rollback describes launch recovery in the SwitchLaunchError docstring; why: introspection and maintainers read this exact docstring fragment to understand switch launch error behavior without executing it. + + # What: define __init__ around error and rollback and accounting; why: its direct callers call __init__ for init and rely on this exact input and result contract. + def __init__(self, error: Exception, rollback: dict, accounting: dict | None): + # What: preserve the exact super init f replacement launch failed literal fragment; why: __init__ passes this fragment verbatim through super().__init__(f"replacement launch failed: {error}"), because changing it would alter a protocol payload, serialized fixture, or public message. + super().__init__(f"replacement launch failed: {error}") + # What: compute rollback from rollback; why: the enclosing return or state update later reads rollback, so __init__ must retain the computed value under that name. + self.rollback = rollback + # What: compute accounting from accounting; why: the enclosing return or state update later reads accounting, so __init__ must retain the computed value under that name. + self.accounting = accounting + + @dataclass class ExitInfo: code: int | None # Popen convention: >=0 exit status, <0 == -signal; None if unknowable source: str # "exited" | "signalled" | "stopped" | "adopted-vanished" | "unknown" +# What: generate dataclass initialization and value semantics for SwitchRecovery; why: SwitchRecovery acts as a typed state record with consistent construction, comparison, and representation. +@dataclass(frozen=True) +# What: define SwitchRecovery as the owner of its declared state; why: daemon callers use this class boundary so those methods share one switch recovery state invariant. +class SwitchRecovery: + # What: compute epoch from the named fixture input; why: superseded self lifecycle epoch ticket epoch later reads epoch, so serve_manager must retain the computed value under that name. + epoch: int + # What: compute child from the named fixture input; why: def close popen owns its child later reads child, so serve_manager must retain the computed value under that name. + child: object + # What: compute previous from the named fixture input; why: previous self model self port list self args later reads previous, so serve_manager must retain the computed value under that name. + previous: tuple[str, int, list[str]] | None + + # --------------------------------------------------------------------------- child abstractions @@ -211,6 +238,8 @@ def __init__( # Serialize complete lifecycle transactions, including prepare -> durable receipt -> signal. # RLock lets switch() compose stop+start without opening an interleaving window. self._lifecycle = threading.RLock() + # What: compute lifecycle epoch from 0; why: the enclosing return or state update later reads lifecycle epoch, so __init__ must retain the computed value under that name. + self._lifecycle_epoch = 0 self._cond = threading.Condition(threading.Lock()) # state guarded by _cond self._child: object | None = None @@ -259,6 +288,8 @@ def start( self, model: str, port: int, args: list[str] | None = None, *, _auto: bool = False ) -> dict: with self._lifecycle: + # What: compute lifecycle epoch from 1; why: the enclosing return or state update later reads lifecycle epoch, so start must retain the computed value under that name. + self._lifecycle_epoch += 1 return self._start(model, port, args, _auto=_auto) def _start( @@ -326,26 +357,49 @@ def _start( def stop(self, timeout: float | None = None, force: bool = False) -> dict: with self._lifecycle: + # What: compute lifecycle epoch from 1; why: the enclosing return or state update later reads lifecycle epoch, so stop must retain the computed value under that name. + self._lifecycle_epoch += 1 return self._stop(timeout, force) def shutdown(self, timeout: float | None = None, force: bool = False) -> dict: - """Permanently close admission to new serves, then stop the current child. - - The latch and stop share one lifecycle transaction. A start already ahead of us is - included in the stop; every start queued behind us observes the latch and is rejected. - If accounting/signalling fails, the daemon remains up and normal lifecycle calls reopen. - """ - with self._lifecycle: + """Permanently close admission and stop the child within one bounded deadline.""" + # Convert the relative timeout once so lifecycle-lock waiting cannot extend the total stop bound. + deadline = None if timeout is None else time.monotonic() + max(0.0, timeout) + # Block indefinitely only for legacy callers that intentionally omitted a timeout. + if deadline is None: + acquired = self._lifecycle.acquire() + else: + # Bound lock acquisition because an in-flight start or switch can otherwise stall system shutdown forever. + acquired = self._lifecycle.acquire(timeout=max(0.0, deadline - time.monotonic())) + # Fail explicitly when another lifecycle operation consumed the complete shutdown deadline. + if not acquired: + raise TimeoutError("timed out waiting for serve lifecycle ownership during shutdown") + try: + # Advance the lifecycle generation so queued recovery tickets cannot revive the departing child. + self._lifecycle_epoch += 1 + # Latch manager shutdown before signalling so starts queued behind this transaction fail closed. with self._cond: + # Mark shutdown requested while the same lifecycle owner still controls the child transition. self._shutdown_requested = True + # Wake waiters so they can observe and reject against the terminal latch. self._cond.notify_all() try: - return self._stop(timeout, force) + # Pass only the remaining total budget to child accounting, signalling, and reap waiting. + remaining = None if deadline is None else max(0.0, deadline - time.monotonic()) + # Stop the exact managed child without allowing lock wait and stop wait to each consume a full timeout. + return self._stop(remaining, force) except Exception: + # Reopen admission only when the terminal child action failed and the daemon remains alive. with self._cond: + # Clear the manager shutdown latch so a later explicit retry can own lifecycle safely. self._shutdown_requested = False + # Wake blocked callers so they observe the rollback immediately. self._cond.notify_all() + # Preserve the actionable accounting, signal, or timeout failure for the caller. raise + finally: + # Release lifecycle ownership on every path so a failed shutdown cannot deadlock later cleanup. + self._lifecycle.release() def _stop(self, timeout: float | None = None, force: bool = False) -> dict: grace = self._grace_s if timeout is None else timeout @@ -409,10 +463,132 @@ def switch( force: bool = False, ) -> dict: with self._lifecycle: + # What: compute lifecycle epoch from 1; why: the enclosing return or state update later reads lifecycle epoch, so switch must retain the computed value under that name. + self._lifecycle_epoch += 1 + # What: enter the cond managed context before previous self model self port list self args; why: switch releases this resource or lock after previous self model self port list self args on both success and failure paths. + with self._cond: + # What: compute previous from model and port and child and stopping; why: can restore self child is and previous is later reads previous, so switch must retain the computed value under that name. + previous = ((self._model, self._port, list(self._args)) + # What: apply the if self child is not and not portion of previous; why: switch uses this clause to evaluate previous as one grouped value. + if self._child is not None and not self._stopping else None) stopped = self._stop(force=force) - started = self._start(model, port, args) + # What: establish the handler boundary for the protected operation; why: ServeManager.switch routes failures to exception while preserving cleanup and success flow. + try: + # What: compute started from start and model and port and args; why: return started accounting stopped accounting later reads started, so switch must retain the computed value under that name. + started = self._start(model, port, args) + # What: handle exception by rollback attempted false launched false; why: ServeManager.switch converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: + # What: map the attempted field as false; why: ServeManager.switch carries attempted through rollback into rollback attempted true. + rollback = {"attempted": False, "launched": False} + # A post-spawn failure may leave an owned child. Never spawn a + # second engine or bypass accounting to remove that child. + # What: enter the cond managed context before can restore self child is and previous is; why: switch releases this resource or lock after can restore self child is and previous is on both success and failure paths. + with self._cond: + # What: compute can restore from child and previous; why: if can restore later reads can restore, so switch must retain the computed value under that name. + can_restore = self._child is None and previous is not None + # What: gate on can restore before rollback; why: switch admits rollback only for this predicate and excludes the opposite state. + if can_restore: + # What: compute rollback entry from true; why: rollback update launched pid restored pid later reads rollback entry, so switch must retain the computed value under that name. + rollback["attempted"] = True + # What: establish the handler boundary for the protected operation; why: ServeManager.switch routes failures to exception while preserving cleanup and success flow. + try: + # What: compute restored from start and previous; why: rollback update launched pid restored pid later reads restored, so switch must retain the computed value under that name. + restored = self._start(*previous) + # What: preserve the exact rollback update launched pid restored pid literal fragment; why: switch passes this fragment verbatim through rollback.update(launched=True, pid=restored["pid"]), because changing it would alter a protocol payload, serialized fixture, or public message. + rollback.update(launched=True, pid=restored["pid"]) + # What: preserve the exact self emit replacement launch failed previous engine literal fragment; why: switch passes this fragment verbatim through self._emit("replacement launch failed. + self._emit("replacement launch failed; previous engine relaunched") + # What: handle exception by rollback error str recovery exc; why: ServeManager.switch converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as recovery_exc: + # What: compute rollback entry from str and recovery exc; why: self emit f replacement launch rollback failed later reads rollback entry, so switch must retain the computed value under that name. + rollback["error"] = str(recovery_exc) + # What: preserve the exact self emit f replacement launch rollback failed literal fragment; why: switch passes this fragment verbatim through self._emit(f"replacement launch rollback failed: {recovery_exc}"), because changing it would alter a protocol payload, serialized fixture, or public mess. + self._emit(f"replacement launch rollback failed: {recovery_exc}") + # What: raise SwitchLaunchError for the caller; why: ServeManager.switch stops this rejected path before it can mutate state, dispatch work, or report success. + raise SwitchLaunchError(exc, rollback, stopped["accounting"]) from exc return {**started, "accounting": stopped["accounting"]} + # What: define switch_for_readiness around model and port and args and force; why: its direct callers call switch_for_readiness for switch for readiness and rely on this exact input and result contract. + def switch_for_readiness(self, model, port, args=None, force=False): + """Capture a recovery ticket atomically; never hold the lock during HTTP probes.""" + # What: document capture a recovery ticket atomically never in the switch_for_readiness docstring; why: introspection and maintainers read this exact docstring fragment to understand switch for readiness behavior without executing it. + # What: enter the lifecycle managed context before with self cond; why: switch_for_readiness releases this resource or lock after with self cond on both success and failure paths. + with self._lifecycle: + # What: enter the cond managed context before previous self model self port list self args; why: switch_for_readiness releases this resource or lock after previous self model self port list self args on both success and failure paths. + with self._cond: + # What: compute previous from model and port and child and stopping; why: ticket switch recovery self lifecycle epoch self child previous later reads previous, so switch_for_readiness must retain the computed value under that name. + previous = ((self._model, self._port, list(self._args)) + # What: apply the if self child is not and not portion of previous; why: switch_for_readiness uses this clause to evaluate previous as one grouped value. + if self._child is not None and not self._stopping else None) + # What: compute result from switch and model and port and args; why: return result ticket later reads result, so switch_for_readiness must retain the computed value under that name. + result = self.switch(model, port, args, force) + # What: enter the cond managed context before ticket switch recovery self lifecycle epoch self child previous; why: switch_for_readiness releases this resource or lock after ticket switch recovery self lifecycle epoch self child previous on both success and failure paths. + with self._cond: + # What: compute ticket from switch recovery and lifecycle epoch and child and previous; why: return result ticket later reads ticket, so switch_for_readiness must retain the computed value under that name. + ticket = SwitchRecovery(self._lifecycle_epoch, self._child, previous) + # What: return result and ticket from switch_for_readiness; why: switch_for_readiness exposes result and ticket so its caller can continue with the function\'s computed outcome. + return result, ticket + + # What: define recover_switch around ticket and force; why: its direct callers call recover_switch for recover switch and rely on this exact input and result contract. + def recover_switch(self, ticket: SwitchRecovery, force=False): + """Recover only this switch, without overriding newer lifecycle intent. + + All stop/accounting safeguards still apply. A failed readiness check is + not permission to force-kill an engine or discard its accounting. + """ + # What: document recover only this switch without overriding in the recover_switch docstring; why: introspection and maintainers read this exact docstring fragment to understand recover switch behavior without executing it. + # What: document all stop accounting safeguards still apply in the recover_switch docstring; why: introspection and maintainers read this exact docstring fragment to understand recover switch behavior without executing it. + # What: document not permission to force kill an engine in the recover_switch docstring; why: introspection and maintainers read this exact docstring fragment to understand recover switch behavior without executing it. + # What: preserve the paragraph boundary in the the recover_switch docstring; why: introspection and maintainers read this paragraph break to understand recover switch behavior without executing it. + # What: enter the lifecycle managed context before with self cond; why: recover_switch releases this resource or lock after with self cond on both success and failure paths. + with self._lifecycle: + # What: enter the cond managed context before superseded self lifecycle epoch ticket epoch; why: recover_switch releases this resource or lock after superseded self lifecycle epoch ticket epoch on both success and failure paths. + with self._cond: + # What: compute superseded from shutdown requested and lifecycle epoch and epoch and ticket; why: if superseded later reads superseded, so recover_switch must retain the computed value under that name. + superseded = (self._lifecycle_epoch != ticket.epoch + # What: apply the or self shutdown requested portion of superseded; why: recover_switch uses this clause to evaluate superseded as one grouped value. + or self._shutdown_requested + # What: apply the or self child is not and self child portion of superseded; why: recover_switch uses this clause to evaluate superseded as one grouped value. + or (self._child is not None and self._child is not ticket.child)) + # What: compute reaping from child and child and ticket; why: if reaping and not ticket child reaped wait self reap wait s later reads reaping, so recover_switch must retain the computed value under that name. + reaping = self._child is None and ticket.child is not None + # What: gate on superseded before the computed value; why: recover_switch admits the computed value only for this predicate and excludes the opposite state. + if superseded: + # What: map the attempted field as false; why: ServeManager.recover_switch carries attempted into return {"attempted": False, "launched": False, "reason": "superseded"}. + return {"attempted": False, "launched": False, "reason": "superseded"} + # What: gate on previous and ticket before the computed value; why: recover_switch admits the computed value only for this predicate and excludes the opposite state. + if ticket.previous is None: + # What: map the attempted field as false; why: ServeManager.recover_switch carries attempted into return {"attempted": False, "launched": False, "reason": "no-previous-en. + return {"attempted": False, "launched": False, "reason": "no-previous-engine"} + # What: compute lifecycle epoch from 1; why: the enclosing return or state update later reads lifecycle epoch, so recover_switch must retain the computed value under that name. + self._lifecycle_epoch += 1 # consume ticket before any fallible operation + # What: establish the handler boundary for the protected operation; why: ServeManager.recover_switch routes failures to exception while preserving cleanup and success flow. + try: + # The monitor clears _child before clearing its persisted state. + # Wait for that cleanup so it cannot erase the restored pidfile. + # What: gate on reaping and wait and reap wait s and reaped and child before runtime error; why: recover_switch admits runtime error only for this predicate and excludes the opposite state. + if reaping and not ticket.child.reaped.wait(self._reap_wait_s): + # What: raise RuntimeError for the caller; why: ServeManager.recover_switch stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("replacement exit cleanup has not completed") + # What: compute stopped from stop and force; why: port ticket previous accounting stopped accounting later reads stopped, so recover_switch must retain the computed value under that name. + stopped = self._stop(force=force) + # What: compute restored from start and previous and ticket; why: return attempted launched pid restored pid later reads restored, so recover_switch must retain the computed value under that name. + restored = self._start(*ticket.previous) + # What: handle exception by self emit f readiness rollback failed exc; why: ServeManager.recover_switch converts that failure into this concrete recovery, response, or cleanup behavior. + except Exception as exc: + # What: preserve the exact self emit f readiness rollback failed exc literal fragment; why: recover_switch passes this fragment verbatim through self._emit(f"readiness rollback failed: {exc}"), because changing it would alter a protocol payload, serialized fixture, or public message. + self._emit(f"readiness rollback failed: {exc}") + # What: map the attempted field as true; why: ServeManager.recover_switch carries attempted into return {"attempted": True, "launched": False, "error": str(exc). + return {"attempted": True, "launched": False, "error": str(exc), + # What: map the engine preserved field as current pid; why: ServeManager.recover_switch carries engine preserved into "enginePreserved": self.current_pid() is not None}. + "enginePreserved": self.current_pid() is not None} + # What: preserve the exact self emit replacement readiness failed previous engine literal fragment; why: recover_switch passes this fragment verbatim through self._emit("replacement readiness failed. + self._emit("replacement readiness failed; previous engine relaunched") + # What: map the attempted field as true; why: ServeManager.recover_switch carries attempted into return {"attempted": True, "launched": True, "pid": restored["pid"]. + return {"attempted": True, "launched": True, "pid": restored["pid"], + # What: map the port field as previous and ticket and 1; why: ServeManager.recover_switch carries port into "port": ticket.previous[1], "accounting": stopped["accounting"]}. + "port": ticket.previous[1], "accounting": stopped["accounting"]} + def pending_accounting(self) -> list[dict[str, Any]]: return self._accounting.pending() @@ -783,6 +959,18 @@ def _reap(self, child, info: ExitInfo) -> None: with self._cond: self._stop_requested = True + # What: enter the cond managed context before is current self child is child; why: _reap releases this resource or lock after is current self child is child on both success and failure paths. + with self._cond: + # What: compute is current from child and child; why: if is current later reads is current, so _reap must retain the computed value under that name. + is_current = self._child is child + # Clear durable adoption state before publishing the stopped state. Otherwise callers + # can observe running=false and still find a dead pidfile long enough to attempt an + # invalid re-adoption or a conflicting recovery. + # What: gate on is current before clear and store; why: _reap admits clear and store only for this predicate and excludes the opposite state. + if is_current: + # What: call self._store.clear with the declared inputs; why: _reap invokes self._store.clear while performing with self cond; the call advances that operation through its result or side effect. + self._store.clear() + with self._cond: is_current = self._child is child if is_current: @@ -793,11 +981,8 @@ def _reap(self, child, info: ExitInfo) -> None: info = ExitInfo(info.code, "stopped") self._last_exit = info self._cond.notify_all() - # Outside the lock. Clear the persisted state BEFORE waking stop() waiters, so a caller - # that sees stop() return also sees an empty pidfile — no window where a racing re-adopt - # could latch onto the just-killed pid. - if is_current: - self._store.clear() + # The pidfile was cleared before publishing stopped state, so a caller that sees either + # status.running=false or stop() return cannot re-adopt this dead generation. child.reaped.set() if getattr(child, "tailer", None) is not None: try: diff --git a/python/freetoken/daemon/server.py b/python/freetoken/daemon/server.py index d4e742b0be..7219d08c5d 100644 --- a/python/freetoken/daemon/server.py +++ b/python/freetoken/daemon/server.py @@ -24,6 +24,22 @@ DEFAULT_SERVE_PORT = 1919 +def _is_loopback_host(host: str) -> bool: + # Normalize the bind name so capitalization and surrounding whitespace do not alter policy. + normalized = host.strip().lower() + # Accept only explicit loopback spellings because wildcard and interface addresses expose the API. + return normalized in {"127.0.0.1", "::1", "localhost"} + + +def _catalog_has_release_authentication(catalog, token: str | None) -> bool: + # Ignore the management-only token here because it does not protect proxied inference routes. + _ = token + # Snapshot inference API keys so startup validates one consistent catalog state. + keys = tuple(catalog.settings.api_keys) + # Reject empty and shipped-template credentials because neither safely protects a network bind. + return bool(keys) and all(key.strip() and key != "REPLACE_WITH_A_GENERATED_SECRET" for key in keys) + + def _default_state_dir() -> str: env = os.environ.get("FREETOKEN_DAEMON_DIR") if env: @@ -50,6 +66,12 @@ def _build_parser(prog: str) -> argparse.ArgumentParser: p.add_argument("--state-dir", default=_default_state_dir(), help="Lock/pidfile/log directory") p.add_argument("--token", default=os.environ.get("FREETOKEN_DAEMON_TOKEN"), help="Optional X-FT-Token shared secret") p.add_argument("--default-serve-port", type=int, default=DEFAULT_SERVE_PORT, help="Port used when /engine/start omits one") + # What: preserve the exact p add argument catalog default os environ get freetoken swap catalog help literal frag; why: _build_parser passes this fragment verbatim through p.add_argument("--catalog", default=os.environ.get("FREETOKEN_SWAP_CATAL, because changing it would alter a protocol payload, serialized fixtur. + p.add_argument("--catalog", default=os.environ.get("FREETOKEN_SWAP_CATALOG"), help="TOML named-model catalog (or $FREETOKEN_SWAP_CATALOG)") + # What: preserve the exact p add argument catalog watch interval type float default literal fragment; why: _build_parser passes this fragment verbatim through p.add_argument("--catalog-watch-interval", type=float, default=1.0, because changing it would alter a protocol payload, serialized fixture, or public messag. + p.add_argument("--catalog-watch-interval", type=float, default=1.0, + # What: preserve the catalog-watch interval help text; why: _build_parser presents this wording for the delay separating safe catalog change checks. + help="Seconds between safe catalog change checks; 0 disables watching (default 1)") p.add_argument("--serve-python", default=sys.executable, help="Interpreter used to launch ft serve") p.add_argument("--grace", type=float, default=10.0, help="SIGTERM→SIGKILL grace seconds on stop") p.add_argument("--poll-interval", type=float, default=1.0, help="Adopted-serve liveness / OOM reapply interval") @@ -103,16 +125,26 @@ def _run() -> None: def main(argv: Sequence[str] | None = None, *, prog: str = "ft daemon") -> int: args = _build_parser(prog).parse_args(list(argv) if argv is not None else None) + # What: gate on catalog watch interval and args before print and stderr and sys; why: main admits print and stderr and sys only for this predicate and excludes the opposite state. + if args.catalog_watch_interval < 0: + # What: preserve the exact print ft daemon catalog watch interval must be literal fragment; why: main passes this fragment verbatim through print("ft daemon: --catalog-watch-interval must be non-negative", file=s, because changing it would alter a protocol payload, serialized fixture, or public message. + print("ft daemon: --catalog-watch-interval must be non-negative", file=sys.stderr) + # What: return 2 from main; why: main exposes 2 so its caller can continue with the function\'s computed outcome. + return 2 logging.basicConfig( level=getattr(logging, args.log_level.upper(), logging.INFO), format="%(asctime)s [ft-daemon] %(levelname)s %(message)s", ) from .checkpoint import CheckpointManager + # What: import catalog error and model catalog for main using catalog and catalog error and model catalog; why: main uses the catalog error annotation in main and model catalog load, making that imported dependency available to its named operation. + from .catalog import CatalogError, ModelCatalog from .logring import LogRing from .metrics import FootprintCache from .pidfile import AlreadyRunning, ServeStateStore, SingleInstance from .proxy import ServeProbe + # What: import routing coordinator for main using router and routing coordinator; why: main uses routing coordinator, making that imported dependency available to its named operation. + from .router import RoutingCoordinator from .serve_manager import ServeManager from .tailer import LogTailer @@ -120,6 +152,27 @@ def main(argv: Sequence[str] | None = None, *, prog: str = "ft daemon") -> int: log_dir = os.path.join(state_dir, "logs") os.makedirs(log_dir, exist_ok=True) + # What: establish the handler boundary for the protected operation; why: main routes failures to catalog error while preserving cleanup and success flow. + try: + # What: compute catalog from catalog and args and load and empty; why: print f ft daemon invalid model later reads catalog, so main must retain the computed value under that name. + catalog = ModelCatalog.load(args.catalog) if args.catalog else ModelCatalog.empty() + # What: handle catalog error by print f ft daemon invalid model catalog; why: main converts that failure into this concrete recovery, response, or cleanup behavior. + except CatalogError as exc: + # What: preserve the exact print f ft daemon invalid model literal fragment; why: main passes this fragment verbatim through print(f"ft daemon: invalid model catalog: {exc}", file=sys.stderr), because changing it would alter a protocol payload, serialized fixture, or public message. + print(f"ft daemon: invalid model catalog: {exc}", file=sys.stderr) + # What: return 2 from main; why: main exposes 2 so its caller can continue with the function\'s computed outcome. + return 2 + + # Require real authentication before listening beyond loopback so a catalog mistake cannot expose management APIs. + if not _is_loopback_host(args.host) and not _catalog_has_release_authentication(catalog, args.token): + # Explain the refusal so operators can repair the bind or credential without inspecting source. + print( + "ft daemon: non-loopback --host requires non-placeholder router.api_keys", + file=sys.stderr, + ) + # Return argparse's configuration-error status because the daemon intentionally did not start. + return 2 + # The ONE hard refusal: two daemons cannot co-own one engine. Everything else degrades. lock = SingleInstance(os.path.join(state_dir, "daemon.pid")) try: @@ -166,6 +219,13 @@ def tailer_factory(child): except Exception as exc: # noqa: BLE001 logger.warning("re-adoption skipped: %s", exc) + # What: compute router from routing coordinator and manager and catalog and probe; why: router coordinated exit stop child later reads router, so main must retain the computed value under that name. + router = RoutingCoordinator( + # What: supply default port to RoutingCoordinator; why: main binds this default serve port and args value to RoutingCoordinator's default port input. + manager, catalog, probe, default_port=args.default_serve_port + # What: complete the RoutingCoordinator call with default port; why: main groups the supplied clauses as one RoutingCoordinator call before its value is consumed. + ) + stop_reaper = threading.Event() if not args.no_oom: _start_oom_reaper(manager, args.poll_interval, stop_reaper) @@ -177,11 +237,13 @@ def shutdown_hook() -> None: stop_reaper.set() if args.stop_serve_on_exit: logger.info("stopping serve on daemon exit (--stop-serve-on-exit)") - manager.stop() + # What: supply stop child to router.coordinated_exit; why: shutdown_hook binds this true value to router.coordinated_exit's stop child input. + router.coordinated_exit(stop_child=True) else: # Default: the engine outlives the daemon. Leave it running and # persisted so the next daemon re-adopts it; just stop following its log. - manager.detach() + # What: supply stop child to router.coordinated_exit; why: shutdown_hook binds this false value to router.coordinated_exit's stop child input. + router.coordinated_exit(stop_child=False) from .app import build_app @@ -197,6 +259,16 @@ def shutdown_hook() -> None: checkpoints=checkpoints, started_wall=time.time(), shutdown_hook=shutdown_hook, + # What: supply catalog to build_app; why: main binds this catalog value to build_app's catalog input. + catalog=catalog, + # What: supply router to build_app; why: main binds this router value to build_app's router input. + router=router, + # What: supply catalog path to build_app; why: main binds this catalog and args value to build_app's catalog path input. + catalog_path=args.catalog, + # What: supply catalog watch interval s to build_app; why: main binds this catalog and catalog watch interval and args and 0 value to build_app's catalog watch interval s input. + catalog_watch_interval_s=args.catalog_watch_interval if args.catalog else 0, + # What: supply activity path to os.path.join; why: main binds this join and state dir and path and os and activity value to os.path.join's activity path input. + activity_path=os.path.join(state_dir, "activity.jsonl"), ) import uvicorn diff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.py index 543012f396..bcbe6bcf2c 100644 --- a/python/freetoken/engine/config.py +++ b/python/freetoken/engine/config.py @@ -23,6 +23,8 @@ class EngineConfig: moe_backend: str = "auto" # NVFP4 routed-expert GEMM backend (--nvfp4-backend): auto|marlin|flashinfer|triton. nvfp4_backend: str = "triton" + # PLE table backend: "disk" (default) reads rows from the checkpoint files per fill, "pinned" preloads the table into page-locked host RAM. + ple_backend: str = "disk" # Expert-bank host load (--expert-load): auto|serial|parallel. "auto" reads scattered # experts in parallel but falls back to serial when free RAM can't cover the banks + the # parallel reader's extra (non-reclaimable) whole-shard buffer; "serial" forces the diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py index cd6505d2d1..2f46f47ad3 100644 --- a/python/freetoken/engine/engine.py +++ b/python/freetoken/engine/engine.py @@ -114,9 +114,7 @@ def _backend_requirements_met(name: str) -> bool: return True -def _resolve_auto_attention_backend( - required: frozenset[AttnType], hybrid_linear: bool -) -> str: +def _resolve_auto_attention_backend(required: frozenset[AttnType]) -> str: """First candidate (in per-type priority order) whose arch condition holds, whose packages are installed, and whose every comma part serves ALL required types. Reproduces the historical hardware tree for FULL-only models: @@ -128,6 +126,8 @@ def _resolve_auto_attention_backend( candidates.append(("dsa", True)) if AttnType.BSA in required: candidates.append(("m3_sparse", True)) + if AttnType.QSA in required: + candidates.append(("qsa_sparse", True)) if AttnType.SWA in required: candidates.append(("triton", True)) if AttnType.FULL in required: @@ -142,10 +142,6 @@ def _resolve_auto_attention_backend( continue if not _backend_parts_serve(name, required): continue - if hybrid_linear and not all( - attention_backend_info(p).hybrid_linear_ok for p in name.split(",") - ): - continue if not _backend_requirements_met(name): continue return name @@ -176,7 +172,10 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Att if missing: valid = [ name - for name in ("fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse", "m3_sparse") + for name in ( + "fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse", "m3_sparse", + "qsa_sparse", + ) if required <= attention_backend_info(name).supported_types ] missing_names = "/".join(sorted(t.value for t in missing)) @@ -185,11 +184,6 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Att f"attention, which backend {part!r} does not support; valid backends: " f"{', '.join(valid)} (or auto), got {config.attention_backend!r}." ) - if getattr(model_config, "has_linear_attention", False) and not info.hybrid_linear_ok: - raise ValueError( - f"backend {part!r} does not support hybrid-linear (GDN/mamba) models, " - f"got {config.attention_backend!r}." - ) if AttnType.SWA in required and not info.consumes_attn_spec: # SWA models drive window/sinks/sm_scale through the per-call AttentionSpec; # a backend that drops it would attend with the wrong window silently. @@ -222,13 +216,19 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Att "Use --attention-backend fi (or triton) instead." ) - if required & {AttnType.MLA, AttnType.DSA} and config.page_size != 1: - # The MLA backend's row addressing (latent scatter, DSA index keys, sparse - # top-k page indices) assumes page_size == 1 throughout; reject explicitly - # like the SWA models do rather than corrupting addressing silently. - raise ValueError( - f"latent-KV MLA models require --page-size 1, got {config.page_size}." + if required & {AttnType.MLA, AttnType.DSA}: + # Plain MLA/DSA runs on page_size 1; the kpool indexer layout needs 64. + _kpool_ratio = max( + (s.index_ratio for s in model_config.kv_cache_group_specs() if s.mla), + default=1, ) + want_page = 64 if _kpool_ratio > 1 else 1 + if config.page_size != want_page: + logger.warning_rank0( + f"Page size {config.page_size} is auto-adjusted to {want_page} " + f"for latent-KV attention." + ) + override("page_size", want_page) for part in backend_parts: info = attention_backend_info(part) @@ -333,6 +333,12 @@ def __init__(self, config: EngineConfig): self._post_weights_free = post_weights_free self.moe_offload_cache = None self.cpu_moe_executor = None + # Host-side auxiliary stores (qwen4_exp's pinned PLE table): after the weights so a + # load failure is not masked, before the MoE offload cache so the bank residency + # planning sees the pin quota the table already spent. + self._host_tables_bytes = 0 + if hasattr(self.model, "load_host_tables"): + self._host_tables_bytes = int(self.model.load_host_tables(config) or 0) if is_offload_moe_backend(config.moe_backend): self._init_offload_moe_cache(config) if hasattr(self.model, "prepare_for_runtime"): @@ -361,6 +367,7 @@ def __init__(self, config: EngineConfig): dtype=self.dtype, device=self.device, tp_size=config.tp_info.size, + slot_states=config.model_config.slot_states, ) self.ctx.linear_state_pool = self.linear_state_pool else: @@ -422,7 +429,22 @@ def __init__(self, config: EngineConfig): ) if config.attention_backend.split(",")[0] == "triton": # Prefill runs on the first comma part; warm its autotune cache. - self._warmup_prefill() + # ROCm's HIP graph and large-prompt warmup path is exercised by the first + # real request just like CUDA. Do not force that optional precompile on + # HIP at server construction: current AMD Triton releases can reject the + # synthetic 80/128-token NVFP4 MoE launch before the API becomes ready. + # Inference itself remains native HIP and eager prefill still compiles on + # demand. Operators may set this explicit opt-in for targeted testing. + should_warmup_prefill = torch.version.hip is None or os.environ.get( + "FREETOKEN_ROCM_PREFILL_WARMUP", "" + ).lower() in ("1", "true", "yes", "on") + if should_warmup_prefill: + self._warmup_prefill() + else: + logger.info_rank0( + "Skipping optional Triton prefill warmup on ROCm; " + "set FREETOKEN_ROCM_PREFILL_WARMUP=1 to enable it." + ) def _init_communication(self, config: EngineConfig) -> torch.distributed.ProcessGroup: if config.tp_info.size == 1 or config.use_pynccl: @@ -518,9 +540,11 @@ def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache: not cpu_layer_ids and config.moe_cpu_layers is None and config.moe_backend in ("offload", "hybrid") - and _pin_budget_bytes() is not None + and _pin_budget_bytes(self._host_tables_bytes) is not None ): - cpu_layer_ids = _auto_cpu_layers(config, config.model_config.num_moe_layers) + cpu_layer_ids = _auto_cpu_layers( + config, config.model_config.num_moe_layers, reserved=self._host_tables_bytes + ) if config.moe_backend == "hybrid": decode_target = "hybrid" elif cpu_layer_ids: @@ -533,13 +557,13 @@ def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache: split_residency = ( bool(cpu_layer_ids) and config.moe_backend in ("offload", "hybrid") - and _pin_budget_bytes() is not None + and _pin_budget_bytes(self._host_tables_bytes) is not None ) if config.moe_backend == "cpu" and not split_residency: # cpu mode pins every bank for the prefill double buffer; over the pin cap that dies in cudaHostRegister, so lock everything instead from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes - budget = _pin_budget_bytes() + budget = _pin_budget_bytes(self._host_tables_bytes) bank_bytes = None if budget is not None: bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config) @@ -618,6 +642,33 @@ def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache: cache.cpu_layer_ids = cpu_layer_ids cache.set_bank_sources(banks.sources, layer_residency=banks.layer_residency) cache.set_alphas(banks.gate_up_alpha, banks.down_alpha) + auxiliary_cache = None + if banks.auxiliary_sources is not None: + if decode_target != "gpu": + raise NotImplementedError( + "Qwen GGUF Q6_K down layers currently support only GPU offload decode" + ) + if config.moe_prefill_overlap: + raise NotImplementedError( + "Qwen GGUF Q6_K down layers require --disable-moe-prefill-overlap" + ) + if not banks.auxiliary_layer_ids: + raise ValueError("auxiliary expert banks are missing their model-layer mapping") + # Each exceptional Qwen layer contains all experts, so a 256-slot + # cache makes its prefill bank a direct expert-id mapping and also + # avoids reloading a Q6_K row after its first decode use. + auxiliary_cache = OffloadMoeCache( + num_layers=len(banks.auxiliary_layer_ids), + num_experts=config.model_config.num_experts, + cache_size=config.model_config.num_experts, + device=self.device, + cache_policy=config.moe_cache_policy, + prefill_overlap=False, + prefill_hit_d2d=False, + quant_format=banks.auxiliary_quant_format, + decode_target="gpu", + ) + auxiliary_cache.set_bank_sources(banks.auxiliary_sources) else: cache = cache_factory(config, self.device) cache.decode_target = decode_target @@ -632,6 +683,18 @@ def _init_offload_moe_cache(self, config: EngineConfig) -> OffloadMoeCache: # _iter_offload_moe_layers() hook when its MoE blocks are bespoke nn.Modules (DSV4). layers = attach_offload_moe_cache(self.model, cache) assert len(layers) == config.model_config.num_moe_layers + if cache_factory is None and auxiliary_cache is not None: + layer_to_auxiliary = { + layer_id: index for index, layer_id in enumerate(banks.auxiliary_layer_ids) + } + for layer in layers: + auxiliary_layer_id = layer_to_auxiliary.get(layer.layer_id) + if auxiliary_layer_id is not None: + layer.auxiliary_offload_cache = auxiliary_cache + layer.auxiliary_layer_id = auxiliary_layer_id + # Keep an ownership reference for diagnostics and future cache rebuild + # work. The main cache remains the scheduler's authoritative cache. + cache.auxiliary_caches = [auxiliary_cache] if cache.decode_target in ("cpu", "hybrid"): self._init_cpu_moe_executor(config, cache, layers) self.ctx.moe_offload_cache = cache @@ -914,11 +977,9 @@ def rebuild_runtime_cache( def forward_batch(self, batch: Batch, args: BatchSamplingArgs) -> ForwardOutput: assert torch.cuda.current_stream() == self.stream - with self.ctx.forward_batch(batch): - if self.graph_runner.can_use_cuda_graph(batch): - logits = self.graph_runner.replay(batch) - else: - logits = self.model.forward() + use_graph = self.graph_runner.can_use_cuda_graph(batch) + with self.ctx.forward_batch(batch), self.model.forward_host_ctx(batch, use_graph): + logits = self.graph_runner.replay(batch) if use_graph else self.model.forward() if self.cpu_moe_executor is not None: # One pinned read: surfaces a fired flag-handshake watchdog (dead coordinator # -> stale expert outputs) as a loud error instead of silent corruption. @@ -1134,6 +1195,7 @@ def _resolve_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[ # expert activations the CPU MoE executor supports (csrc ActKind) _CPU_MOE_ACTS = ( "silu", "swish", "gelu", "gelu_tanh", "gelu_pytorch_tanh", "swigluoai", + "swiglu_clamp", ) @@ -1158,18 +1220,20 @@ def _cpu_moe_executor_viable(model_config) -> bool: return fmt == "mxfp4" or fmt in _WFMT_IDS -def _pin_budget_bytes() -> int | None: - """Bytes this process can safely cudaHostRegister, or None when the platform does not cap pinning (plain Linux). +def _pin_budget_bytes(reserved: int = 0) -> int | None: + """Bytes this process can still safely cudaHostRegister, or None when the platform does not cap pinning (plain Linux). - WSL's WDDM-backed CUDA caps pinning near half of RAM, shared across processes -- budget 40%. FREETOKEN_PIN_BUDGET_GB overrides anywhere.""" + WSL's WDDM-backed CUDA caps pinning near half of RAM, shared across processes -- budget 40%. FREETOKEN_PIN_BUDGET_GB overrides anywhere. ``reserved`` subtracts host bytes already pinned outside the expert banks (qwen4_exp's PLE table).""" if env := os.environ.get("FREETOKEN_PIN_BUDGET_GB"): - return int(float(env) * 2**30) - if not hasattr(os, "uname") or "microsoft" not in os.uname().release.lower(): # WSL kernel tag + cap = int(float(env) * 2**30) + elif not hasattr(os, "uname") or "microsoft" not in os.uname().release.lower(): # WSL kernel tag return None - return int(os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") * 0.4) + else: + cap = int(os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") * 0.4) + return max(0, cap - reserved) -def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[int]: +def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int, reserved: int = 0) -> frozenset[int]: """Pick CPU (locked) MoE layers automatically when the banks exceed the pin budget. Locks just enough head+tail layers: per-layer decode miss rates are U-shaped, so the ends are the cheapest to move off the slot cache.""" @@ -1178,7 +1242,7 @@ def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[int bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config) if not bank_bytes: return frozenset() - budget = _pin_budget_bytes() + budget = _pin_budget_bytes(reserved) if budget is None or bank_bytes <= budget: return frozenset() if not _cpu_moe_executor_viable(config.model_config): @@ -1292,8 +1356,12 @@ def override(attr: str, value: Any): # this is dangerous, use with caution # comma part must serve every required type, with packages/arch available. required_attn_types = _required_attn_types(model_config) _dtype = getattr(config, "dtype", None) # duck-typed test configs omit it - if AttnType.BSA in required_attn_types and _dtype is not None and _dtype.itemsize != 2: - # Reject at config time: the BSA pool's own assert only fires after the + if ( + required_attn_types & {AttnType.BSA, AttnType.QSA} + and _dtype is not None + and _dtype.itemsize != 2 + ): + # Reject at config time: the BSA/QSA pool's own assert only fires after the # model is resident (and not at all under `python -O`). raise ValueError( f"--dtype {config.dtype}: block-sparse attention serves 16-bit " @@ -1314,7 +1382,7 @@ def override(attr: str, value: Any): # this is dangerous, use with caution if config.attention_backend == "auto": override( "attention_backend", - _resolve_auto_attention_backend(required_attn_types, has_linear_attention), + _resolve_auto_attention_backend(required_attn_types), ) logger.info_rank0(f"Auto-selected attention backend: {config.attention_backend}") _validate_attention_backend_choice(config, override, required_attn_types) diff --git a/python/freetoken/kernel/aot.py b/python/freetoken/kernel/aot.py index 5e87f8c923..cff16177c9 100644 --- a/python/freetoken/kernel/aot.py +++ b/python/freetoken/kernel/aot.py @@ -76,9 +76,14 @@ def build(build_directory: pathlib.Path) -> object: def _fast_index_copy_spec(feature_size: int) -> KernelSpec: - from .fast_index_copy import default_worker_args + from .fast_index_copy import default_worker_args, legacy_fast_index_copy_is_supported worker_threads, worker_feature_size, num_block = default_worker_args(feature_size) + if not legacy_fast_index_copy_is_supported(feature_size): + raise ValueError( + "legacy fast_index_copy requires a whole 128-byte worker row; " + f"feature_size={feature_size} resolves to {worker_feature_size} bytes" + ) args = make_cpp_args(feature_size, worker_threads, worker_feature_size, 1024, num_block, 1) def build(build_directory: pathlib.Path) -> object: @@ -144,6 +149,11 @@ def build(build_directory: pathlib.Path) -> object: def default_kernel_specs() -> tuple[KernelSpec, ...]: + # Import lazily with the other kernel builders. This keeps importing the + # catalog inexpensive while sharing the validity rule with the runtime + # argument derivation rather than duplicating a 128-byte magic number. + from .fast_index_copy import legacy_fast_index_copy_is_supported + specs: list[KernelSpec] = [] specs.extend(_store_spec(element_size) for element_size in DEFAULT_STORE_ELEMENT_SIZES) specs.extend(_index_spec(*variant) for variant in DEFAULT_INDEX_VARIANTS) @@ -151,6 +161,10 @@ def default_kernel_specs() -> tuple[KernelSpec, ...]: specs.extend( _fast_index_copy_spec(feature_size) for feature_size in DEFAULT_FAST_INDEX_COPY_FEATURE_SIZES + # The fused multi-bank cache path supports these small rows directly. + # Do not emit legacy per-bank templates that cannot satisfy their own + # 128-byte vector-loop static assertion. + if legacy_fast_index_copy_is_supported(feature_size) ) specs.append(_fast_index_copy_multi_spec(num_threads=1024, blocks_per_bank=8)) # prefill hit-D2D gather (HBM-bound: wide grid) + its miss-side batch H2D binding. diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py index a00154d099..8a8fe9a3ea 100644 --- a/python/freetoken/kernel/aot_models.py +++ b/python/freetoken/kernel/aot_models.py @@ -63,6 +63,13 @@ class AotModel: arch_aliases: tuple[str, ...] = () +def fp8_block_scale_pad(rows: int, cols: int) -> int: + """Trailing scale-bank dim padded so per-expert row bytes are 16B-aligned (fused copy).""" + while (rows * cols * 2) % 16: + cols += 1 + return cols + + def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int) -> dict[str, int]: """Per-expert row bytes for each offload bank a format registers. @@ -75,17 +82,29 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int # models/loader.py stream_moe_expert_sources: gate_up [E, 2I, H], down [E, H, I], bf16 return {"gate_up": 2 * I * H * 2, "down": H * I * 2} if fmt == "fp8_block": - # qwen3_5_moe/weight.py _build_fp8_expert_banks: fp8 weights + bf16 128x128 block scales + # qwen3_5_moe/weight.py _build_fp8_expert_banks: fp8 weights + bf16 128x128 block + # scales, trailing scale dim 16B-padded (same helper as the loader) B = 128 return { "gate_up": 2 * I * H, - "gate_up_scale": (2 * I // B) * (H // B) * 2, + "gate_up_scale": (2 * I // B) * fp8_block_scale_pad(2 * I // B, H // B) * 2, "down": H * I, - "down_scale": (H // B) * (I // B) * 2, + "down_scale": (H // B) * fp8_block_scale_pad(H // B, I // B) * 2, } if fmt == "q4_0": # gemma4/gguf.py _q4_0_expert_specs: GGML Q4_0 rows, 32 elems -> 18 bytes return {"gate_up": 2 * I * (H // 32 * 18), "down": H * (I // 32 * 18)} + if fmt == "q4_k_q5_k": + # qwen3_5_moe/gguf.py _expert_specs: Q4_K gate/up rows have 144-byte + # 256-element blocks; Q5_K down rows have 176-byte blocks. + return { + "gate_up": 2 * I * (H // 256 * 144), + "down": H * (I // 256 * 176), + } + if fmt == "q6_k_down": + # qwen3_5_moe/gguf.py _q6_down_specs: exceptional late Qwen layers + # keep their byte-exact Q6_K down rows in a separate one-bank cache. + return {"down": H * (I // 256 * 210)} if fmt in ("nvfp4", "nvfp4_marlin", "nvfp4_b12x"): # models/nvfp4_banks.py: packed e2m1 pairs + per-16 fp8-e4m3 scales + fp16 # per-row globals; marlin/b12x repacks are byte-identical with the globals @@ -174,6 +193,19 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int moe_intermediate_size=512, expert_formats=("fp8_block",), ), + AotModel( + # The UnsLOTH Q4_K_M GGUF recipe uses Q4_K gate/up and Q5_K down + # experts, with Q6_K down rows on three late layers. Both cache + # formats appear here so a strict HIP launch cannot JIT the copy helper. + name="unsloth/Qwen3.6-35B-A3B-GGUF-Q4_K_M", + architecture="Qwen3_5MoeForConditionalGeneration", + hidden_size=2048, + kv_groups=((2, 256),), + top_k=8, + moe_intermediate_size=512, + expert_formats=("q4_k_q5_k", "q6_k_down"), + arch_aliases=("Qwen3_5MoeGGUFForCausalLM",), + ), AotModel( name="nvidia/Qwen3.6-35B-A3B-NVFP4", architecture="Qwen3_5MoeForConditionalGeneration", @@ -183,6 +215,19 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int moe_intermediate_size=512, expert_formats=_NVFP4_FORMATS, ), + AotModel( + # QSA compressed-sparse attention (12 of 48 layers): the QSAKVCache stores K/V + # through store_cache (2 kv heads x 256 head_dim), the compressed index-key slab + # and the pending ring write via the vendored qsa triton kernels. Hyper-connections + # carry the residual, so the embedding row indexing() sees is still hidden_size. + name="RadixArk/Qwen3.8-Flash-Next-NVFP4", + architecture="Qwen4ExpForConditionalGeneration", + hidden_size=2560, + kv_groups=((2, 256),), + top_k=10, + moe_intermediate_size=640, + expert_formats=(*_NVFP4_FORMATS, "fp8_block"), + ), AotModel( name="google/gemma-4-26B-A4B-it", architecture="Gemma4ForConditionalGeneration", @@ -253,6 +298,20 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int moe_intermediate_size=2048, expert_formats=_NVFP4_FORMATS, ), + AotModel( + # GLM-5.3-Flash: hybrid KDA + NoPE-MLA/DSA (kpool indexer). Latent writes + # go through torch scatter like GLM-5.2 (no paged-KV store groups); the + # KDA conv/recurrent state lives in the LinearStatePool, not paged KV. + name="RedHatAI/GLM-5.3-Flash-NVFP4", + architecture="Glm5NextForCausalLM", + arch_aliases=("Glm5NextForConditionalGeneration",), + hidden_size=4096, + kv_groups=(), + top_k=8, + moe_intermediate_size=2048, + expert_formats=_NVFP4_FORMATS, + aliases=("zai-org/GLM-5.3-Flash", "LibertAIDAI/GLM-5.3-Flash-NVFP4"), + ), AotModel( # MiniMaxAI/MiniMax-M2.5 ships block-fp8, which has no expert-bank # provider for this arch on main -- the NVFP4 release is the servable @@ -401,7 +460,8 @@ def aggregate_fast_index_copy_feature_sizes() -> tuple[int, ...]: sizes: set[int] = set(TEST_FEATURE_SIZES) for model in SUPPORTED_MODELS: sizes.update(fast_index_copy_feature_sizes(model)) - return tuple(sorted(sizes)) + # the per-bank kernel copies rows in fixed 128-byte steps; other sizes cannot compile + return tuple(sorted(size for size in sizes if size % 128 == 0)) __all__ = [ diff --git a/python/freetoken/kernel/backend.py b/python/freetoken/kernel/backend.py index 3037ad8d73..137177dfbc 100644 --- a/python/freetoken/kernel/backend.py +++ b/python/freetoken/kernel/backend.py @@ -11,6 +11,20 @@ import importlib.util +@functools.cache +def is_rocm_runtime() -> bool: + """Return whether PyTorch is backed by HIP rather than NVIDIA CUDA. + + Optional packages in this module publish CUDA binaries. Import discovery + alone is insufficient on ROCm because a stale CUDA package may be present + in an otherwise healthy environment. Returning ``False`` from each + CUDA-only capability probe preserves the existing pure-Triton fallback. + """ + import torch + + return bool(getattr(torch.version, "hip", None)) + + def _importable(name: str) -> bool: # find_spec normally returns None when a package is absent, but it can raise # (broken parent package, or a meta_path finder that blocks the name); treat @@ -23,12 +37,12 @@ def _importable(name: str) -> bool: @functools.cache def is_flashinfer_installed() -> bool: - return _importable("flashinfer") + return not is_rocm_runtime() and _importable("flashinfer") @functools.cache def is_sgl_kernel_installed() -> bool: - return _importable("sgl_kernel") + return not is_rocm_runtime() and _importable("sgl_kernel") @functools.cache @@ -39,7 +53,7 @@ def is_triton_kernels_installed() -> bool: source tree and has no Windows wheel. It is also not one of the six ops ``freetoken.kernel.triton`` reimplements, so its call-site carries its own fallback. """ - return _importable("triton_kernels") + return not is_rocm_runtime() and _importable("triton_kernels") @functools.cache @@ -50,6 +64,8 @@ def driver_cuda_version() -> int | None: toolkit version. Resolved through the ``_pinned_tensor`` extension's link-time cudart, so it works wherever the extension builds (including Windows) -- no dlopen by soname.""" + if is_rocm_runtime(): + return None try: from freetoken.kernel.pinned import _load_pinned_extension diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp index 880e8637a0..ac31d1c089 100644 --- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp +++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp @@ -29,7 +29,7 @@ #include #include -#include +#include "../hip_compat.h" #include #if defined(__linux__) @@ -71,7 +71,15 @@ inline bf16_t f32_to_bf16(float f) { // MiniMax-M3): gate/up are combined jointly with the runtime alpha/limit // scalars, so it is handled in the do_pass1 epilogue (act_apply never sees it; // the mxfp4 kernel additionally fuses its own copy of the same math). -enum ActKind { ACT_SILU = 0, ACT_GELU = 1, ACT_GELU_TANH = 2, ACT_SWIGLUOAI = 3 }; +// ACT_SWIGLU_CLAMP (GLM-5.3 "swiglu_limit") is the same clamped form WITHOUT +// the (up + 1) bias: clamp(gate, max=lim) * sigmoid(alpha*gate) * clamp(up, +-lim). +enum ActKind { + ACT_SILU = 0, + ACT_GELU = 1, + ACT_GELU_TANH = 2, + ACT_SWIGLUOAI = 3, + ACT_SWIGLU_CLAMP = 4, +}; inline float act_apply(int act, float x) { if (act == ACT_SILU) return x / (1.0f + std::exp(-x)); @@ -1604,7 +1612,8 @@ struct CpuMoeExecutor { bf16_t* g_row = g_scratch.data() + ((size_t)tok * top_k + k) * I; const int i0 = static_cast(ib) * IBLK; const int i1 = std::min(I, i0 + IBLK); - const bool swigluoai = act == ACT_SWIGLUOAI; + const bool clamped = act == ACT_SWIGLUOAI || act == ACT_SWIGLU_CLAMP; + const float up_bias = act == ACT_SWIGLUOAI ? 1.0f : 0.0f; const float lim = swiglu_limit, alpha = swiglu_alpha; for (int i = i0; i < i1; ++i) { // gate = row i, up = row I+i @@ -1612,14 +1621,15 @@ struct CpuMoeExecutor { gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, i, x_row, xe, xo, xi8, xas) * w_in; float up = gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, I + i, x_row, xe, xo, xi8, xas) * w_in; - if (swigluoai) { - // clamp(gate, max=lim) * sigmoid(alpha * gate) * (clamp(up, +-lim) + 1) - // -- same math as the mxfp4 kernel's fused epilogue (lim == +inf: no clamp). + if (clamped) { + // clamp(gate, max=lim) * sigmoid(alpha * gate) * (clamp(up, +-lim) + up_bias) + // -- swigluoai carries the +1 up bias (gpt-oss/MiniMax); swiglu_clamp + // (GLM-5.3) does not. lim == +inf: no clamp. if (gate > lim) gate = lim; if (up > lim) up = lim; else if (up < -lim) up = -lim; const float glu = gate / (1.0f + std::exp(-gate * alpha)); - g_row[i] = f32_to_bf16(glu * (up + 1.0f)); + g_row[i] = f32_to_bf16(glu * (up + up_bias)); } else { g_row[i] = f32_to_bf16(act_apply(act, gate) * up); } @@ -2146,5 +2156,5 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { // accepts id 3 without error and silently computes the wrong activation // (act_apply falls through to gelu_tanh); the probe turns a stale extension // into a loud rebuild instruction instead of wrong model outputs. - m.def("max_generic_act_id", []() { return static_cast(ACT_SWIGLUOAI); }); + m.def("max_generic_act_id", []() { return static_cast(ACT_SWIGLU_CLAMP); }); } diff --git a/python/freetoken/kernel/csrc/gguf/dispatch.h b/python/freetoken/kernel/csrc/gguf/dispatch.h index f42a216332..17d2a0db83 100644 --- a/python/freetoken/kernel/csrc/gguf/dispatch.h +++ b/python/freetoken/kernel/csrc/gguf/dispatch.h @@ -11,6 +11,20 @@ #endif // Warp-shuffle wrappers the donor pulls from sgl-kernel's utils.h (CUDA variants). +// HIP's __shfl_xor_sync requires a 64-bit mask unconditionally (amd_warp_sync_functions.h +// static_asserts sizeof(mask) == 8) regardless of actual wavefront width; the donor's +// CUDA-style callers pass a 32-bit `unsigned int` mask (e.g. 0xffffffff), so widen it here +// rather than touching every call site. +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) +#ifndef SGLANG_SHFL_XOR_SYNC +#define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) \ + __shfl_xor_sync((unsigned long long)(mask), (var), (lane_mask)) +#endif +#ifndef SGLANG_SHFL_XOR_SYNC_WIDTH +#define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) \ + __shfl_xor_sync((unsigned long long)(mask), (var), (lane_mask), (width)) +#endif +#else #ifndef SGLANG_SHFL_XOR_SYNC #define SGLANG_SHFL_XOR_SYNC(mask, var, lane_mask) __shfl_xor_sync((mask), (var), (lane_mask)) #endif @@ -18,6 +32,7 @@ #define SGLANG_SHFL_XOR_SYNC_WIDTH(mask, var, lane_mask, width) \ __shfl_xor_sync((mask), (var), (lane_mask), (width)) #endif +#endif #define DISPATCH_CASE_FLOAT_TYPES(...) \ AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ diff --git a/python/freetoken/kernel/csrc/gguf/ggml-common.h b/python/freetoken/kernel/csrc/gguf/ggml-common.h index 88c21a4ab3..55c2c804e6 100644 --- a/python/freetoken/kernel/csrc/gguf/ggml-common.h +++ b/python/freetoken/kernel/csrc/gguf/ggml-common.h @@ -8,7 +8,12 @@ #define CUDA_DEQUANTIZE_BLOCK_SIZE 256 #define CUDA_QUANTIZE_BLOCK_SIZE 256 #define GGML_CUDA_DMMV_X 32 +// Keep one output row per workgroup by default. Isolated ROCm experiments may +// override this at compile time to compare a second row per workgroup without +// changing the checked-in production default or silently changing arithmetic. +#ifndef GGML_CUDA_MMV_Y #define GGML_CUDA_MMV_Y 1 +#endif // Data Structures // QK = number of values after dequantization @@ -1004,7 +1009,15 @@ static __device__ __forceinline__ int __vsubss4(const int a, const int b) { } static __device__ __forceinline__ int __dp4a(const int a, const int b, int c) { -#if __has_builtin(__builtin_amdgcn_sdot4) +#if __has_builtin(__builtin_amdgcn_sudot4) && (defined(__gfx1100__) || defined(__gfx1150__) || defined(__gfx1151__)) + // RDNA3-family HIP compilers can lower the signed-dot form through sudot4. + // The two `true` operand flags preserve the signed four-byte dot-product + // semantics of sdot4, while matching the intrinsic selection in the current + // llama.cpp HIP implementation. This branch is intentionally limited to + // gfx1100/gfx1150/gfx1151 so older AMD targets and every CUDA build retain + // their proven implementation below. + c = __builtin_amdgcn_sudot4(true, a, true, b, c, false); +#elif __has_builtin(__builtin_amdgcn_sdot4) c = __builtin_amdgcn_sdot4(a, b, c, false); #else const int8x4_t va = reinterpret_cast(a); diff --git a/python/freetoken/kernel/csrc/gguf/moe_vec.cuh b/python/freetoken/kernel/csrc/gguf/moe_vec.cuh index 8cef9e080a..3d12f1e965 100644 --- a/python/freetoken/kernel/csrc/gguf/moe_vec.cuh +++ b/python/freetoken/kernel/csrc/gguf/moe_vec.cuh @@ -51,6 +51,23 @@ static __global__ void moe_vec_q( } } +#if defined(USE_ROCM) +// The HIP launcher is defined after the CUDA-compatible wrapper so the +// generic wrappers remain grouped by quantization format below. +template +static void moe_vec_q4_0_q8_1_hip_two_rows_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream); +#endif + template static void moe_vec_q4_0_q8_1_cuda( const void* vx, @@ -63,12 +80,103 @@ static void moe_vec_q4_0_q8_1_cuda( const int nrows, const int token_stride, cudaStream_t stream) { +#if defined(USE_ROCM) + // Route AMD builds through the one-wave/two-row specialization above. CUDA + // retains the established generic implementation until it has independent + // NVIDIA evidence, so this HIP experiment cannot alter CUDA behavior. + moe_vec_q4_0_q8_1_hip_two_rows_cuda( + vx, vy, dst, topk_ids, top_k, tokens, ncols, nrows, token_stride, stream); +#else const int block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; const dim3 block_nums(block_num_y, 1, tokens * top_k); const dim3 block_dims(WARP_SIZE, GGML_CUDA_MMV_Y, 1); moe_vec_q <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); +#endif +} + +#if defined(USE_ROCM) +// HIP Q4_0 MoE specialization derived from the current llama.cpp MMVQ row +// structure. Unlike the older GGML_CUDA_MMV_Y=2 experiment, this launch uses +// one 32-lane wave for two rows, rather than two independent waves. The two +// float accumulators share the same packed Q4_0 activation block and expert +// selection, reducing grid work while preserving FreeToken's existing packed +// bank layout, route indexing, and BF16 output contract. +template +__launch_bounds__(WARP_SIZE, 1) +static __global__ void moe_vec_q4_0_hip_two_rows( + const void* __restrict__ vx, + const void* __restrict__ vy, + scalar_t* __restrict__ dst, + const int* __restrict__ topk_ids, + const int topk, + const int ncols, + const int nrows, + const int token_stride) { + // X indexes adjacent pairs of output rows. Y is the flattened + // token/top-k route index, matching the former Z dimension exactly. + const int row0 = 2 * blockIdx.x; + const int route = blockIdx.y; + if (row0 >= nrows) { + return; + } + + const int token = route / topk; + const int expert = topk_ids[route]; + const int blocks_per_row = ncols / QK4_0; + const int blocks_per_wave = VDR_Q4_0_Q8_1_MMVQ * WARP_SIZE / QI4_0; + const block_q4_0* x = ((const block_q4_0*)vx) + expert * nrows * blocks_per_row; + const block_q8_1* y = (const block_q8_1*)(((const int*)vy) + token * token_stride); + + // Each lane owns the same packed-Q4 range for both rows. Keeping the + // reductions separate preserves the original arithmetic for each result. + float tmp0 = 0.0f; + float tmp1 = 0.0f; + for (int i = threadIdx.x / (QI4_0 / VDR_Q4_0_Q8_1_MMVQ); i < blocks_per_row; + i += blocks_per_wave) { + const int iby = i * (QK4_0 / QK8_1); + const int iqs = VDR_Q4_0_Q8_1_MMVQ * (threadIdx.x % (QI4_0 / VDR_Q4_0_Q8_1_MMVQ)); + tmp0 += vec_dot_q4_0_q8_1(&x[row0 * blocks_per_row + i], &y[iby], iqs); + if (row0 + 1 < nrows) { + tmp1 += vec_dot_q4_0_q8_1(&x[(row0 + 1) * blocks_per_row + i], &y[iby], iqs); + } + } + + // A wave-level XOR reduction leaves the same sum in every lane. Lane zero + // writes row zero and lane one writes row one, avoiding shared memory. +#pragma unroll + for (int mask = WARP_SIZE / 2; mask > 0; mask >>= 1) { + tmp0 += SGLANG_SHFL_XOR_SYNC(uint32_t(-1), tmp0, mask); + tmp1 += SGLANG_SHFL_XOR_SYNC(uint32_t(-1), tmp1, mask); + } + if (threadIdx.x == 0) { + dst[route * nrows + row0] = tmp0; + } + if (threadIdx.x == 1 && row0 + 1 < nrows) { + dst[route * nrows + row0 + 1] = tmp1; + } +} + +template +static void moe_vec_q4_0_q8_1_hip_two_rows_cuda( + const void* vx, + const void* vy, + scalar_t* dst, + const int* topk_ids, + const int top_k, + const int tokens, + const int ncols, + const int nrows, + const int token_stride, + cudaStream_t stream) { + // One block now covers two rows and one route. ``tokens * top_k`` remains + // the complete flattened routing domain used by the original launcher. + const dim3 block_nums((nrows + 1) / 2, tokens * top_k, 1); + const dim3 block_dims(WARP_SIZE, 1, 1); + moe_vec_q4_0_hip_two_rows + <<>>(vx, vy, dst, topk_ids, top_k, ncols, nrows, token_stride); } +#endif template static void moe_vec_q4_1_q8_1_cuda( diff --git a/python/freetoken/kernel/csrc/hip_compat.h b/python/freetoken/kernel/csrc/hip_compat.h new file mode 100644 index 0000000000..8ece57aea0 --- /dev/null +++ b/python/freetoken/kernel/csrc/hip_compat.h @@ -0,0 +1,59 @@ +#pragma once + +// Lets pinned_tensor.cpp and cpu_moe_ext.cpp call the CUDA Runtime API names they +// were written against while actually linking HIP on ROCm builds. Only the calls +// those two files use are covered -- this is not a general CUDA/HIP compat layer. +// Host C++ extension compilation can use a normal C++ frontend even when the +// active PyTorch distribution is ROCm. setup.py therefore supplies the explicit +// FREETOKEN_USE_ROCM build macro, while the compiler macros retain compatibility +// with HIP device translation units and standalone hipcc builds. +#if defined(FREETOKEN_USE_ROCM) || defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) +#include + +// CUDA's host-callback calling-convention annotation; empty on POSIX (matches +// cuda_runtime_api.h's own definition there). hipHostFn_t has no such annotation. +#define CUDART_CB + +using cudaError_t = hipError_t; +using cudaStream_t = hipStream_t; +constexpr hipError_t cudaSuccess = hipSuccess; +constexpr unsigned int cudaHostAllocPortable = hipHostMallocPortable; +constexpr unsigned int cudaHostAllocMapped = hipHostMallocMapped; +constexpr unsigned int cudaHostRegisterPortable = hipHostRegisterPortable; +constexpr unsigned int cudaHostRegisterMapped = hipHostRegisterMapped; +constexpr hipDeviceAttribute_t cudaDevAttrUnifiedAddressing = + hipDeviceAttributeUnifiedAddressing; +constexpr hipDeviceAttribute_t cudaDevAttrCanUseHostPointerForRegisteredMem = + hipDeviceAttributeCanUseHostPointerForRegisteredMem; + +inline hipError_t cudaMallocHost(void **ptr, size_t size) { + return hipHostMalloc(ptr, size, hipHostMallocDefault); +} +inline hipError_t cudaFreeHost(void *ptr) { return hipHostFree(ptr); } +inline hipError_t cudaHostAlloc(void **ptr, size_t size, unsigned int flags) { + return hipHostMalloc(ptr, size, flags); +} +inline hipError_t cudaGetDevice(int *device) { return hipGetDevice(device); } +inline hipError_t cudaDeviceGetAttribute(int *value, hipDeviceAttribute_t attr, + int device) { + return hipDeviceGetAttribute(value, attr, device); +} +inline hipError_t cudaHostGetDevicePointer(void **devPtr, void *hostPtr, + unsigned int flags) { + return hipHostGetDevicePointer(devPtr, hostPtr, flags); +} +inline hipError_t cudaHostRegister(void *ptr, size_t size, unsigned int flags) { + return hipHostRegister(ptr, size, flags); +} +inline hipError_t cudaDriverGetVersion(int *v) { return hipDriverGetVersion(v); } +inline const char *cudaGetErrorString(hipError_t e) { return hipGetErrorString(e); } +inline hipError_t cudaStreamSynchronize(hipStream_t s) { + return hipStreamSynchronize(s); +} +inline hipError_t cudaLaunchHostFunc(hipStream_t s, hipHostFn_t fn, void *data) { + return hipLaunchHostFunc(s, fn, data); +} + +#else +#include +#endif diff --git a/python/freetoken/kernel/csrc/include/freetoken/utils.cuh b/python/freetoken/kernel/csrc/include/freetoken/utils.cuh index 8e917832c1..ea472b632d 100644 --- a/python/freetoken/kernel/csrc/include/freetoken/utils.cuh +++ b/python/freetoken/kernel/csrc/include/freetoken/utils.cuh @@ -10,6 +10,50 @@ #include #include +// nvcc implicitly pulls in the CUDA runtime for .cu translation units; hipcc does +// not do the equivalent for HIP, so it must be included explicitly here. On the +// HIP path there is no cudaLaunchKernelEx/cudaLaunchConfig_t equivalent (that API +// is Hopper PDL-specific), so LaunchKernel gets its own HIP-side definition below +// instead of a name-aliasing shim -- see PDL below for why that also means +// with_attr(true) is a no-op on this path. +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) +#include + +using cudaError_t = hipError_t; +constexpr hipError_t cudaSuccess = hipSuccess; +using cudaStream_t = hipStream_t; + +inline const char *cudaGetErrorString(hipError_t e) { return hipGetErrorString(e); } +inline hipError_t cudaGetLastError() { return hipGetLastError(); } +inline hipError_t cudaFuncSetAttribute(const void *func, hipFuncAttribute attr, + int value) { + return hipFuncSetAttribute(func, attr, value); +} +constexpr hipFuncAttribute cudaFuncAttributeMaxDynamicSharedMemorySize = + hipFuncAttributeMaxDynamicSharedMemorySize; + +inline hipError_t cudaGetDevice(int *device) { return hipGetDevice(device); } +inline hipError_t cudaDeviceGetAttribute(int *value, hipDeviceAttribute_t attr, + int device) { + return hipDeviceGetAttribute(value, attr, device); +} +inline hipError_t cudaHostGetDevicePointer(void **devPtr, void *hostPtr, + unsigned int flags) { + return hipHostGetDevicePointer(devPtr, hostPtr, flags); +} +constexpr hipDeviceAttribute_t cudaDevAttrUnifiedAddressing = + hipDeviceAttributeUnifiedAddressing; +constexpr hipDeviceAttribute_t cudaDevAttrCanUseHostPointerForRegisteredMem = + hipDeviceAttributeCanUseHostPointerForRegisteredMem; + +// CUDA-only kernel-parameter annotation (passes large by-value params via constant +// memory instead of copying them into local/generic memory first); HIP has no +// equivalent attribute, so this just falls back to an ordinary by-value parameter. +#define __grid_constant__ +#else +#include +#endif + namespace device { inline constexpr auto kWarpThreads = 32u; @@ -42,16 +86,24 @@ __always_inline __device__ auto offset(const T *ptr, U... offset) -> const namespace PDL { +// Programmatic Dependent Launch is a Hopper-only CUDA hardware feature; the PTX +// below has no HIP/ROCm equivalent. Callers gate kUsePDL off for non-Hopper CUDA +// targets already, and LaunchKernel::with_attr is a no-op on HIP (see below), so +// this stays unconditionally a no-op there rather than a compile failure. template __always_inline __device__ void wait() { +#if !(defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)) if constexpr (kUsePDL) { asm volatile("griddepcontrol.wait;" ::: "memory"); } +#endif } template __always_inline __device__ void launch() { +#if !(defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)) if constexpr (kUsePDL) { asm volatile("griddepcontrol.launch_dependents;" :::); } +#endif } } // namespace PDL @@ -88,6 +140,50 @@ template inline void set_smem_once(std::size_t smem_size) { last_smem_size, " bytes"); } +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + +// HIP has no cudaLaunchKernelEx/cudaLaunchConfig_t analog (that API only exists to +// carry Hopper PDL attributes, which ROCm hardware has no equivalent for), so this +// launches via the plain triple-chevron form instead. with_attr(true) is therefore +// a no-op here -- there is no attribute to carry. +struct LaunchKernel { +public: + explicit LaunchKernel(dim3 grid_dim, dim3 block_dim, DLDevice device, + std::size_t dynamic_shared_mem_bytes = 0) noexcept + : m_grid_dim(grid_dim), m_block_dim(block_dim), + m_smem(dynamic_shared_mem_bytes), m_stream(resolve_device(device)) {} + + explicit LaunchKernel(dim3 grid_dim, dim3 block_dim, cudaStream_t stream, + std::size_t dynamic_shared_mem_bytes = 0) noexcept + : m_grid_dim(grid_dim), m_block_dim(block_dim), + m_smem(dynamic_shared_mem_bytes), m_stream(stream) {} + + static auto resolve_device(DLDevice device) -> cudaStream_t { + return static_cast( + ::TVMFFIEnvGetStream(device.device_type, device.device_id)); + } + + LaunchKernel(const LaunchKernel &) = delete; + LaunchKernel &operator=(const LaunchKernel &) = delete; + + template + auto operator()(T &&kernel, Args &&...args) const -> void { + kernel<<>>( + std::forward(args)...); + CUDA_CHECK(::cudaGetLastError()); + } + + auto with_attr(bool /*use_pdl*/) -> LaunchKernel & { return *this; } + +private: + dim3 m_grid_dim; + dim3 m_block_dim; + std::size_t m_smem; + cudaStream_t m_stream; +}; + +#else + struct LaunchKernel { public: explicit LaunchKernel(dim3 grid_dim, dim3 block_dim, DLDevice device, @@ -141,4 +237,6 @@ private: cudaLaunchAttribute m_attr_cache; }; +#endif + } // namespace host diff --git a/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh b/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh index bb83c23ed2..2d1dbc05b0 100644 --- a/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh +++ b/python/freetoken/kernel/csrc/jit/fast_index_copy.cuh @@ -33,6 +33,38 @@ inline constexpr auto get_mem_package() { } } +// The ld.global.L1::no_allocate / st.global.wt PTX below are cache-policy hints +// (skip L1 allocate on read, write-through on store) with no HIP equivalent -- AMD +// ROCm builds fall back to plain loads/stores. Correctness is unchanged; only the +// cache-policy hint is lost. +#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__) + +__always_inline __device__ auto load_nc(const uint1* __restrict__ src) -> uint1 { + return *src; +} + +__always_inline __device__ auto load_nc(const uint2* __restrict__ src) -> uint2 { + return *src; +} + +__always_inline __device__ auto load_nc(const uint4* __restrict__ src) -> uint4 { + return *src; +} + +__always_inline __device__ void store_nc(uint1* __restrict__ dst, const uint1& value) { + *dst = value; +} + +__always_inline __device__ void store_nc(uint2* __restrict__ dst, const uint2& value) { + *dst = value; +} + +__always_inline __device__ void store_nc(uint4* __restrict__ dst, const uint4& value) { + *dst = value; +} + +#else + __always_inline __device__ auto load_nc(const uint1* __restrict__ src) -> uint1 { uint32_t tmp; asm volatile("ld.global.L1::no_allocate.b32 %0,[%1];" : "=r"(tmp) : "l"(src)); @@ -70,6 +102,8 @@ __always_inline __device__ void store_nc(uint4* __restrict__ dst, const uint4& v asm volatile("st.global.wt.v4.b32 [%0],{%1,%2,%3,%4};" ::"l"(dst), "r"(tmp0), "r"(tmp1), "r"(tmp2), "r"(tmp3)); } +#endif + __always_inline __device__ void wait_flag_clear(const int32_t* __restrict__ flag_ptr) { // Exponential backoff to avoid hammering a global atomic in a tight loop. auto* flag = reinterpret_cast(const_cast(flag_ptr)); @@ -344,17 +378,17 @@ struct FastIndexCopyKernel { TensorMatcher({-1, D}) .with_dtype(data_dtype) - .with_device() + .with_device() .verify(src); TensorMatcher({-1, D}) .with_dtype(data_dtype) - .with_device() + .with_device() .verify(dst); TensorMatcher({L}) .with_dtype(indices_dtype) - .with_device(device) + .with_device(device) .verify(src_indices) .verify(dst_indices); @@ -363,7 +397,7 @@ struct FastIndexCopyKernel { const auto num_indices_tensor = num_indices.value(); TensorMatcher({1}) .with_dtype(num_indices_dtype) - .with_device(device) + .with_device(device) .verify(num_indices_tensor); num_indices_data_ptr = static_cast(num_indices_tensor.data_ptr()); diff --git a/python/freetoken/kernel/csrc/jit/index.cu b/python/freetoken/kernel/csrc/jit/index.cu index ca0e1db26e..aca58383d3 100644 --- a/python/freetoken/kernel/csrc/jit/index.cu +++ b/python/freetoken/kernel/csrc/jit/index.cu @@ -114,15 +114,15 @@ struct IndexKernel { TensorMatcher({-1, D}) // .with_dtype(weights_dtype_) - .with_device(device_) + .with_device(device_) .verify(weights); TensorMatcher({L, D}) // .with_dtype(weights_dtype_) - .with_device(device_) + .with_device(device_) .verify(output); TensorMatcher({L}) // .with_dtype(indices_dtype_) - .with_device(device_) + .with_device(device_) .verify(indices); const auto device = device_.unwrap(); diff --git a/python/freetoken/kernel/csrc/jit/store.cu b/python/freetoken/kernel/csrc/jit/store.cu index 8d84d76ef1..162dfdfe7a 100644 --- a/python/freetoken/kernel/csrc/jit/store.cu +++ b/python/freetoken/kernel/csrc/jit/store.cu @@ -72,18 +72,18 @@ struct StoreKernel { TensorMatcher({-1, D}) // .with_strides({X, 1}) - .with_device(device_) + .with_device(device_) .with_dtype(dtype_) .verify(k_cache) .verify(v_cache); TensorMatcher({L, D}) // .with_strides({Y, 1}) - .with_device(device_) + .with_device(device_) .with_dtype(dtype_) .verify(k) .verify(v); TensorMatcher({L}) // - .with_device(device_) + .with_device(device_) .with_dtype(indices_dtype_) .verify(indices); diff --git a/python/freetoken/kernel/csrc/pinned_tensor.cpp b/python/freetoken/kernel/csrc/pinned_tensor.cpp index c3947adfa3..4cb983f265 100644 --- a/python/freetoken/kernel/csrc/pinned_tensor.cpp +++ b/python/freetoken/kernel/csrc/pinned_tensor.cpp @@ -1,5 +1,5 @@ #include -#include +#include "hip_compat.h" #include namespace { diff --git a/python/freetoken/kernel/csrc/ple_store/ple_store_ext.cpp b/python/freetoken/kernel/csrc/ple_store/ple_store_ext.cpp new file mode 100644 index 0000000000..2d54d692d6 --- /dev/null +++ b/python/freetoken/kernel/csrc/ple_store/ple_store_ext.cpp @@ -0,0 +1,621 @@ +// Disk-backed PLE row store: rows read straight from the checkpoint's fp8 shard tensors +// through an extent table (PleRowSource in ple_ssd.py). Engine-thread only, no locks. +// Duplicate rows in one fill dedup into ONE batched read round; no RAM cache and no +// per-sequence state. Hash reference: tests/models/qwen4_exp/test_ple_disk.py. +// Platform seams: TableFile (O_DIRECT+pread; Win: NO_BUFFERING), BatchReader (io_uring, +// pread-pool fallback = the portable shape), cumemop_* (dlopen libcuda; Win: nvcuda). + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#if defined(__linux__) && __has_include() +#include +#include +#include +#define PLE_HAS_IO_URING 1 +#else +#define PLE_HAS_IO_URING 0 +#endif + +#include + +namespace py = pybind11; + +namespace { + +constexpr int64_t kPage = 4096; +constexpr int64_t kSpanMax = 2 * kPage; // a row is <= one page, so it spans at most two +constexpr unsigned kBatchEntries = 64; +// fio on this class of disk: pread saturates at ~16 threads; more only adds latency +constexpr unsigned kReaderThreads = 16; + +// ---- portability shims ---- + +void release_store_i64(int64_t *ptr, int64_t value) { + __atomic_store_n(ptr, value, __ATOMIC_RELEASE); +} + +uint8_t *page_aligned_alloc(size_t bytes) { + void *p = nullptr; + if (posix_memalign(&p, kPage, bytes) != 0) throw std::bad_alloc(); + return static_cast(p); +} + +// Stream memops for the flag-sync fast path, resolved from the driver at runtime. +using CuMemOp64Fn = int (*)(void *stream, unsigned long long addr, unsigned long long value, + unsigned int flags); +CuMemOp64Fn g_cu_write64 = nullptr; +CuMemOp64Fn g_cu_wait64 = nullptr; +constexpr unsigned kCuWaitValueGeq = 0x0; +constexpr unsigned kCuWriteDefault = 0x0; + +bool cumemop_resolve() { + static bool resolved = [] { + void *h = dlopen("libcuda.so.1", RTLD_LAZY | RTLD_LOCAL); + if (h == nullptr) h = dlopen("libcuda.so", RTLD_LAZY | RTLD_LOCAL); + if (h == nullptr) return false; + g_cu_write64 = reinterpret_cast(dlsym(h, "cuStreamWriteValue64_v2")); + if (g_cu_write64 == nullptr) + g_cu_write64 = reinterpret_cast(dlsym(h, "cuStreamWriteValue64")); + g_cu_wait64 = reinterpret_cast(dlsym(h, "cuStreamWaitValue64_v2")); + if (g_cu_wait64 == nullptr) + g_cu_wait64 = reinterpret_cast(dlsym(h, "cuStreamWaitValue64")); + return g_cu_write64 != nullptr && g_cu_wait64 != nullptr; + }(); + return resolved; +} + +int memop_write(uintptr_t stream, uintptr_t addr, uint64_t value) { + if (!cumemop_resolve()) return -1; + return g_cu_write64(reinterpret_cast(stream), addr, value, kCuWriteDefault); +} + +int memop_wait_geq(uintptr_t stream, uintptr_t addr, uint64_t value) { + if (!cumemop_resolve()) return -1; + return g_cu_wait64(reinterpret_cast(stream), addr, value, kCuWaitValueGeq); +} + +// WAIT(>=1) then RESET: resetting first would race a fast host signal and deadlock the stream. +void memop_wait_reset(uintptr_t stream, uintptr_t flag_addr) { + if (memop_wait_geq(stream, flag_addr, 1) != 0 || memop_write(stream, flag_addr, 0) != 0) + throw std::runtime_error("stream memops rejected in capture; set FREETOKEN_PLE_SYNC=gate"); +} + +void signal_flag(uintptr_t flag_addr) { + release_store_i64(reinterpret_cast(flag_addr), 1); +} + +// ---- TableFile: platform seam for on-disk files ---- + +// Read at least need bytes; len is the larger aligned span the request must keep. +// Resuming past need is not safe: a read that crossed EOF ends at an unaligned offset. +void pread_min(int fd, uint8_t *buf, int64_t len, int64_t need, int64_t off) { + int64_t done = 0; + while (done < need) { + ssize_t got = ::pread(fd, buf + done, len - done, off + done); + if (got < 0) { + if (errno == EINTR) continue; + throw std::runtime_error(std::string("pread: ") + std::strerror(errno)); + } + if (got == 0) break; + done += got; + } + if (done < need) + throw std::runtime_error("short read at offset " + std::to_string(off) + ": got " + + std::to_string(done) + " of " + std::to_string(need)); +} + +class TableFile { + public: + explicit TableFile(const std::string &path) { + fd_ = ::open(path.c_str(), O_RDONLY | O_CLOEXEC | O_DIRECT); + direct_ = fd_ >= 0; + if (fd_ < 0) { + fd_ = ::open(path.c_str(), O_RDONLY | O_CLOEXEC); + direct_ = false; + } + if (fd_ < 0) throw std::runtime_error(path + ": " + std::strerror(errno)); + struct stat st{}; + if (fstat(fd_, &st) != 0) { + ::close(fd_); + throw std::runtime_error(path + ": fstat: " + std::strerror(errno)); + } + size_ = st.st_size; + if (!direct_) posix_fadvise(fd_, 0, 0, POSIX_FADV_RANDOM); + } + + ~TableFile() { + if (fd_ >= 0) ::close(fd_); + } + + TableFile(const TableFile &) = delete; + TableFile &operator=(const TableFile &) = delete; + + bool direct_io() const { return direct_; } + int native_fd() const { return fd_; } + int64_t size() const { return size_; } + + // keep buffered fallback reads out of the page cache; a no-op under direct I/O + void discard_cache(int64_t off, int64_t len) const { + if (!direct_) posix_fadvise(fd_, off, len, POSIX_FADV_DONTNEED); + } + + private: + int fd_ = -1; + bool direct_ = false; + int64_t size_ = 0; +}; + +// ---- BatchReader: platform seam for parallel positioned reads ---- + +// Pipelined: at most capacity() reads in flight; wait_one() returns a finished tag to refill. +class BatchReader { + public: + virtual ~BatchReader() = default; + virtual std::string name() const = 0; + virtual unsigned capacity() const = 0; + virtual void submit(unsigned tag, int fd, uint8_t *buf, int64_t len, int64_t need, + int64_t off) = 0; + virtual unsigned wait_one() = 0; + // reap every in-flight read so stale completions cannot leak into the next fill + virtual void drain() noexcept = 0; +}; + +class ThreadPoolBatchReader final : public BatchReader { + public: + ThreadPoolBatchReader() { + // these threads block on I/O, not compute, so the core count is only a default + unsigned n = std::max(1u, std::min(kReaderThreads, std::thread::hardware_concurrency())); + if (const char *env = std::getenv("FREETOKEN_PLE_READER_THREADS")) { + // kBatchEntries is the submit depth, so threads past it never get a read + const int v = std::atoi(env); + if (v > 0) n = std::min((unsigned)v, kBatchEntries); + } + for (unsigned i = 0; i < n; i++) workers_.emplace_back([this] { work(); }); + } + + ~ThreadPoolBatchReader() override { + { + std::lock_guard lock(mu_); + stop_ = true; + } + work_cv_.notify_all(); + for (auto &w : workers_) w.join(); + } + + std::string name() const override { return "pread-pool x" + std::to_string(workers_.size()); } + unsigned capacity() const override { return kBatchEntries; } + + void submit(unsigned tag, int fd, uint8_t *buf, int64_t len, int64_t need, + int64_t off) override { + { + std::lock_guard lock(mu_); + queue_.push_back(Req{tag, fd, buf, len, need, off}); + in_flight_++; + } + work_cv_.notify_one(); + } + + unsigned wait_one() override { + std::unique_lock lock(mu_); + done_cv_.wait(lock, [this] { return !done_.empty(); }); + Done d = std::move(done_.front()); + done_.pop_front(); + in_flight_--; + if (!d.error.empty()) throw std::runtime_error(d.error); + return d.tag; + } + + void drain() noexcept override { + // wait out all in-flight reads: a late worker write must not race the slot's reuse + std::unique_lock lock(mu_); + done_cv_.wait(lock, [this] { return done_.size() == in_flight_; }); + in_flight_ = 0; + done_.clear(); + } + + private: + struct Req { + unsigned tag; + int fd; + uint8_t *buf; + int64_t len; + int64_t need; + int64_t off; + }; + struct Done { + unsigned tag; + std::string error; + }; + + void work() { + for (;;) { + Req r; + { + std::unique_lock lock(mu_); + work_cv_.wait(lock, [this] { return stop_ || !queue_.empty(); }); + if (stop_) return; + r = queue_.front(); + queue_.pop_front(); + } + Done d{r.tag, {}}; + try { + pread_min(r.fd, r.buf, r.len, r.need, r.off); + } catch (const std::exception &e) { + d.error = e.what(); + } + { + std::lock_guard lock(mu_); + done_.push_back(std::move(d)); + } + done_cv_.notify_one(); + } + } + + std::vector workers_; + std::mutex mu_; + std::condition_variable work_cv_, done_cv_; + std::deque queue_; + std::deque done_; + size_t in_flight_ = 0; + bool stop_ = false; +}; + +#if PLE_HAS_IO_URING + +// Minimal single-issuer io_uring: submit up to `entries` reads, wait for all. +class IoUringBatchReader final : public BatchReader { + public: + IoUringBatchReader() = default; + + bool init(unsigned entries) { + struct io_uring_params p{}; + fd_ = (int)syscall(__NR_io_uring_setup, entries, &p); + if (fd_ < 0) return false; + sq_size_ = p.sq_off.array + p.sq_entries * sizeof(uint32_t); + cq_size_ = p.cq_off.cqes + p.cq_entries * sizeof(io_uring_cqe); + if (p.features & IORING_FEAT_SINGLE_MMAP) sq_size_ = cq_size_ = std::max(sq_size_, cq_size_); + sq_ptr_ = mmap(nullptr, sq_size_, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, fd_, + IORING_OFF_SQ_RING); + if (sq_ptr_ == MAP_FAILED) return false; + cq_ptr_ = (p.features & IORING_FEAT_SINGLE_MMAP) + ? sq_ptr_ + : mmap(nullptr, cq_size_, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE, + fd_, IORING_OFF_CQ_RING); + if (cq_ptr_ == MAP_FAILED) return false; + sqes_size_ = p.sq_entries * sizeof(io_uring_sqe); + sqes_ = (io_uring_sqe *)mmap(nullptr, sqes_size_, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, fd_, IORING_OFF_SQES); + if (sqes_ == MAP_FAILED) return false; + + auto at = [&](void *base, uint32_t off) { return (uint8_t *)base + off; }; + sq_tail_ = (uint32_t *)at(sq_ptr_, p.sq_off.tail); + sq_mask_ = (uint32_t *)at(sq_ptr_, p.sq_off.ring_mask); + sq_array_ = (uint32_t *)at(sq_ptr_, p.sq_off.array); + cq_head_ = (uint32_t *)at(cq_ptr_, p.cq_off.head); + cq_tail_ = (uint32_t *)at(cq_ptr_, p.cq_off.tail); + cq_mask_ = (uint32_t *)at(cq_ptr_, p.cq_off.ring_mask); + cqes_ = (io_uring_cqe *)at(cq_ptr_, p.cq_off.cqes); + entries_ = p.sq_entries; + lens_.assign(entries_, 0); + sq_shadow_tail_ = *sq_tail_; + return true; + } + + ~IoUringBatchReader() override { + if (sqes_ && sqes_ != MAP_FAILED) munmap(sqes_, sqes_size_); + if (cq_ptr_ && cq_ptr_ != MAP_FAILED && cq_ptr_ != sq_ptr_) munmap(cq_ptr_, cq_size_); + if (sq_ptr_ && sq_ptr_ != MAP_FAILED) munmap(sq_ptr_, sq_size_); + if (fd_ >= 0) ::close(fd_); + } + + std::string name() const override { return "io_uring"; } + unsigned capacity() const override { return entries_; } + + void submit(unsigned tag, int fd, uint8_t *buf, int64_t len, int64_t need, + int64_t off) override { + io_uring_sqe *sqe = &sqes_[sq_shadow_tail_ & *sq_mask_]; + std::memset(sqe, 0, sizeof(*sqe)); + sqe->opcode = IORING_OP_READ; + sqe->fd = fd; + sqe->addr = (uint64_t)(uintptr_t)buf; + sqe->len = (uint32_t)len; + sqe->off = (uint64_t)off; + sqe->user_data = tag; + sq_array_[sq_shadow_tail_ & *sq_mask_] = sq_shadow_tail_ & *sq_mask_; + sq_shadow_tail_++; + __atomic_store_n(sq_tail_, sq_shadow_tail_, __ATOMIC_RELEASE); + lens_[tag] = need; + to_submit_++; + in_flight_++; + } + + unsigned wait_one() override { + for (;;) { + uint32_t head = *cq_head_; + uint32_t ctail = __atomic_load_n(cq_tail_, __ATOMIC_ACQUIRE); + if (head != ctail) { + const io_uring_cqe &cqe = cqes_[head & *cq_mask_]; + const unsigned tag = (unsigned)cqe.user_data; + const int res = cqe.res; + __atomic_store_n(cq_head_, head + 1, __ATOMIC_RELEASE); + in_flight_--; + if (res < 0) + throw std::runtime_error(std::string("io_uring read: ") + std::strerror(-res)); + if (res < lens_[tag]) throw std::runtime_error("io_uring short read"); + return tag; + } + const unsigned to_submit = to_submit_; + long rc = syscall(__NR_io_uring_enter, fd_, to_submit, 1, IORING_ENTER_GETEVENTS, nullptr, 0); + if (rc < 0) { + if (errno == EINTR) continue; + throw std::runtime_error(std::string("io_uring_enter: ") + std::strerror(errno)); + } + // partial submission is legal (signal, transient alloc); the rest stay in the ring + to_submit_ = to_submit - (unsigned)rc; + } + } + + void drain() noexcept override { + while (in_flight_ > 0) { + uint32_t head = *cq_head_; + uint32_t ctail = __atomic_load_n(cq_tail_, __ATOMIC_ACQUIRE); + if (head != ctail) { + __atomic_store_n(cq_head_, head + 1, __ATOMIC_RELEASE); + in_flight_--; + continue; + } + const unsigned to_submit = to_submit_; + long rc = syscall(__NR_io_uring_enter, fd_, to_submit, 1, IORING_ENTER_GETEVENTS, nullptr, 0); + if (rc < 0) { + if (errno != EINTR) return; + continue; + } + to_submit_ = to_submit - (unsigned)rc; + } + } + + private: + int fd_ = -1; + void *sq_ptr_ = nullptr, *cq_ptr_ = nullptr; + io_uring_sqe *sqes_ = nullptr; + size_t sq_size_ = 0, cq_size_ = 0, sqes_size_ = 0; + uint32_t *sq_tail_ = nullptr, *sq_mask_ = nullptr, *sq_array_ = nullptr; + uint32_t *cq_head_ = nullptr, *cq_tail_ = nullptr, *cq_mask_ = nullptr; + io_uring_cqe *cqes_ = nullptr; + unsigned entries_ = 0; + uint32_t sq_shadow_tail_ = 0; + unsigned to_submit_ = 0; + unsigned in_flight_ = 0; + std::vector lens_; +}; + +#endif // PLE_HAS_IO_URING + +std::unique_ptr make_batch_reader(bool use_io_uring) { +#if PLE_HAS_IO_URING + if (use_io_uring) { + auto ring = std::make_unique(); + if (ring->init(kBatchEntries)) return ring; + } +#else + (void)use_io_uring; +#endif + return std::make_unique(); +} + +// ---- row store (platform-free) ---- + +int64_t wrap_mul(int64_t a, int64_t b) { + return (int64_t)((uint64_t)a * (uint64_t)b); +} + +int64_t pos_mod(int64_t v, int64_t m) { + int64_t r = v % m; + return r < 0 ? r + m : r; +} + +class PleStore { + struct Extent { + const TableFile *file; + int64_t base; + }; + + public: + PleStore(std::vector paths, std::vector extent_file, + std::vector extent_base, int64_t rows_per_extent, int64_t row_bytes, + int64_t row_stride, std::vector multipliers, std::vector head_vocab_sizes, + std::vector head_offsets, int64_t eos_token_id, bool use_io_uring) + : row_bytes_(row_bytes), + row_stride_(row_stride), + rows_per_extent_(rows_per_extent), + mult_(std::move(multipliers)), + sizes_(std::move(head_vocab_sizes)), + offsets_(std::move(head_offsets)), + eos_(eos_token_id) { + if (mult_.size() != 3 || sizes_.size() != offsets_.size() || sizes_.empty()) + throw std::runtime_error("PLE hash geometry: want 3 multipliers and equal-length head tables"); + if (row_bytes_ > kPage) + throw std::runtime_error("PLE row_bytes " + std::to_string(row_bytes_) + + " exceeds a page; bounce slots assume one-page rows"); + for (const std::string &p : paths) + files_.push_back(std::make_unique(p)); + const int64_t extent_bytes = (rows_per_extent_ - 1) * row_stride_ + row_bytes_; + for (size_t e = 0; e < extent_file.size(); e++) { + const size_t fi = (size_t)extent_file.at(e); + const int64_t base = extent_base.at(e); + if (base + extent_bytes > files_.at(fi)->size()) + throw std::runtime_error(paths[fi] + ": extent needs " + + std::to_string(base + extent_bytes) + " bytes, file has " + + std::to_string(files_[fi]->size())); + extents_.push_back(Extent{files_[fi].get(), base}); + } + reader_ = make_batch_reader(use_io_uring); + bounce_ = page_aligned_alloc((size_t)reader_->capacity() * kSpanMax); + } + + // reader first: a still-running read must not land in freed bounce memory + ~PleStore() { + reader_.reset(); + free(bounce_); + } + + PleStore(const PleStore &) = delete; + PleStore &operator=(const PleStore &) = delete; + + // Row ids for the token at w[2] with context (w[0], w[1]); mirrors NGramEmbedding.row_ids incl. the eos barrier. + void hash_rows(const int64_t *w, int64_t *rows) { + const int64_t prev1 = w[1]; + const int64_t prev2 = prev1 == eos_ ? eos_ : w[0]; + const int64_t bigram = wrap_mul(w[2], mult_[0]) ^ wrap_mul(prev1, mult_[1]); + const int64_t trigram = bigram ^ wrap_mul(prev2, mult_[2]); + const size_t half = sizes_.size() / 2; + for (size_t h = 0; h < sizes_.size(); h++) + rows[h] = pos_mod(h < half ? bigram : trigram, sizes_[h]) + offsets_[h]; + } + + // Hash and queue one run: tokens_addr holds n+2 ids, the leading two are context. No I/O until flush(). + void stage(uintptr_t tokens_addr, int64_t n, uintptr_t staging_addr) { + const int64_t *tokens = reinterpret_cast(tokens_addr); + uint8_t *staging = reinterpret_cast(staging_addr); + const size_t heads = sizes_.size(); + std::vector rows(heads); + for (int64_t i = 0; i < n; i++) { + hash_rows(tokens + i, rows.data()); + for (size_t h = 0; h < heads; h++) + request_row(rows[h], staging + ((size_t)i * heads + h) * row_bytes_); + } + } + + // One batched disk round for everything staged; signals even when nothing was. + void flush(uintptr_t signal_addr) { + flush_pending(); + if (signal_addr) signal_flag(signal_addr); + } + + std::string io_backend() const { + size_t direct = 0; + for (const auto &f : files_) direct += f->direct_io() ? 1 : 0; + std::string s = reader_->name(); + if (direct == files_.size()) return s + ", O_DIRECT"; + return s + ", buffered " + std::to_string(files_.size() - direct) + "/" + + std::to_string(files_.size()) + " files"; + } + + private: + struct Pending { + const TableFile *file; + int64_t read_off; + int64_t read_len; + int64_t row_off; // row payload start inside the read buffer + std::vector dsts; + }; + + // Queue dst on this fill's pending batch; duplicate rows fan out from one read. + void request_row(int64_t row_id, uint8_t *dst) { + auto pit = pending_index_.find(row_id); + if (pit != pending_index_.end()) { + pending_[pit->second].dsts.push_back(dst); + return; + } + const Extent &ext = extents_[row_id / rows_per_extent_]; + const int64_t off = ext.base + (row_id % rows_per_extent_) * row_stride_; + Pending p{ext.file, off, row_bytes_, 0, {dst}}; + if (ext.file->direct_io()) { + // full aligned span even past EOF; truncating would break direct-I/O alignment + p.read_off = off & ~(kPage - 1); + p.row_off = off - p.read_off; + p.read_len = ((off + row_bytes_ + kPage - 1) & ~(kPage - 1)) - p.read_off; + } + pending_index_.emplace(row_id, pending_.size()); + pending_.push_back(std::move(p)); + } + + // Read every pending row in reader-capacity batches and fan out the copies. + void flush_pending() { + if (pending_.empty()) return; + struct Cleanup { + PleStore *s; + ~Cleanup() { + s->pending_.clear(); + s->pending_index_.clear(); + } + } cleanup{this}; + + const unsigned cap = reader_->capacity(); + const size_t total = pending_.size(); + std::vector tag_pending(cap); + size_t next = 0; + auto submit_slot = [&](unsigned tag) { + const Pending &p = pending_[next]; + tag_pending[tag] = next++; + reader_->submit(tag, p.file->native_fd(), bounce_ + (size_t)tag * kSpanMax, p.read_len, + p.row_off + row_bytes_, p.read_off); + }; + try { + for (unsigned tag = 0; tag < std::min((size_t)cap, total); tag++) submit_slot(tag); + for (size_t completed = 0; completed < total; completed++) { + const unsigned tag = reader_->wait_one(); + const Pending &p = pending_[tag_pending[tag]]; + const uint8_t *row = bounce_ + (size_t)tag * kSpanMax + p.row_off; + for (uint8_t *dst : p.dsts) std::memcpy(dst, row, row_bytes_); + p.file->discard_cache(p.read_off, p.read_len); + if (next < total) submit_slot(tag); + } + } catch (...) { + reader_->drain(); + throw; + } + } + + int64_t row_bytes_, row_stride_, rows_per_extent_; + std::vector mult_, sizes_, offsets_; + int64_t eos_; + std::vector> files_; + std::vector extents_; + std::unique_ptr reader_; + uint8_t *bounce_ = nullptr; + + std::vector pending_; + std::unordered_map pending_index_; +}; + +} // namespace + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + py::class_(m, "PleStore") + .def(py::init, std::vector, std::vector, + int64_t, int64_t, int64_t, std::vector, + std::vector, std::vector, int64_t, bool>(), + py::arg("paths"), py::arg("extent_file"), py::arg("extent_base"), + py::arg("rows_per_extent"), py::arg("row_bytes"), py::arg("row_stride"), + py::arg("multipliers"), + py::arg("head_vocab_sizes"), py::arg("head_offsets"), py::arg("eos_token_id"), + py::arg("use_io_uring") = true) + .def("stage", &PleStore::stage, py::arg("tokens_addr"), py::arg("n"), + py::arg("staging_addr"), py::call_guard()) + .def("flush", &PleStore::flush, py::arg("signal_addr") = 0, + py::call_guard()) + .def("io_backend", &PleStore::io_backend); + m.def("memop_write", &memop_write, py::arg("stream"), py::arg("addr"), py::arg("value")); + m.def("memop_wait_geq", &memop_wait_geq, py::arg("stream"), py::arg("addr"), py::arg("value")); + m.def("memop_wait_reset", &memop_wait_reset, py::arg("stream"), py::arg("flag_addr")); + m.def("signal_flag", &signal_flag, py::arg("flag_addr")); +} diff --git a/python/freetoken/kernel/fast_index_copy.py b/python/freetoken/kernel/fast_index_copy.py index 1aaa1303d2..88e53a3c62 100644 --- a/python/freetoken/kernel/fast_index_copy.py +++ b/python/freetoken/kernel/fast_index_copy.py @@ -15,7 +15,32 @@ DEFAULT_NUM_BLOCKS = 4 SKIP_FAST_INDEX_COPY_ENV = "FREETOKEN_SKIP_FAST_INDEX_COPY" +FUSED_COPY_BLOCKS_PER_BANK_ENV = "FREETOKEN_FUSED_COPY_BLOCKS_PER_BANK" _TRUE_VALUES = {"1", "true", "yes", "on"} +# The legacy per-bank C++ kernel issues one 128-byte vectorized transaction per +# worker iteration. The fused multi-bank path has a tail-aware implementation +# and supports every 16-byte-aligned bank row, but this legacy specialization +# cannot represent a 240- or 400-byte worker row. +_LEGACY_COPY_ITERATION_BYTES = 128 + + +def fused_copy_blocks_per_bank() -> int: + """Return the AOT-compiled fused-copy grid width selected for one cache bank. + + Eight blocks is the established default. Sixty-four blocks widens the same + vector-copy grid without changing indices, source bytes, destination slots, + or arithmetic. Restricting the setting to the two cache-built variants keeps + strict no-JIT launches reproducible on gfx1151. + """ + + raw = os.getenv(FUSED_COPY_BLOCKS_PER_BANK_ENV, "8") + try: + value = int(raw) + except ValueError as exc: + raise ValueError(f"{FUSED_COPY_BLOCKS_PER_BANK_ENV} must be 8 or 64, got {raw!r}") from exc + if value not in (8, 64): + raise ValueError(f"{FUSED_COPY_BLOCKS_PER_BANK_ENV} must be 8 or 64, got {value}") + return value def _skip_fast_index_copy_enabled() -> bool: @@ -88,6 +113,21 @@ def default_worker_args(feature_size: int) -> tuple[int, int, int]: ) +def legacy_fast_index_copy_is_supported(feature_size: int) -> bool: + """Return whether the legacy per-bank template can represent ``feature_size``. + + ``FastIndexCopyKernel`` has no scalar tail: every worker copies an integral + number of 128-byte transactions. The fused multi-bank production path does + support smaller 16-byte-aligned rows, so this predicate only controls AOT + generation for the unused legacy fallback. Keeping the condition beside the + runtime argument derivation prevents the cache catalog from emitting a HIP + specialization that fails its own compile-time assertion. + """ + + _, worker_feature_size, _ = default_worker_args(feature_size) + return worker_feature_size % _LEGACY_COPY_ITERATION_BYTES == 0 + + def fast_index_copy_jit( dst: torch.Tensor, dst_indices: torch.Tensor, @@ -159,7 +199,7 @@ def fast_index_copy_multi_jit( num_indices: torch.Tensor | None = None, *, num_threads: int = 1024, - blocks_per_bank: int = 8, + blocks_per_bank: int | None = None, ) -> None: """Fused multi-bank index copy: copy the same rows for every bank in ONE launch. @@ -179,8 +219,11 @@ def fast_index_copy_multi_jit( """ if _skip_fast_index_copy_enabled(): return + selected_blocks = fused_copy_blocks_per_bank() if blocks_per_bank is None else blocks_per_bank + if selected_blocks not in (8, 64): + raise ValueError(f"blocks_per_bank must be 8 or 64, got {selected_blocks}") module = _jit_fast_index_copy_multi_module( - num_threads=num_threads, blocks_per_bank=blocks_per_bank + num_threads=num_threads, blocks_per_bank=selected_blocks ) module.launch(dst_ptrs, src_ptrs, feat_bytes, dst_indices, src_indices, num_indices) diff --git a/python/freetoken/kernel/fla/__init__.py b/python/freetoken/kernel/fla/__init__.py index e3fff01842..4d5320778d 100644 --- a/python/freetoken/kernel/fla/__init__.py +++ b/python/freetoken/kernel/fla/__init__.py @@ -21,15 +21,37 @@ stripped/inlined on vendoring). Keep ``chunk_delta_h.py``'s single fixed ``triton.Config`` — restoring upstream's multi-config autotune corrupts the in-place state pool. Tune via the env knobs ``SGLANG_GDN_CHUNK_H_BV`` / ``SGLANG_GDN_CHUNK_H_NUM_WARPS`` / ``SGLANG_GDN_CHUNK_H_NUM_STAGES``. + +KDA (GLM-5.3-Flash Kimi Delta Attention) kernels are vendored separately from vLLM's +``third_party/flash_linear_attention`` (same fla lineage): ``kda.py`` (chunked prefill + +fused gate cumsum + recurrent decode wrapper), ``kda_chunk_delta_h.py`` (exp2-gate chunk +recurrence), ``fused_recurrent.py`` (pool-indexed decode kernel with in-kernel KDA gate), +``solve_tril.py``. Public entry points: +- ``chunk_kda_with_fused_gate`` -- chunked prefill from raw gate logits; gathered initial + state in, final state out (scatter back to the pool is the caller's job). + WARNING: clobbers the ``v`` argument (the output is written into that buffer to save + an allocation, as in vLLM where v is an ephemeral projection). Never pass a tensor + that is read again afterwards. +- ``fused_recurrent_kda`` -- decode; per-slot state read/write via ``ssm_state_indices``, + gate + beta-sigmoid + q/k l2norm computed in-kernel. The per-token state store reads + ``ssm_state_indices`` as a CONTIGUOUS [N, T] block; materialize, never ``expand()``. """ from freetoken.kernel.fla.chunk import chunk_gated_delta_rule from freetoken.kernel.fla.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) +from freetoken.kernel.fla.kda import ( + chunk_kda_with_fused_gate, + fused_kda_gate, + fused_recurrent_kda, +) from freetoken.kernel.fla.layernorm_gated import rms_norm_gated __all__ = [ "chunk_gated_delta_rule", "fused_sigmoid_gating_delta_rule_update", + "chunk_kda_with_fused_gate", + "fused_kda_gate", + "fused_recurrent_kda", "rms_norm_gated", ] diff --git a/python/freetoken/kernel/fla/fused_recurrent.py b/python/freetoken/kernel/fla/fused_recurrent.py new file mode 100644 index 0000000000..cc15fef983 --- /dev/null +++ b/python/freetoken/kernel/fla/fused_recurrent.py @@ -0,0 +1,659 @@ +# Vendored from vLLM's third_party/flash_linear_attention (PR #53906, commit 933876c3), +# itself copied from the flash-linear-attention project (MIT, (c) 2023-2025 Songlin Yang, +# Yu Zhang). Imports are remapped onto freetoken.kernel.fla's shared helpers; keep this +# file in sync with upstream when pulling KDA kernel fixes. +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang +# +# This file contains code copied from the flash-linear-attention project. +# The original source code was licensed under the MIT license and included +# the following copyright notice: +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang +# ruff: noqa: E501 + +import torch + +import triton +import triton.language as tl + +from .op import exp, log + + +@triton.heuristics( + { + "USE_INITIAL_STATE": lambda args: args["h0"] is not None, + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + "IS_CONTINUOUS_BATCHING": lambda args: args["ssm_state_indices"] is not None, + "IS_SPEC_DECODING": lambda args: args["num_accepted_tokens"] is not None, + } +) +@triton.jit(do_not_specialize=["N", "T"]) +def fused_recurrent_gated_delta_rule_fwd_kernel( + q, + k, + v, + g, + beta, + o, + h0, + ht, + cu_seqlens, + ssm_state_indices, + num_accepted_tokens, + a_log, + g_bias, + scale, + N: tl.int64, # num of sequences + T: tl.int64, # num of tokens + B: tl.constexpr, + H: tl.constexpr, + HV: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + stride_init_state_token: tl.constexpr, + stride_final_state_token: tl.constexpr, + stride_indices_seq: tl.constexpr, + stride_indices_tok: tl.constexpr, + USE_INITIAL_STATE: tl.constexpr, # whether to use initial state + INPLACE_FINAL_STATE: tl.constexpr, # whether to store final state inplace + IS_BETA_HEADWISE: tl.constexpr, # whether beta is headwise vector or scalar, + USE_QK_L2NORM_IN_KERNEL: tl.constexpr, + IS_VARLEN: tl.constexpr, + IS_CONTINUOUS_BATCHING: tl.constexpr, + IS_SPEC_DECODING: tl.constexpr, + IS_KDA: tl.constexpr, + SIGMOID_BETA: tl.constexpr, # beta holds raw logits; sigmoid at fp32 load + COMPUTE_GATE: tl.constexpr, # g holds raw logits; KDA gate computed in-kernel + SAFE_GATE: tl.constexpr, # bounded gate variant (only branch implemented) + LOWER_BOUND: tl.constexpr, +): + i_k, i_v, i_nh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_n, i_hv = i_nh // HV, i_nh % HV + i_h = i_hv // (HV // H) + if IS_VARLEN: + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int64), + tl.load(cu_seqlens + i_n + 1).to(tl.int64), + ) + all = T + T = eos - bos + else: + bos, eos = i_n * T, i_n * T + T + all = B * T + + if T == 0: + # no tokens to process for this sequence + return + + o_k = i_k * BK + tl.arange(0, BK) + o_v = i_v * BV + tl.arange(0, BV) + + p_q = q + (bos * H + i_h) * K + o_k + p_k = k + (bos * H + i_h) * K + o_k + p_v = v + (bos * HV + i_hv) * V + o_v + if IS_BETA_HEADWISE: + p_beta = beta + (bos * HV + i_hv) * V + o_v + else: + p_beta = beta + bos * HV + i_hv + + if not IS_KDA: + p_g = g + bos * HV + i_hv + else: + p_gk = g + (bos * HV + i_hv) * K + o_k + + # Per-head gate amplitude, hoisted out of the token loop (COMPUTE_GATE). + if COMPUTE_GATE: + b_a_log = tl.exp(tl.load(a_log + i_h).to(tl.float32)) + + p_o = o + ((i_k * all + bos) * HV + i_hv) * V + o_v + + mask_k = o_k < K + mask_v = o_v < V + mask_h = mask_v[:, None] & mask_k[None, :] + + b_h = tl.zeros([BV, BK], dtype=tl.float32) + if USE_INITIAL_STATE: + if IS_CONTINUOUS_BATCHING: + if IS_SPEC_DECODING: + i_t = tl.load(num_accepted_tokens + i_n).to(tl.int64) - 1 + else: + i_t = 0 + # Load state index and check for invalid entries + state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq + i_t).to( + tl.int64 + ) + # DIVERGENCE from upstream: vLLM treats slot 0 as its NULL_BLOCK_ID + # sentinel (`state_idx <= 0`), which silently skips a real request that + # keys state by raw table_idx == 0 (--cache-type naive). FreeToken's + # GDN kernel has no sentinel; match it -- only negative ids are invalid. + # Under hybrid, padded rows now read/write the reserved padding slot + # instead of early-returning (a benign write-only sink, same as GDN). + if state_idx < 0: + return + p_h0 = h0 + state_idx * stride_init_state_token + else: + p_h0 = h0 + bos * HV * V * K + p_h0 = p_h0 + i_hv * V * K + o_v[:, None] * K + o_k[None, :] + b_h += tl.load(p_h0, mask=mask_h, other=0).to(tl.float32) + + for i_t in range(0, T): + b_q = tl.load(p_q, mask=mask_k, other=0).to(tl.float32) + b_k = tl.load(p_k, mask=mask_k, other=0).to(tl.float32) + b_v = tl.load(p_v, mask=mask_v, other=0).to(tl.float32) + + if USE_QK_L2NORM_IN_KERNEL: + b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6) + b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6) + b_q = b_q * scale + # [BV, BK] + if not IS_KDA: + b_g = tl.load(p_g).to(tl.float32) + b_h *= exp(b_g) + else: + b_gk = tl.load(p_gk).to(tl.float32) + if COMPUTE_GATE: + # Replicates kda_gate_fwd_kernel's SAFE_GATE branch + # bit-for-bit (same tl.exp, same fp32 math; the intermediate + # gate value this replaces was stored/reloaded as fp32, + # which is lossless): y = lb / (1 + exp(-exp(A)*(g+bias))). + b_gk += tl.load( + g_bias + i_h * K + o_k, mask=mask_k, other=0.0 + ).to(tl.float32) + b_gk = LOWER_BOUND / (1.0 + tl.exp(-(b_a_log * b_gk))) + b_h *= exp(b_gk[None, :]) + # [BV] + b_v -= tl.sum(b_h * b_k[None, :], 1) + if IS_BETA_HEADWISE: + b_beta = tl.load(p_beta, mask=mask_v, other=0).to(tl.float32) + else: + b_beta = tl.load(p_beta).to(tl.float32) + # Matches torch's `x.float().sigmoid()` pre-computation bit-for-bit + # on the input side (bf16->fp32 is exact); only the sigmoid impl itself + # can differ by <=1 ULP. + if SIGMOID_BETA: + b_beta = tl.sigmoid(b_beta) + b_v *= b_beta + # [BV, BK] + b_h += b_v[:, None] * b_k[None, :] + # [BV] + b_o = tl.sum(b_h * b_q[None, :], 1) + tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v) + + # keep the states for multi-query tokens + if INPLACE_FINAL_STATE: + # Load state index and check for invalid entries + final_state_idx = tl.load( + ssm_state_indices + i_n * stride_indices_seq + i_t + ).to(tl.int64) + # DIVERGENCE from upstream: no slot-0 sentinel (see the load-side note). + if final_state_idx >= 0: + p_ht = ht + final_state_idx * stride_final_state_token + p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :] + tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h) + else: + p_ht = ht + (bos + i_t) * stride_final_state_token + p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :] + tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h) + + p_q += H * K + p_k += H * K + p_o += HV * V + p_v += HV * V + if not IS_KDA: + p_g += HV + else: + p_gk += HV * K + p_beta += HV * (V if IS_BETA_HEADWISE else 1) + + +def fused_recurrent_gated_delta_rule_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state: torch.Tensor, + inplace_final_state: bool = True, + cu_seqlens: torch.Tensor | None = None, + ssm_state_indices: torch.Tensor | None = None, + num_accepted_tokens: torch.Tensor | None = None, + use_qk_l2norm_in_kernel: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + B, T, H, K, V = *k.shape, v.shape[-1] + HV = v.shape[2] + N = B if cu_seqlens is None else len(cu_seqlens) - 1 + BK, BV = triton.next_power_of_2(K), min(triton.next_power_of_2(V), 32) + NK, NV = triton.cdiv(K, BK), triton.cdiv(V, BV) + assert NK == 1, "NK > 1 is not supported yet" + num_stages = 3 + num_warps = 1 + + o = q.new_empty(NK, *v.shape) + if inplace_final_state: + final_state = initial_state + else: + final_state = q.new_empty(T, HV, V, K, dtype=initial_state.dtype) + + stride_init_state_token = initial_state.stride(0) + stride_final_state_token = final_state.stride(0) + + if ssm_state_indices is None: + stride_indices_seq, stride_indices_tok = 1, 1 + elif ssm_state_indices.ndim == 1: + stride_indices_seq, stride_indices_tok = ssm_state_indices.stride(0), 1 + else: + stride_indices_seq, stride_indices_tok = ssm_state_indices.stride() + + grid = (NK, NV, N * HV) + fused_recurrent_gated_delta_rule_fwd_kernel[grid]( + q=q, + k=k, + v=v, + g=g, + beta=beta, + o=o, + h0=initial_state, + ht=final_state, + cu_seqlens=cu_seqlens, + ssm_state_indices=ssm_state_indices, + num_accepted_tokens=num_accepted_tokens, + scale=scale, + N=N, + T=T, + B=B, + H=H, + HV=HV, + K=K, + V=V, + BK=BK, + BV=BV, + stride_init_state_token=stride_init_state_token, + stride_final_state_token=stride_final_state_token, + stride_indices_seq=stride_indices_seq, + stride_indices_tok=stride_indices_tok, + IS_BETA_HEADWISE=beta.ndim == v.ndim, + USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, + INPLACE_FINAL_STATE=inplace_final_state, + IS_KDA=False, + SIGMOID_BETA=False, + a_log=None, + g_bias=None, + COMPUTE_GATE=False, + SAFE_GATE=True, + LOWER_BOUND=-5.0, + num_warps=num_warps, + num_stages=num_stages, + ) + o = o.squeeze(0) + return o, final_state + + +@triton.jit +def fused_recurrent_gated_delta_rule_packed_decode_kernel( + mixed_qkv, + a, + b, + A_log, + dt_bias, + o, + h0, + ht, + ssm_state_indices, + scale, + stride_mixed_qkv_tok: tl.constexpr, + stride_a_tok: tl.constexpr, + stride_b_tok: tl.constexpr, + stride_init_state_token: tl.constexpr, + stride_final_state_token: tl.constexpr, + stride_indices_seq: tl.constexpr, + H: tl.constexpr, + HV: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + SOFTPLUS_THRESHOLD: tl.constexpr, + USE_QK_L2NORM_IN_KERNEL: tl.constexpr, +): + i_v, i_nh = tl.program_id(0), tl.program_id(1) + i_n, i_hv = i_nh // HV, i_nh % HV + i_h = i_hv // (HV // H) + + o_k = tl.arange(0, BK) + o_v = i_v * BV + tl.arange(0, BV) + mask_k = o_k < K + mask_v = o_v < V + mask_h = mask_v[:, None] & mask_k[None, :] + + state_idx = tl.load(ssm_state_indices + i_n * stride_indices_seq).to(tl.int64) + p_o = o + (i_n * HV + i_hv) * V + o_v + + # Skip if state index is invalid (NULL_BLOCK_ID=0) + if state_idx <= 0: + zero = tl.zeros([BV], dtype=tl.float32).to(p_o.dtype.element_ty) + tl.store(p_o, zero, mask=mask_v) + return + + p_h0 = h0 + state_idx * stride_init_state_token + p_h0 = p_h0 + i_hv * V * K + o_v[:, None] * K + o_k[None, :] + b_h = tl.load(p_h0, mask=mask_h, other=0).to(tl.float32) + + p_mixed = mixed_qkv + i_n * stride_mixed_qkv_tok + q_off = i_h * K + o_k + k_off = (H * K) + i_h * K + o_k + v_off = (2 * H * K) + i_hv * V + o_v + b_q = tl.load(p_mixed + q_off, mask=mask_k, other=0).to(tl.float32) + b_k = tl.load(p_mixed + k_off, mask=mask_k, other=0).to(tl.float32) + b_v = tl.load(p_mixed + v_off, mask=mask_v, other=0).to(tl.float32) + + if USE_QK_L2NORM_IN_KERNEL: + b_q = b_q / tl.sqrt(tl.sum(b_q * b_q) + 1e-6) + b_k = b_k / tl.sqrt(tl.sum(b_k * b_k) + 1e-6) + b_q = b_q * scale + + a_val = tl.load(a + i_n * stride_a_tok + i_hv).to(tl.float32) + b_val = tl.load(b + i_n * stride_b_tok + i_hv).to(tl.float32) + A_log_val = tl.load(A_log + i_hv).to(tl.float32) + dt_bias_val = tl.load(dt_bias + i_hv).to(tl.float32) + x = a_val + dt_bias_val + softplus_x = tl.where(x <= SOFTPLUS_THRESHOLD, tl.log(1.0 + tl.exp(x)), x) + g_val = -tl.exp(A_log_val) * softplus_x + beta_val = tl.sigmoid(b_val).to(b.dtype.element_ty).to(tl.float32) + + b_h *= exp(g_val) + b_v -= tl.sum(b_h * b_k[None, :], 1) + b_v *= beta_val + b_h += b_v[:, None] * b_k[None, :] + b_o = tl.sum(b_h * b_q[None, :], 1) + tl.store(p_o, b_o.to(p_o.dtype.element_ty), mask=mask_v) + + p_ht = ht + state_idx * stride_final_state_token + p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :] + tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h) + + +def fused_recurrent_gated_delta_rule_packed_decode( + mixed_qkv: torch.Tensor, + a: torch.Tensor, + b: torch.Tensor, + A_log: torch.Tensor, + dt_bias: torch.Tensor, + scale: float, + initial_state: torch.Tensor, + out: torch.Tensor, + ssm_state_indices: torch.Tensor, + use_qk_l2norm_in_kernel: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + if mixed_qkv.ndim != 2: + raise ValueError( + f"`mixed_qkv` must be a 2D tensor (got ndim={mixed_qkv.ndim})." + ) + if mixed_qkv.stride(-1) != 1: + raise ValueError("`mixed_qkv` must be contiguous in the last dim.") + if a.ndim != 2 or b.ndim != 2: + raise ValueError( + f"`a` and `b` must be 2D tensors (got a.ndim={a.ndim}, b.ndim={b.ndim})." + ) + if a.stride(-1) != 1 or b.stride(-1) != 1: + raise ValueError("`a`/`b` must be contiguous in the last dim.") + if A_log.ndim != 1 or dt_bias.ndim != 1: + raise ValueError("`A_log`/`dt_bias` must be 1D tensors.") + if A_log.stride(0) != 1 or dt_bias.stride(0) != 1: + raise ValueError("`A_log`/`dt_bias` must be contiguous.") + if ssm_state_indices.ndim != 1: + raise ValueError( + f"`ssm_state_indices` must be 1D for packed decode (got ndim={ssm_state_indices.ndim})." + ) + if not out.is_contiguous(): + raise ValueError("`out` must be contiguous.") + + dev = mixed_qkv.device + if ( + a.device != dev + or b.device != dev + or A_log.device != dev + or dt_bias.device != dev + or initial_state.device != dev + or out.device != dev + or ssm_state_indices.device != dev + ): + raise ValueError("All inputs must be on the same device.") + + B = mixed_qkv.shape[0] + if a.shape[0] != B or b.shape[0] != B: + raise ValueError( + "Mismatched batch sizes: " + f"mixed_qkv.shape[0]={B}, a.shape[0]={a.shape[0]}, b.shape[0]={b.shape[0]}." + ) + if ssm_state_indices.shape[0] != B: + raise ValueError( + f"`ssm_state_indices` must have shape [B] (got {tuple(ssm_state_indices.shape)}; expected ({B},))." + ) + + if initial_state.ndim != 4: + raise ValueError( + f"`initial_state` must be a 4D tensor (got ndim={initial_state.ndim})." + ) + if initial_state.stride(-1) != 1: + raise ValueError("`initial_state` must be contiguous in the last dim.") + HV, V, K = initial_state.shape[-3:] + if a.shape[1] != HV or b.shape[1] != HV: + raise ValueError( + f"`a`/`b` must have shape [B, HV] with HV={HV} (got a.shape={tuple(a.shape)}, b.shape={tuple(b.shape)})." + ) + if A_log.numel() != HV or dt_bias.numel() != HV: + raise ValueError( + f"`A_log` and `dt_bias` must have {HV} elements (got A_log.numel()={A_log.numel()}, dt_bias.numel()={dt_bias.numel()})." + ) + if out.shape != (B, 1, HV, V): + raise ValueError( + f"`out` must have shape {(B, 1, HV, V)} (got out.shape={tuple(out.shape)})." + ) + + qkv_dim = mixed_qkv.shape[1] + qk_dim = qkv_dim - HV * V + if qk_dim <= 0 or qk_dim % 2 != 0: + raise ValueError( + f"Invalid packed `mixed_qkv` last dim={qkv_dim} for HV={HV}, V={V}." + ) + q_dim = qk_dim // 2 + if q_dim % K != 0: + raise ValueError(f"Invalid packed Q size {q_dim}: must be divisible by K={K}.") + H = q_dim // K + if H <= 0 or HV % H != 0: + raise ValueError( + f"Invalid head config inferred from mixed_qkv: H={H}, HV={HV}." + ) + + BK = triton.next_power_of_2(K) + if triton.cdiv(K, BK) != 1: + raise ValueError( + f"Packed decode kernel only supports NK=1 (got K={K}, BK={BK})." + ) + BV = min(triton.next_power_of_2(V), 32) + num_stages = 3 + num_warps = 1 + + stride_mixed_qkv_tok = mixed_qkv.stride(0) + stride_a_tok = a.stride(0) + stride_b_tok = b.stride(0) + stride_init_state_token = initial_state.stride(0) + stride_final_state_token = initial_state.stride(0) + stride_indices_seq = ssm_state_indices.stride(0) + + NV = triton.cdiv(V, BV) + grid = (NV, B * HV) + fused_recurrent_gated_delta_rule_packed_decode_kernel[grid]( + mixed_qkv=mixed_qkv, + a=a, + b=b, + A_log=A_log, + dt_bias=dt_bias, + o=out, + h0=initial_state, + ht=initial_state, + ssm_state_indices=ssm_state_indices, + scale=scale, + stride_mixed_qkv_tok=stride_mixed_qkv_tok, + stride_a_tok=stride_a_tok, + stride_b_tok=stride_b_tok, + stride_init_state_token=stride_init_state_token, + stride_final_state_token=stride_final_state_token, + stride_indices_seq=stride_indices_seq, + H=H, + HV=HV, + K=K, + V=V, + BK=BK, + BV=BV, + SOFTPLUS_THRESHOLD=20.0, + USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, + num_warps=num_warps, + num_stages=num_stages, + ) + return out, initial_state + + +class FusedRecurrentFunction(torch.autograd.Function): + @staticmethod + def forward( + ctx, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state: torch.Tensor, + inplace_final_state: bool = True, + cu_seqlens: torch.Tensor | None = None, + ssm_state_indices: torch.Tensor | None = None, + num_accepted_tokens: torch.Tensor | None = None, + use_qk_l2norm_in_kernel: bool = False, + ): + o, final_state = fused_recurrent_gated_delta_rule_fwd( + q=q.contiguous(), + k=k.contiguous(), + v=v.contiguous(), + g=g.contiguous(), + beta=beta.contiguous(), + scale=scale, + initial_state=initial_state, + inplace_final_state=inplace_final_state, + cu_seqlens=cu_seqlens, + ssm_state_indices=ssm_state_indices, + num_accepted_tokens=num_accepted_tokens, + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + ) + + return o, final_state + + +def fused_recurrent_gated_delta_rule( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor = None, + scale: float = None, + initial_state: torch.Tensor = None, + inplace_final_state: bool = True, + cu_seqlens: torch.Tensor | None = None, + ssm_state_indices: torch.Tensor | None = None, + num_accepted_tokens: torch.Tensor | None = None, + use_qk_l2norm_in_kernel: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + r""" + Args: + q (torch.Tensor): + queries of shape `[B, T, H, K]`. + k (torch.Tensor): + keys of shape `[B, T, H, K]`. + v (torch.Tensor): + values of shape `[B, T, HV, V]`. + GVA is applied if `HV > H`. + g (torch.Tensor): + g (decays) of shape `[B, T, HV]`. + beta (torch.Tensor): + betas of shape `[B, T, HV]`. + scale (Optional[int]): + Scale factor for the RetNet attention scores. + If not provided, it will default to `1 / sqrt(K)`. Default: `None`. + initial_state (Optional[torch.Tensor]): + Initial state of shape `[N, HV, V, K]` for `N` input sequences. + For equal-length input sequences, `N` equals the batch size `B`. + Default: `None`. + inplace_final_state: bool: + Whether to store the final state in-place to save memory. + Default: `True`. + cu_seqlens (torch.Tensor): + Cumulative sequence lengths of shape `[N+1]` used for variable-length training, + consistent with the FlashAttention API. + ssm_state_indices (Optional[torch.Tensor]): + Indices to map the input sequences to the initial/final states. + num_accepted_tokens (Optional[torch.Tensor]): + Number of accepted tokens for each sequence during decoding. + + Returns: + o (torch.Tensor): + Outputs of shape `[B, T, HV, V]`. + final_state (torch.Tensor): + Final state of shape `[N, HV, V, K]`. + + Examples:: + >>> import torch + >>> import torch.nn.functional as F + >>> from einops import rearrange + >>> from fla.ops.gated_delta_rule import fused_recurrent_gated_delta_rule + # inputs with equal lengths + >>> B, T, H, HV, K, V = 4, 2048, 4, 8, 512, 512 + >>> q = torch.randn(B, T, H, K, device='cuda') + >>> k = F.normalize(torch.randn(B, T, H, K, device='cuda'), p=2, dim=-1) + >>> v = torch.randn(B, T, HV, V, device='cuda') + >>> g = F.logsigmoid(torch.rand(B, T, HV, device='cuda')) + >>> beta = torch.rand(B, T, HV, device='cuda').sigmoid() + >>> h0 = torch.randn(B, HV, V, K, device='cuda') + >>> o, ht = fused_gated_recurrent_delta_rule( + q, k, v, g, beta, + initial_state=h0, + ) + # for variable-length inputs, the batch size `B` is expected to be 1 and `cu_seqlens` is required + >>> q, k, v, g, beta = map(lambda x: rearrange(x, 'b t ... -> 1 (b t) ...'), (q, k, v, g, beta)) + # for a batch with 4 sequences, `cu_seqlens` with 5 start/end positions are expected + >>> cu_seqlens = q.new_tensor([0, 2048, 4096, 6144, 8192], dtype=torch.int32) + >>> o_var, ht_var = fused_gated_recurrent_delta_rule( + q, k, v, g, beta, + initial_state=h0, + cu_seqlens=cu_seqlens + ) + """ + if cu_seqlens is not None and q.shape[0] != 1: + raise ValueError( + f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`." + f"Please flatten variable-length inputs before processing." + ) + if scale is None: + scale = k.shape[-1] ** -0.5 + else: + assert scale > 0, "scale must be positive" + if beta is None: + beta = torch.ones_like(q[..., 0]) + o, final_state = FusedRecurrentFunction.apply( + q, + k, + v, + g, + beta, + scale, + initial_state, + inplace_final_state, + cu_seqlens, + ssm_state_indices, + num_accepted_tokens, + use_qk_l2norm_in_kernel, + ) + return o, final_state diff --git a/python/freetoken/kernel/fla/kda.py b/python/freetoken/kernel/fla/kda.py new file mode 100644 index 0000000000..44abc8927e --- /dev/null +++ b/python/freetoken/kernel/fla/kda.py @@ -0,0 +1,1395 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang +# +# This file contains code copied from the flash-linear-attention project. +# The original source code was licensed under the MIT license and included +# the following copyright notice: +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang +# ruff: noqa: E501 + + +# Vendored from vLLM's third_party/flash_linear_attention (PR #53906, commit 933876c3), +# itself copied from the flash-linear-attention project (MIT, (c) 2023-2025 Songlin Yang, +# Yu Zhang). Imports are remapped onto freetoken.kernel.fla's shared helpers; keep this +# file in sync with upstream when pulling KDA kernel fixes. +import torch +import triton +import triton.language as tl + +from freetoken.kernel.fla.cumsum import chunk_local_cumsum +from freetoken.kernel.fla.fused_recurrent import ( + fused_recurrent_gated_delta_rule_fwd_kernel, +) +from freetoken.kernel.fla.index import prepare_chunk_indices +from freetoken.kernel.fla.kda_chunk_delta_h import chunk_gated_delta_rule_fwd_h +from freetoken.kernel.fla.l2norm import l2norm_fwd +from freetoken.kernel.fla.op import exp2, log +from freetoken.kernel.fla.solve_tril import solve_tril +from freetoken.kernel.fla.utils import FLA_CHUNK_SIZE, is_amd + +RCP_LN2 = 1.4426950408889634 # 1 / ln(2) +cdiv = triton.cdiv +next_power_of_2 = triton.next_power_of_2 + +BT_LIST_AUTOTUNE = [32, 64, 128] +NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32] + + +def fused_recurrent_kda_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state: torch.Tensor, + inplace_final_state: bool = True, + cu_seqlens: torch.Tensor | None = None, + ssm_state_indices: torch.Tensor | None = None, + num_accepted_tokens: torch.Tensor | None = None, + use_qk_l2norm_in_kernel: bool = False, + out: torch.Tensor | None = None, + sigmoid_beta: bool = False, + a_log: torch.Tensor | None = None, + g_bias: torch.Tensor | None = None, + compute_gate: bool = False, + lower_bound: float | None = -5.0, +) -> tuple[torch.Tensor, torch.Tensor]: + B, T, H, K, V = *k.shape, v.shape[-1] + HV = v.shape[2] + N = B if cu_seqlens is None else len(cu_seqlens) - 1 + BK, BV = next_power_of_2(K), min(next_power_of_2(V), 8) + NK, NV = cdiv(K, BK), cdiv(V, BV) + assert NK == 1, "NK > 1 is not supported yet" + num_stages = 3 + num_warps = 1 + + if compute_gate: + assert a_log is not None and g_bias is not None, ( + "compute_gate requires a_log and g_bias" + ) + assert lower_bound is not None, ( + "compute_gate implements the bounded (safe_gate) branch only" + ) + a_log = a_log.reshape(-1).contiguous() + g_bias = g_bias.reshape(-1).contiguous() + + if out is None: + o = torch.empty_like(k) + else: + # Caller-provided output buffer; must be layout-compatible with the + # tensor the kernel indexes (contiguous, same shape/dtype as k). + assert out.shape == k.shape and out.dtype == k.dtype + assert out.is_contiguous() + o = out + if inplace_final_state: + final_state = initial_state + else: + final_state = q.new_empty(T, HV, V, K, dtype=initial_state.dtype) + + stride_init_state_token = initial_state.stride(0) + stride_final_state_token = final_state.stride(0) + + if ssm_state_indices is None: + stride_indices_seq, stride_indices_tok = 1, 1 + elif ssm_state_indices.ndim == 1: + stride_indices_seq, stride_indices_tok = ssm_state_indices.stride(0), 1 + else: + stride_indices_seq, stride_indices_tok = ssm_state_indices.stride() + + grid = (NK, NV, N * HV) + fused_recurrent_gated_delta_rule_fwd_kernel[grid]( + q=q, + k=k, + v=v, + g=g, + beta=beta, + o=o, + h0=initial_state, + ht=final_state, + cu_seqlens=cu_seqlens, + ssm_state_indices=ssm_state_indices, + num_accepted_tokens=num_accepted_tokens, + scale=scale, + N=N, + T=T, + B=B, + H=H, + HV=HV, + K=K, + V=V, + BK=BK, + BV=BV, + stride_init_state_token=stride_init_state_token, + stride_final_state_token=stride_final_state_token, + stride_indices_seq=stride_indices_seq, + stride_indices_tok=stride_indices_tok, + IS_BETA_HEADWISE=beta.ndim == v.ndim, + USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, + INPLACE_FINAL_STATE=inplace_final_state, + IS_KDA=True, + SIGMOID_BETA=sigmoid_beta, + a_log=a_log, + g_bias=g_bias, + COMPUTE_GATE=compute_gate, + SAFE_GATE=True, + LOWER_BOUND=lower_bound if lower_bound is not None else -5.0, + num_warps=num_warps, + num_stages=num_stages, + ) + + return o, final_state + + +def fused_recurrent_kda( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor = None, + scale: float = None, + initial_state: torch.Tensor = None, + inplace_final_state: bool = True, + use_qk_l2norm_in_kernel: bool = True, + cu_seqlens: torch.Tensor | None = None, + ssm_state_indices: torch.LongTensor | None = None, + num_accepted_tokens: torch.Tensor | None = None, + out: torch.Tensor | None = None, + sigmoid_beta: bool = False, + a_log: torch.Tensor | None = None, + g_bias: torch.Tensor | None = None, + compute_gate: bool = False, + lower_bound: float | None = -5.0, + **kwargs, +) -> tuple[torch.Tensor, torch.Tensor]: + if cu_seqlens is not None and q.shape[0] != 1: + raise ValueError( + f"The batch size is expected to be 1 rather than {q.shape[0]} when using `cu_seqlens`." + f"Please flatten variable-length inputs before processing." + ) + if scale is None: + scale = k.shape[-1] ** -0.5 + + o, final_state = fused_recurrent_kda_fwd( + q=q.contiguous(), + k=k.contiguous(), + v=v.contiguous(), + g=g.contiguous(), + beta=beta.contiguous(), + scale=scale, + initial_state=initial_state, + inplace_final_state=inplace_final_state, + cu_seqlens=cu_seqlens, + ssm_state_indices=ssm_state_indices, + num_accepted_tokens=num_accepted_tokens, + use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, + out=out, + sigmoid_beta=sigmoid_beta, + a_log=a_log, + g_bias=g_bias, + compute_gate=compute_gate, + lower_bound=lower_bound, + ) + return o, final_state + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +@triton.autotune( + configs=[ + triton.Config({"BK": BK}, num_warps=num_warps, num_stages=num_stages) + for BK in [32, 64] + for num_warps in [1, 2, 4, 8] + for num_stages in [2, 3, 4] + ], + key=["BC"], +) +@triton.jit(do_not_specialize=["T"]) +def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter( + q, + k, + g, + beta, + A, + Aqk, + scale, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + K: tl.constexpr, + BT: tl.constexpr, + BC: tl.constexpr, + BK: tl.constexpr, + NC: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_t, i_c, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_b, i_h = i_bh // H, i_bh % H + i_i, i_j = i_c // NC, i_c % NC + if IS_VARLEN: + i_n, i_t = ( + tl.load(chunk_indices + i_t * 2).to(tl.int32), + tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32), + ) + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + + if i_t * BT + i_i * BC >= T: + return + if i_i <= i_j: + return + + q += (bos * H + i_h) * K + k += (bos * H + i_h) * K + g += (bos * H + i_h) * K + A += (bos * H + i_h) * BT + Aqk += (bos * H + i_h) * BT + + p_b = tl.make_block_ptr( + beta + bos * H + i_h, (T,), (H,), (i_t * BT + i_i * BC,), (BC,), (0,) + ) + b_b = tl.load(p_b, boundary_check=(0,)) + + b_A = tl.zeros([BC, BC], dtype=tl.float32) + b_Aqk = tl.zeros([BC, BC], dtype=tl.float32) + for i_k in range(tl.cdiv(K, BK)): + p_q = tl.make_block_ptr( + q, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0) + ) + p_k = tl.make_block_ptr( + k, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0) + ) + p_g = tl.make_block_ptr( + g, (T, K), (H * K, 1), (i_t * BT + i_i * BC, i_k * BK), (BC, BK), (1, 0) + ) + b_kt = tl.make_block_ptr( + k, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1) + ) + p_gk = tl.make_block_ptr( + g, (K, T), (1, H * K), (i_k * BK, i_t * BT + i_j * BC), (BK, BC), (0, 1) + ) + + o_k = i_k * BK + tl.arange(0, BK) + m_k = o_k < K + # [BK,] + b_gn = tl.load(g + (i_t * BT + i_i * BC) * H * K + o_k, mask=m_k, other=0) + # [BC, BK] + b_g = tl.load(p_g, boundary_check=(0, 1)) + b_k = tl.load(p_k, boundary_check=(0, 1)) * exp2(b_g - b_gn[None, :]) + # [BK, BC] + b_gk = tl.load(p_gk, boundary_check=(0, 1)) + b_kt = tl.load(b_kt, boundary_check=(0, 1)) + # [BC, BC] + b_ktg = b_kt * exp2(b_gn[:, None] - b_gk) + b_A += tl.dot(b_k, b_ktg) + + b_q = tl.load(p_q, boundary_check=(0, 1)) + b_qg = b_q * exp2(b_g - b_gn[None, :]) * scale + b_Aqk += tl.dot(b_qg, b_ktg) + + b_A *= b_b[:, None] + + p_A = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0) + ) + tl.store(p_A, b_A.to(A.dtype.element_ty), boundary_check=(0, 1)) + p_Aqk = tl.make_block_ptr( + Aqk, (T, BT), (H * BT, 1), (i_t * BT + i_i * BC, i_j * BC), (BC, BC), (1, 0) + ) + tl.store(p_Aqk, b_Aqk.to(Aqk.dtype.element_ty), boundary_check=(0, 1)) + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +@triton.autotune( + configs=[triton.Config({}, num_warps=num_warps) for num_warps in [1, 2, 4, 8]], + key=["BK", "BT"], +) +@triton.jit(do_not_specialize=["T"]) +def chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra( + q, + k, + g, + beta, + A, + Aqk, + scale, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + K: tl.constexpr, + BT: tl.constexpr, + BC: tl.constexpr, + BK: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_t, i_i, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = ( + tl.load(chunk_indices + i_t * 2).to(tl.int32), + tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32), + ) + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + + if i_t * BT + i_i * BC >= T: + return + + o_i = tl.arange(0, BC) + o_k = tl.arange(0, BK) + m_k = o_k < K + m_A = (i_t * BT + i_i * BC + o_i) < T + o_A = (bos + i_t * BT + i_i * BC + o_i) * H * BT + i_h * BT + i_i * BC + + p_q = tl.make_block_ptr( + q + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT + i_i * BC, 0), + (BC, BK), + (1, 0), + ) + p_k = tl.make_block_ptr( + k + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT + i_i * BC, 0), + (BC, BK), + (1, 0), + ) + p_g = tl.make_block_ptr( + g + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT + i_i * BC, 0), + (BC, BK), + (1, 0), + ) + b_q = tl.load(p_q, boundary_check=(0, 1)) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_g = tl.load(p_g, boundary_check=(0, 1)) + + p_b = beta + (bos + i_t * BT + i_i * BC + o_i) * H + i_h + b_k = b_k * tl.load(p_b, mask=m_A, other=0)[:, None] + + p_kt = k + (bos + i_t * BT + i_i * BC) * H * K + i_h * K + o_k + p_gk = g + (bos + i_t * BT + i_i * BC) * H * K + i_h * K + o_k + + for j in range(0, min(BC, T - i_t * BT - i_i * BC)): + b_kt = tl.load(p_kt, mask=m_k, other=0).to(tl.float32) + b_gk = tl.load(p_gk, mask=m_k, other=0).to(tl.float32) + b_ktg = b_kt[None, :] * exp2(b_g - b_gk[None, :]) + b_A = tl.sum(b_k * b_ktg, 1) + b_A = tl.where(o_i > j, b_A, 0.0) + b_Aqk = tl.sum(b_q * b_ktg, 1) + b_Aqk = tl.where(o_i >= j, b_Aqk * scale, 0.0) + tl.store(A + o_A + j, b_A, mask=m_A) + tl.store(Aqk + o_A + j, b_Aqk, mask=m_A) + p_kt += H * K + p_gk += H * K + + +def chunk_kda_scaled_dot_kkt_fwd( + q: torch.Tensor, + k: torch.Tensor, + gk: torch.Tensor | None = None, + beta: torch.Tensor | None = None, + scale: float | None = None, + cu_seqlens: torch.Tensor | None = None, + chunk_indices: torch.Tensor | None = None, + chunk_size: int = FLA_CHUNK_SIZE, + output_dtype: torch.dtype = torch.float32, +) -> tuple[torch.Tensor, torch.Tensor]: + r""" + Compute beta * K * K^T. + + Args: + k (torch.Tensor): + The key tensor of shape `[B, T, H, K]`. + beta (torch.Tensor): + The beta tensor of shape `[B, T, H]`. + gk (torch.Tensor): + The cumulative sum of the gate tensor of shape `[B, T, H, K]` applied to the key tensor. Default: `None`. + cu_seqlens (torch.Tensor): + The cumulative sequence lengths of the input tensor. + Default: None + chunk_size (int): + The chunk size. Default: 64. + output_dtype (torch.dtype): + The dtype of the output tensor. Default: `torch.float32` + + Returns: + beta * K * K^T of shape `[B, T, H, BT]` where `BT` is the chunk size. + """ + B, T, H, K = k.shape + assert K <= 256 + BT = chunk_size + if chunk_indices is None and cu_seqlens is not None: + chunk_indices = prepare_chunk_indices(cu_seqlens, BT) + NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) + + BC = min(16, BT) + NC = cdiv(BT, BC) + BK = max(next_power_of_2(K), 16) + A = torch.zeros(B, T, H, BT, device=k.device, dtype=output_dtype) + Aqk = torch.zeros(B, T, H, BT, device=k.device, dtype=output_dtype) + grid = (NT, NC * NC, B * H) + chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_inter[grid]( + q=q, + k=k, + g=gk, + beta=beta, + A=A, + Aqk=Aqk, + scale=scale, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + H=H, + K=K, + BT=BT, + BC=BC, + NC=NC, + ) + + grid = (NT, NC, B * H) + chunk_kda_scaled_dot_kkt_fwd_kernel_intra_sub_intra[grid]( + q=q, + k=k, + g=gk, + beta=beta, + A=A, + Aqk=Aqk, + scale=scale, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + H=H, + K=K, + BT=BT, + BC=BC, + BK=BK, + ) + return A, Aqk + + +@triton.heuristics( + { + "STORE_QG": lambda args: args["qg"] is not None, + "STORE_KG": lambda args: args["kg"] is not None, + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + } +) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=num_warps, num_stages=num_stages) + for num_warps in [2, 4, 8] + for num_stages in [2, 3, 4] + ], + key=["H", "K", "V", "BT", "BK", "BV", "IS_VARLEN"], +) +@triton.jit(do_not_specialize=["T"]) +def recompute_w_u_fwd_kernel( + q, + k, + qg, + kg, + v, + beta, + w, + u, + A, + gk, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BT: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + STORE_QG: tl.constexpr, + STORE_KG: tl.constexpr, + IS_VARLEN: tl.constexpr, + DOT_PRECISION: tl.constexpr, +): + i_t, i_bh = tl.program_id(0), tl.program_id(1) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = ( + tl.load(chunk_indices + i_t * 2).to(tl.int32), + tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32), + ) + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + p_b = tl.make_block_ptr(beta + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,)) + b_b = tl.load(p_b, boundary_check=(0,)) + + p_A = tl.make_block_ptr( + A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0) + ) + b_A = tl.load(p_A, boundary_check=(0, 1)) + + for i_v in range(tl.cdiv(V, BV)): + p_v = tl.make_block_ptr( + v + (bos * H + i_h) * V, + (T, V), + (H * V, 1), + (i_t * BT, i_v * BV), + (BT, BV), + (1, 0), + ) + p_u = tl.make_block_ptr( + u + (bos * H + i_h) * V, + (T, V), + (H * V, 1), + (i_t * BT, i_v * BV), + (BT, BV), + (1, 0), + ) + b_v = tl.load(p_v, boundary_check=(0, 1)) + b_vb = (b_v * b_b[:, None]).to(b_v.dtype) + b_u = tl.dot(b_A, b_vb, input_precision=DOT_PRECISION) + tl.store(p_u, b_u.to(p_u.dtype.element_ty), boundary_check=(0, 1)) + + for i_k in range(tl.cdiv(K, BK)): + p_w = tl.make_block_ptr( + w + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + p_k = tl.make_block_ptr( + k + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_kb = b_k * b_b[:, None] + + p_gk = tl.make_block_ptr( + gk + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + b_gk = tl.load(p_gk, boundary_check=(0, 1)) + b_kb *= exp2(b_gk) + if STORE_QG: + p_q = tl.make_block_ptr( + q + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + p_qg = tl.make_block_ptr( + qg + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + b_q = tl.load(p_q, boundary_check=(0, 1)) + b_qg = b_q * exp2(b_gk) + tl.store(p_qg, b_qg.to(p_qg.dtype.element_ty), boundary_check=(0, 1)) + if STORE_KG: + last_idx = min(i_t * BT + BT, T) - 1 + + o_k = i_k * BK + tl.arange(0, BK) + m_k = o_k < K + b_gn = tl.load( + gk + ((bos + last_idx) * H + i_h) * K + o_k, mask=m_k, other=0.0 + ) + b_kg = b_k * exp2(b_gn - b_gk) + + p_kg = tl.make_block_ptr( + kg + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + tl.store(p_kg, b_kg.to(p_kg.dtype.element_ty), boundary_check=(0, 1)) + + b_w = tl.dot(b_A, b_kb.to(b_k.dtype)) + tl.store(p_w, b_w.to(p_w.dtype.element_ty), boundary_check=(0, 1)) + + +def recompute_w_u_fwd( + k: torch.Tensor, + v: torch.Tensor, + beta: torch.Tensor, + A: torch.Tensor, + q: torch.Tensor | None = None, + gk: torch.Tensor | None = None, + cu_seqlens: torch.Tensor | None = None, + chunk_indices: torch.Tensor | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + B, T, H, K, V = *k.shape, v.shape[-1] + BT = A.shape[-1] + BK = 64 + BV = 64 + + if chunk_indices is None and cu_seqlens is not None: + chunk_indices = prepare_chunk_indices(cu_seqlens, BT) + NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) + + w = torch.empty_like(k) + u = torch.empty_like(v) + kg = torch.empty_like(k) if gk is not None else None + recompute_w_u_fwd_kernel[(NT, B * H)]( + q=q, + k=k, + qg=None, + kg=kg, + v=v, + beta=beta, + w=w, + u=u, + A=A, + gk=gk, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + H=H, + K=K, + V=V, + BT=BT, + BK=BK, + BV=BV, + DOT_PRECISION="ieee", + ) + return w, u, None, kg + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +@triton.autotune( + configs=[ + triton.Config({"BK": BK, "BV": BV}, num_warps=num_warps, num_stages=num_stages) + for BK in [32, 64] + for BV in [64, 128] + for num_warps in [2, 4, 8] + for num_stages in [2, 3, 4] + ], + key=["BT"], +) +@triton.jit(do_not_specialize=["T"]) +def chunk_gla_fwd_kernel_o( + q, + v, + g, + h, + o, + A, + cu_seqlens, + chunk_indices, + scale, + T, + H: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BT: tl.constexpr, + BK: tl.constexpr, + BV: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_v, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_tg = i_t + i_n, i_t = ( + tl.load(chunk_indices + i_t * 2).to(tl.int32), + tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32), + ) + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + NT = tl.cdiv(T, BT) + else: + NT = tl.cdiv(T, BT) + i_tg = i_b * NT + i_t + bos, eos = i_b * T, i_b * T + T + + m_s = tl.arange(0, BT)[:, None] >= tl.arange(0, BT)[None, :] + + b_o = tl.zeros([BT, BV], dtype=tl.float32) + for i_k in range(tl.cdiv(K, BK)): + p_q = tl.make_block_ptr( + q + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + p_g = tl.make_block_ptr( + g + (bos * H + i_h) * K, + (T, K), + (H * K, 1), + (i_t * BT, i_k * BK), + (BT, BK), + (1, 0), + ) + p_h = tl.make_block_ptr( + # int64 BEFORE the K*V multiply: the int32 product wraps at chunk + # index 2048 (131072-token prefill at H=64, K=V=128). + h + (i_tg * H + i_h).to(tl.int64) * K * V, + (V, K), + (K, 1), + (i_v * BV, i_k * BK), + (BV, BK), + (1, 0), + ) + + # [BT, BK] + b_q = tl.load(p_q, boundary_check=(0, 1)) + b_q = (b_q * scale).to(b_q.dtype) + # [BT, BK] + b_g = tl.load(p_g, boundary_check=(0, 1)) + # [BT, BK] + b_qg = (b_q * exp2(b_g)).to(b_q.dtype) + # [BV, BK] + b_h = tl.load(p_h, boundary_check=(0, 1)) + # [BT, BV] + if i_k >= 0: + b_o += tl.dot(b_qg, tl.trans(b_h).to(b_qg.dtype)) + p_v = tl.make_block_ptr( + v + (bos * H + i_h) * V, + (T, V), + (H * V, 1), + (i_t * BT, i_v * BV), + (BT, BV), + (1, 0), + ) + p_o = tl.make_block_ptr( + o + (bos * H + i_h) * V, + (T, V), + (H * V, 1), + (i_t * BT, i_v * BV), + (BT, BV), + (1, 0), + ) + p_A = tl.make_block_ptr( + A + (bos * H + i_h) * BT, (T, BT), (H * BT, 1), (i_t * BT, 0), (BT, BT), (1, 0) + ) + # [BT, BV] + b_v = tl.load(p_v, boundary_check=(0, 1)) + # [BT, BT] + b_A = tl.load(p_A, boundary_check=(0, 1)) + b_A = tl.where(m_s, b_A, 0.0).to(b_v.dtype) + b_o += tl.dot(b_A, b_v, allow_tf32=False) + tl.store(p_o, b_o.to(p_o.dtype.element_ty), boundary_check=(0, 1)) + + +def chunk_gla_fwd_o_gk( + q: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + A: torch.Tensor, + h: torch.Tensor, + o: torch.Tensor, + scale: float, + cu_seqlens: torch.Tensor | None = None, + chunk_indices: torch.Tensor | None = None, + chunk_size: int = FLA_CHUNK_SIZE, +): + B, T, H, K, V = *q.shape, v.shape[-1] + BT = chunk_size + + if chunk_indices is None and cu_seqlens is not None: + chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size) + NT = cdiv(T, BT) if cu_seqlens is None else len(chunk_indices) + + def grid(meta): + return (cdiv(V, meta["BV"]), NT, B * H) + + chunk_gla_fwd_kernel_o[grid]( + q=q, + v=v, + g=g, + h=h, + o=o, + A=A, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + scale=scale, + T=T, + H=H, + K=K, + V=V, + BT=BT, + ) + return o + + +@triton.heuristics( + { + "HAS_BIAS": lambda args: args["g_bias"] is not None, + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + } +) +@triton.autotune( + configs=[ + triton.Config({"BD": BD}, num_warps=num_warps) + for BD in [32, 64] + for num_warps in [2, 4, 8] + ], + key=["H", "D", "BT", "IS_VARLEN"], +) +@triton.jit(do_not_specialize=["T"]) +def kda_gate_cumsum_fwd_kernel( + g, + A, + y, + g_bias, + cu_seqlens, + chunk_indices, + cumsum_scale, + beta, + threshold, + SAFE_GATE: tl.constexpr, + LOWER_BOUND: tl.constexpr, + T, + H: tl.constexpr, + D: tl.constexpr, + BT: tl.constexpr, + BD: tl.constexpr, + HAS_BIAS: tl.constexpr, + IS_VARLEN: tl.constexpr, +): + i_d, i_t, i_bh = tl.program_id(0), tl.program_id(1), tl.program_id(2) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = ( + tl.load(chunk_indices + i_t * 2).to(tl.int32), + tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32), + ) + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + else: + bos = i_b * T + + p_g = tl.make_block_ptr( + g + (bos * H + i_h) * D, + (T, D), + (H * D, 1), + (i_t * BT, i_d * BD), + (BT, BD), + (1, 0), + ) + p_y = tl.make_block_ptr( + y + (bos * H + i_h) * D, + (T, D), + (H * D, 1), + (i_t * BT, i_d * BD), + (BT, BD), + (1, 0), + ) + + b_g = tl.load(p_g, boundary_check=(0, 1)).to(tl.float32) + if HAS_BIAS: + o_d = i_d * BD + tl.arange(0, BD) + b_bias = tl.load(g_bias + i_h * D + o_d, mask=o_d < D, other=0.0).to(tl.float32) + b_g = b_g + b_bias[None, :] + + b_a = tl.load(A + i_h).to(tl.float32) + b_a = tl.exp(b_a) if SAFE_GATE else -tl.exp(b_a) + if SAFE_GATE: + # y = lower_bound * sigmoid(exp(A) * (g + g_bias)), bounded to + # (lower_bound, 0) for safe-gate checkpoints. + b_gate = LOWER_BOUND / (1.0 + tl.exp(-(b_a * b_g))) + else: + b_g_scaled = b_g * beta + b_softplus = tl.where( + b_g_scaled > threshold, + b_g, + (1.0 / beta) * log(1.0 + tl.exp(b_g_scaled)), + ) + b_gate = b_a * b_softplus + + # Out-of-bounds rows (load returns 0, but softplus/bias can still make + # b_gate non-zero) participate in the dot product. They only contribute to + # out-of-bounds output rows, which are masked away by `boundary_check` on + # the store, so visible output matches unfused gate + chunk-local cumsum. + o_t = tl.arange(0, BT) + m_cumsum = tl.where(o_t[:, None] >= o_t[None, :], 1.0, 0.0) + b_y = tl.dot(m_cumsum, b_gate, allow_tf32=False) * cumsum_scale + tl.store(p_y, b_y.to(p_y.dtype.element_ty), boundary_check=(0, 1)) + + +def fused_kda_gate_chunk_cumsum( + raw_g: torch.Tensor, + A_log: torch.Tensor, + g_bias: torch.Tensor | None = None, + beta: float = 1.0, + threshold: float = 20.0, + cu_seqlens: torch.Tensor | None = None, + chunk_indices: torch.Tensor | None = None, + chunk_size: int = FLA_CHUNK_SIZE, + output_dtype: torch.dtype | None = torch.float, + safe_gate: bool = False, + lower_bound: float = -5.0, +) -> torch.Tensor: + if cu_seqlens is not None: + assert raw_g.shape[0] == 1, ( + "Only batch size 1 is supported when cu_seqlens are provided" + ) + B, T, H, D = raw_g.shape + if chunk_indices is None and cu_seqlens is not None: + chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size) + NT = cdiv(T, chunk_size) if cu_seqlens is None else len(chunk_indices) + + A_log = A_log.reshape(-1) + if g_bias is not None: + g_bias = g_bias.reshape(-1) + y = torch.empty_like(raw_g, dtype=output_dtype or raw_g.dtype) + + def grid(meta): + return (cdiv(meta["D"], meta["BD"]), NT, B * H) + + kda_gate_cumsum_fwd_kernel[grid]( + g=raw_g, + A=A_log, + y=y, + g_bias=g_bias, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + # RCP_LN2 folds in the natural-log -> log2 conversion so downstream + # exp2-based kernels reproduce exp(g). Keep this in sync with the + # `use_exp2=True` path in `_chunk_kda_fwd_with_cumulative_g`. + cumsum_scale=RCP_LN2, + beta=beta, + threshold=threshold, + SAFE_GATE=safe_gate, + LOWER_BOUND=lower_bound, + T=T, + H=H, + D=D, + BT=chunk_size, + ) + return y + + +def _chunk_kda_fwd_with_cumulative_g( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state: torch.Tensor, + output_final_state: bool, + cu_seqlens: torch.Tensor | None = None, + chunk_indices: torch.Tensor | None = None, + chunk_size: int = FLA_CHUNK_SIZE, + return_h: bool = False, +): + # Token-offset addressing (q/k/g at `(bos*H + i_h) * K`) is still int32 in the + # intra-chunk kernels: refuse a varlen batch long enough to wrap rather than + # silently corrupting (the chunk-STATE offsets are int64 and do not bind first). + _, _T, _H, _K = k.shape # [1, total_tokens, H, K] (varlen packs on dim 1) + if _T * _H * _K >= 2**31: + raise ValueError( + f"KDA prefill batch too long for int32 token addressing: {_T} tokens " + f"x H={_H} x K={_K} exceeds 2**31; keep --max-prefill-length below " + f"{2**31 // (_H * _K)} tokens." + ) + # `g` must already be chunk-local cumulatively-summed AND scaled by + # RCP_LN2 (so the downstream exp2-based kernels reproduce exp(g)). + # Use `chunk_kda_fwd` or `chunk_kda_with_fused_gate_fwd` instead of + # calling this helper directly unless that invariant is upheld. + # the intra Aqk is kept in fp32 + # the computation has very marginal effect on the entire throughput + A, Aqk = chunk_kda_scaled_dot_kkt_fwd( + q=q, + k=k, + gk=g, + beta=beta, + scale=scale, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + chunk_size=chunk_size, + output_dtype=torch.float32, + ) + A = solve_tril(A=A, cu_seqlens=cu_seqlens, output_dtype=k.dtype) + w, u, _, kg = recompute_w_u_fwd( + k=k, + v=v, + beta=beta, + A=A, + gk=g, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + ) + del A + h, v_new, final_state = chunk_gated_delta_rule_fwd_h( + k=kg, + w=w, + u=u, + gk=g, + initial_state=initial_state, + output_final_state=output_final_state, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + use_exp2=True, + ) + del w, u, kg + o = chunk_gla_fwd_o_gk( + q=q, + v=v_new, + g=g, + A=Aqk, + h=h, + o=v, + scale=scale, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + chunk_size=chunk_size, + ) + del Aqk, v_new + if return_h: + # FreeToken addition: expose the per-chunk state snapshots (h[b, i] is the + # state at the START of chunk i) for hybrid-radix track checkpoints. + return o, final_state, h + del h + return o, final_state + + +def chunk_kda_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float, + initial_state: torch.Tensor, + output_final_state: bool, + cu_seqlens: torch.Tensor | None = None, +): + chunk_size = FLA_CHUNK_SIZE + chunk_indices = ( + prepare_chunk_indices(cu_seqlens, chunk_size) + if cu_seqlens is not None + else None + ) + g = chunk_local_cumsum( + g, + chunk_size=chunk_size, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + ) + # KDA evaluates cumulative gate decays with exp2. Convert from natural-log + # space so exp(x) is preserved as exp2(x / ln(2)). + g = g * RCP_LN2 + return _chunk_kda_fwd_with_cumulative_g( + q=q, + k=k, + v=v, + g=g, + beta=beta, + scale=scale, + initial_state=initial_state, + output_final_state=output_final_state, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + chunk_size=chunk_size, + ) + + +def chunk_kda_with_fused_gate_fwd( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + raw_g: torch.Tensor, + beta: torch.Tensor, + A_log: torch.Tensor, + g_bias: torch.Tensor | None, + scale: float, + initial_state: torch.Tensor, + output_final_state: bool, + cu_seqlens: torch.Tensor | None = None, + safe_gate: bool = False, + lower_bound: float = -5.0, + return_h: bool = False, +): + chunk_size = FLA_CHUNK_SIZE + chunk_indices = ( + prepare_chunk_indices(cu_seqlens, chunk_size) + if cu_seqlens is not None + else None + ) + g = fused_kda_gate_chunk_cumsum( + raw_g, + A_log=A_log, + g_bias=g_bias, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + chunk_size=chunk_size, + safe_gate=safe_gate, + lower_bound=lower_bound, + ) + return _chunk_kda_fwd_with_cumulative_g( + q=q, + k=k, + v=v, + g=g, + beta=beta, + scale=scale, + initial_state=initial_state, + output_final_state=output_final_state, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + chunk_size=chunk_size, + return_h=return_h, + ) + + +def chunk_kda( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + g: torch.Tensor, + beta: torch.Tensor, + scale: float = None, + initial_state: torch.Tensor = None, + output_final_state: bool = False, + use_qk_l2norm_in_kernel: bool = False, + cu_seqlens: torch.Tensor | None = None, + **kwargs, +): + if scale is None: + scale = k.shape[-1] ** -0.5 + + if use_qk_l2norm_in_kernel: + q = l2norm_fwd(q.contiguous()) + k = l2norm_fwd(k.contiguous()) + + o, final_state = chunk_kda_fwd( + q=q, + k=k, + v=v.contiguous(), + g=g.contiguous(), + beta=beta.contiguous(), + scale=scale, + initial_state=initial_state.contiguous(), + output_final_state=output_final_state, + cu_seqlens=cu_seqlens, + ) + return o, final_state + + +def chunk_kda_with_fused_gate( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + raw_g: torch.Tensor, + beta: torch.Tensor, + A_log: torch.Tensor, + g_bias: torch.Tensor | None, + scale: float | None = None, + initial_state: torch.Tensor | None = None, + output_final_state: bool = False, + use_qk_l2norm_in_kernel: bool = False, + cu_seqlens: torch.Tensor | None = None, + safe_gate: bool = False, + lower_bound: float = -5.0, + return_h: bool = False, + **kwargs, +): + """Run chunk KDA from raw gate projection using fused gate+cumsum. + + WARNING: the output is written into (and returned as) the ``v`` buffer; never + pass a tensor that is read again after this call. With ``return_h`` the + per-chunk state snapshots (h[b, i] = state at the START of chunk i) are + returned as a third value, for hybrid-radix track checkpoints. + """ + if scale is None: + scale = k.shape[-1] ** -0.5 + + if use_qk_l2norm_in_kernel: + q = l2norm_fwd(q.contiguous()) + k = l2norm_fwd(k.contiguous()) + + return chunk_kda_with_fused_gate_fwd( + q=q, + k=k, + v=v.contiguous(), + raw_g=raw_g.contiguous(), + beta=beta.contiguous(), + A_log=A_log, + g_bias=g_bias, + scale=scale, + initial_state=initial_state.contiguous() if initial_state is not None else None, + output_final_state=output_final_state, + cu_seqlens=cu_seqlens, + safe_gate=safe_gate, + lower_bound=lower_bound, + return_h=return_h, + ) + + +@triton.autotune( + configs=[ + triton.Config({"BT": bt}, num_warps=nw, num_stages=ns) + for bt in BT_LIST_AUTOTUNE + for nw in NUM_WARPS_AUTOTUNE + for ns in [2, 3] + ], + key=["H", "D"], +) +@triton.jit +def kda_gate_fwd_kernel( + g, + A, + y, + g_bias, + beta: tl.constexpr, + threshold: tl.constexpr, + SAFE_GATE: tl.constexpr, + LOWER_BOUND: tl.constexpr, + T, + H, + D: tl.constexpr, + BT: tl.constexpr, + BD: tl.constexpr, + HAS_BIAS: tl.constexpr, +): + i_t, i_h = tl.program_id(0), tl.program_id(1) + n_t = i_t * BT + + b_a = tl.load(A + i_h).to(tl.float32) + b_a = tl.exp(b_a) if SAFE_GATE else -tl.exp(b_a) + + stride_row = H * D + stride_col = 1 + + g_ptr = tl.make_block_ptr( + base=g + i_h * D, + shape=(T, D), + strides=(stride_row, stride_col), + offsets=(n_t, 0), + block_shape=(BT, BD), + order=(1, 0), + ) + + y_ptr = tl.make_block_ptr( + base=y + i_h * D, + shape=(T, D), + strides=(stride_row, stride_col), + offsets=(n_t, 0), + block_shape=(BT, BD), + order=(1, 0), + ) + + b_g = tl.load(g_ptr, boundary_check=(0, 1)).to(tl.float32) + + if HAS_BIAS: + n_d = tl.arange(0, BD) + bias_mask = n_d < D + b_bias = tl.load(g_bias + i_h * D + n_d, mask=bias_mask, other=0.0).to( + tl.float32 + ) + b_g = b_g + b_bias[None, :] + + if SAFE_GATE: + # y = lower_bound * sigmoid(exp(A) * (g + g_bias)), bounded to + # (lower_bound, 0) for safe-gate checkpoints. + b_y = LOWER_BOUND / (1.0 + tl.exp(-(b_a * b_g))) + else: + # softplus(x, beta) = (1/beta) * log(1 + exp(beta * x)) + # When beta * x > threshold, use linear approximation x + # Use threshold to switch to linear when beta*x > threshold + g_scaled = b_g * beta + use_linear = g_scaled > threshold + sp = tl.where(use_linear, b_g, (1.0 / beta) * log(1.0 + tl.exp(g_scaled))) + b_y = b_a * sp + + tl.store(y_ptr, b_y.to(y.dtype.element_ty), boundary_check=(0, 1)) + + +def fused_kda_gate( + g: torch.Tensor, + A: torch.Tensor, + head_k_dim: int, + g_bias: torch.Tensor | None = None, + beta: float = 1.0, + threshold: float = 20.0, + safe_gate: bool = False, + lower_bound: float | None = -5.0, +) -> torch.Tensor: + """ + Forward pass for KDA gate: + input g: [..., H*D] + param A: [H] or [1, 1, H, 1] + beta: softplus beta parameter (softplus branch only) + threshold: softplus threshold parameter (softplus branch only) + safe_gate: when False (default) compute y = -exp(A)*softplus(g+g_bias); + when True compute the bounded y = lower_bound*sigmoid(exp(A)*(g+g_bias)) + lower_bound: floor for the safe_gate branch (default -5.0) + return : [..., H, D] + """ + orig_shape = g.shape[:-1] + + g = g.view(-1, g.shape[-1]) + T = g.shape[0] + HD = g.shape[1] + H = A.numel() + assert H * head_k_dim == HD + + y = torch.empty_like(g, dtype=torch.float32) + + def grid(meta): + return (cdiv(T, meta["BT"]), H) + + kda_gate_fwd_kernel[grid]( + g, + A, + y, + g_bias, + beta, + threshold, + safe_gate, + lower_bound if lower_bound is not None else -5.0, + T, + H, + head_k_dim, + BD=next_power_of_2(head_k_dim), + HAS_BIAS=g_bias is not None, + ) + + y = y.view(*orig_shape, H, head_k_dim) + return y diff --git a/python/freetoken/kernel/fla/kda_chunk_delta_h.py b/python/freetoken/kernel/fla/kda_chunk_delta_h.py new file mode 100644 index 0000000000..55c0ffb21e --- /dev/null +++ b/python/freetoken/kernel/fla/kda_chunk_delta_h.py @@ -0,0 +1,394 @@ +# Vendored from vLLM's third_party/flash_linear_attention (PR #53906, commit 933876c3), +# itself copied from the flash-linear-attention project (MIT, (c) 2023-2025 Songlin Yang, +# Yu Zhang). Imports are remapped onto freetoken.kernel.fla's shared helpers; keep this +# file in sync with upstream when pulling KDA kernel fixes. +# NOTE: this is the KDA-consistent variant (exp2 gate semantics via use_exp2, returns +# the final state). freetoken/kernel/fla/chunk_delta_h.py is the GDN variant (natural-exp +# gk, in-place pool state via initial_state_indices); the two serve different recurrences +# and are kept separate on purpose. +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang +# +# This file contains code copied from the flash-linear-attention project. +# The original source code was licensed under the MIT license and included +# the following copyright notice: +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang +# ruff: noqa: E501 + +import torch + +import triton +import triton.language as tl + +from .index import prepare_chunk_indices, prepare_chunk_offsets +from .op import exp, exp2 +from .utils import FLA_CHUNK_SIZE, use_cuda_graph + +NUM_WARPS = [2, 4, 8, 16] +# Triton's AMD backend fails to lower this kernel with num_stages=4. +_CHUNK_DELTA_H_NUM_STAGES = [2, 3] if torch.version.hip else [2, 3, 4] + + +@triton.heuristics( + { + "USE_G": lambda args: args["g"] is not None, + "USE_GK": lambda args: args["gk"] is not None, + "USE_INITIAL_STATE": lambda args: args["h0"] is not None, + "STORE_FINAL_STATE": lambda args: args["ht"] is not None, + "SAVE_NEW_VALUE": lambda args: args["v_new"] is not None, + "IS_VARLEN": lambda args: args["cu_seqlens"] is not None, + } +) +@triton.autotune( + configs=[ + triton.Config({"BV": BV}, num_warps=num_warps, num_stages=num_stages) + for num_warps in [2, 4] + for num_stages in _CHUNK_DELTA_H_NUM_STAGES + for BV in [32, 64] + ], + key=["H", "K", "V", "BT"], + use_cuda_graph=use_cuda_graph, +) +@triton.jit(do_not_specialize=["T"]) +def chunk_gated_delta_rule_fwd_kernel_h_blockdim64( + k, + v, + w, + v_new, + g, + gk, + h, + h0, + ht, + cu_seqlens, + chunk_offsets, + T, + H: tl.constexpr, + Hg: tl.constexpr, + K: tl.constexpr, + V: tl.constexpr, + BT: tl.constexpr, + BV: tl.constexpr, + USE_G: tl.constexpr, + USE_GK: tl.constexpr, + USE_INITIAL_STATE: tl.constexpr, + STORE_FINAL_STATE: tl.constexpr, + SAVE_NEW_VALUE: tl.constexpr, + IS_VARLEN: tl.constexpr, + USE_EXP2: tl.constexpr, +): + i_v, i_nh = tl.program_id(0), tl.program_id(1) + i_n, i_h = i_nh // H, i_nh % H + if IS_VARLEN: + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + NT = tl.cdiv(T, BT) + boh = tl.load(chunk_offsets + i_n).to(tl.int32) + else: + bos, eos = i_n * T, i_n * T + T + NT = tl.cdiv(T, BT) + boh = i_n * NT + + # [BV, BK] + b_h1 = tl.zeros([BV, 64], dtype=tl.float32) + if K > 64: + b_h2 = tl.zeros([BV, 64], dtype=tl.float32) + if K > 128: + b_h3 = tl.zeros([BV, 64], dtype=tl.float32) + if K > 192: + b_h4 = tl.zeros([BV, 64], dtype=tl.float32) + + # calculate offset. DIVERGENCE from upstream: cast to int64 BEFORE the K*V + # multiply (the GDN chunk_o.py idiom) -- upstream casts the already-wrapped + # int32 product, which overflows at chunk index 2048 (a 131072-token prefill + # at H=64, K=V=128) and silently lands in another head's state. + h += (boh * H + i_h).to(tl.int64) * V * K + v += (bos * H + i_h).to(tl.int64) * V + k += (bos * Hg + i_h // (H // Hg)).to(tl.int64) * K + w += (bos * H + i_h).to(tl.int64) * K + if SAVE_NEW_VALUE: + v_new += (bos * H + i_h).to(tl.int64) * V + stride_v = H * V + stride_h = H * V * K + stride_k = Hg * K + stride_w = H * K + if USE_INITIAL_STATE: + h0 = h0 + i_nh * V * K + if STORE_FINAL_STATE: + ht = ht + i_nh * V * K + + # load initial state + if USE_INITIAL_STATE: + p_h0_1 = tl.make_block_ptr(h0, (V, K), (K, 1), (i_v * BV, 0), (BV, 64), (1, 0)) + b_h1 += tl.load(p_h0_1, boundary_check=(0, 1)).to(tl.float32) + if K > 64: + p_h0_2 = tl.make_block_ptr( + h0, (V, K), (K, 1), (i_v * BV, 64), (BV, 64), (1, 0) + ) + b_h2 += tl.load(p_h0_2, boundary_check=(0, 1)).to(tl.float32) + if K > 128: + p_h0_3 = tl.make_block_ptr( + h0, (V, K), (K, 1), (i_v * BV, 128), (BV, 64), (1, 0) + ) + b_h3 += tl.load(p_h0_3, boundary_check=(0, 1)).to(tl.float32) + if K > 192: + p_h0_4 = tl.make_block_ptr( + h0, (V, K), (K, 1), (i_v * BV, 192), (BV, 64), (1, 0) + ) + b_h4 += tl.load(p_h0_4, boundary_check=(0, 1)).to(tl.float32) + + # main recurrence + for i_t in range(NT): + p_h1 = tl.make_block_ptr( + h + i_t.to(tl.int64) * stride_h, + (V, K), + (K, 1), + (i_v * BV, 0), + (BV, 64), + (1, 0), + ) + tl.store(p_h1, b_h1.to(p_h1.dtype.element_ty), boundary_check=(0, 1)) + if K > 64: + p_h2 = tl.make_block_ptr( + h + i_t.to(tl.int64) * stride_h, + (V, K), + (K, 1), + (i_v * BV, 64), + (BV, 64), + (1, 0), + ) + tl.store(p_h2, b_h2.to(p_h2.dtype.element_ty), boundary_check=(0, 1)) + if K > 128: + p_h3 = tl.make_block_ptr( + h + i_t.to(tl.int64) * stride_h, + (V, K), + (K, 1), + (i_v * BV, 128), + (BV, 64), + (1, 0), + ) + tl.store(p_h3, b_h3.to(p_h3.dtype.element_ty), boundary_check=(0, 1)) + if K > 192: + p_h4 = tl.make_block_ptr( + h + i_t.to(tl.int64) * stride_h, + (V, K), + (K, 1), + (i_v * BV, 192), + (BV, 64), + (1, 0), + ) + tl.store(p_h4, b_h4.to(p_h4.dtype.element_ty), boundary_check=(0, 1)) + + p_w = tl.make_block_ptr( + w, (T, K), (stride_w, 1), (i_t * BT, 0), (BT, 64), (1, 0) + ) + b_w = tl.load(p_w, boundary_check=(0, 1)) + b_v = tl.dot(b_w, tl.trans(b_h1).to(b_w.dtype)) + if K > 64: + p_w = tl.make_block_ptr( + w, (T, K), (stride_w, 1), (i_t * BT, 64), (BT, 64), (1, 0) + ) + b_w = tl.load(p_w, boundary_check=(0, 1)) + b_v += tl.dot(b_w, tl.trans(b_h2).to(b_w.dtype)) + if K > 128: + p_w = tl.make_block_ptr( + w, (T, K), (stride_w, 1), (i_t * BT, 128), (BT, 64), (1, 0) + ) + b_w = tl.load(p_w, boundary_check=(0, 1)) + b_v += tl.dot(b_w, tl.trans(b_h3).to(b_w.dtype)) + if K > 192: + p_w = tl.make_block_ptr( + w, (T, K), (stride_w, 1), (i_t * BT, 192), (BT, 64), (1, 0) + ) + b_w = tl.load(p_w, boundary_check=(0, 1)) + b_v += tl.dot(b_w, tl.trans(b_h4).to(b_w.dtype)) + p_v = tl.make_block_ptr( + v, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) + ) + b_v = tl.load(p_v, boundary_check=(0, 1)) - b_v + + if SAVE_NEW_VALUE: + p_v = tl.make_block_ptr( + v_new, (T, V), (stride_v, 1), (i_t * BT, i_v * BV), (BT, BV), (1, 0) + ) + tl.store(p_v, b_v.to(p_v.dtype.element_ty), boundary_check=(0, 1)) + + last_idx = min((i_t.to(tl.int64) + 1) * BT, T) - 1 + if USE_G: + m_t = (i_t.to(tl.int64) * BT + tl.arange(0, BT)) < T + b_g_last = tl.load(g + bos * H + last_idx * H + i_h) + p_g = tl.make_block_ptr( + g + bos * H + i_h, (T,), (H,), (i_t * BT,), (BT,), (0,) + ) + b_g = tl.load(p_g, boundary_check=(0,)) + if USE_EXP2: + b_v = b_v * tl.where(m_t, exp2(b_g_last - b_g), 0)[:, None] + b_g_last = exp2(b_g_last) + else: + b_v = b_v * tl.where(m_t, exp(b_g_last - b_g), 0)[:, None] + b_g_last = exp(b_g_last) + b_h1 *= b_g_last + if K > 64: + b_h2 *= b_g_last + if K > 128: + b_h3 *= b_g_last + if K > 192: + b_h4 *= b_g_last + + if USE_GK: + o_k1 = tl.arange(0, 64) + b_gk_last1 = tl.load( + gk + (bos + last_idx) * H * K + i_h * K + o_k1, + mask=(o_k1 < K), + other=0.0, + ) + if USE_EXP2: + b_h1 *= exp2(b_gk_last1)[None, :] + else: + b_h1 *= exp(b_gk_last1)[None, :] + if K > 64: + o_k2 = 64 + o_k1 + b_gk_last2 = tl.load( + gk + (bos + last_idx) * H * K + i_h * K + o_k2, + mask=(o_k2 < K), + other=0.0, + ) + if USE_EXP2: + b_h2 *= exp2(b_gk_last2)[None, :] + else: + b_h2 *= exp(b_gk_last2)[None, :] + if K > 128: + o_k3 = 128 + o_k1 + b_gk_last3 = tl.load( + gk + (bos + last_idx) * H * K + i_h * K + o_k3, + mask=(o_k3 < K), + other=0.0, + ) + if USE_EXP2: + b_h3 *= exp2(b_gk_last3)[None, :] + else: + b_h3 *= exp(b_gk_last3)[None, :] + if K > 192: + o_k4 = 192 + o_k1 + b_gk_last4 = tl.load( + gk + (bos + last_idx) * H * K + i_h * K + o_k4, + mask=(o_k4 < K), + other=0.0, + ) + if USE_EXP2: + b_h4 *= exp2(b_gk_last4)[None, :] + else: + b_h4 *= exp(b_gk_last4)[None, :] + b_v = b_v.to(k.dtype.element_ty) + + p_k = tl.make_block_ptr( + k, (K, T), (1, stride_k), (0, i_t * BT), (64, BT), (0, 1) + ) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_h1 += tl.trans(tl.dot(b_k, b_v)) + if K > 64: + p_k = tl.make_block_ptr( + k, (K, T), (1, stride_k), (64, i_t * BT), (64, BT), (0, 1) + ) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_h2 += tl.trans(tl.dot(b_k, b_v)) + if K > 128: + p_k = tl.make_block_ptr( + k, (K, T), (1, stride_k), (128, i_t * BT), (64, BT), (0, 1) + ) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_h3 += tl.trans(tl.dot(b_k, b_v)) + if K > 192: + p_k = tl.make_block_ptr( + k, (K, T), (1, stride_k), (192, i_t * BT), (64, BT), (0, 1) + ) + b_k = tl.load(p_k, boundary_check=(0, 1)) + b_h4 += tl.trans(tl.dot(b_k, b_v)) + # epilogue + if STORE_FINAL_STATE: + p_ht = tl.make_block_ptr(ht, (V, K), (K, 1), (i_v * BV, 0), (BV, 64), (1, 0)) + tl.store(p_ht, b_h1.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) + if K > 64: + p_ht = tl.make_block_ptr( + ht, (V, K), (K, 1), (i_v * BV, 64), (BV, 64), (1, 0) + ) + tl.store(p_ht, b_h2.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) + if K > 128: + p_ht = tl.make_block_ptr( + ht, (V, K), (K, 1), (i_v * BV, 128), (BV, 64), (1, 0) + ) + tl.store(p_ht, b_h3.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) + if K > 192: + p_ht = tl.make_block_ptr( + ht, (V, K), (K, 1), (i_v * BV, 192), (BV, 64), (1, 0) + ) + tl.store(p_ht, b_h4.to(p_ht.dtype.element_ty), boundary_check=(0, 1)) + + +def chunk_gated_delta_rule_fwd_h( + k: torch.Tensor, + w: torch.Tensor, + u: torch.Tensor, + g: torch.Tensor | None = None, + gk: torch.Tensor | None = None, + initial_state: torch.Tensor | None = None, + output_final_state: bool = False, + chunk_size: int = FLA_CHUNK_SIZE, + save_new_value: bool = True, + cu_seqlens: torch.Tensor | None = None, + chunk_indices: torch.Tensor | None = None, + chunk_offsets: torch.Tensor | None = None, + use_exp2: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + # This kernel is slightly different from fla to support Q/K with different head numbers. + # In fla, Q/K always have the same head number, so Hg is always equal to H. + B, T, Hg, K, V = *k.shape, u.shape[-1] + H = u.shape[-2] + BT = chunk_size + + if chunk_indices is None and cu_seqlens is not None: + chunk_indices = prepare_chunk_indices(cu_seqlens, chunk_size) + # N: the actual number of sequences in the batch with either equal or variable lengths + if cu_seqlens is None: + N, NT, chunk_offsets = B, triton.cdiv(T, BT), None + else: + N, NT = len(cu_seqlens) - 1, len(chunk_indices) + if chunk_offsets is None: + chunk_offsets = prepare_chunk_offsets(cu_seqlens, BT) + assert K <= 256, "current kernel does not support head dimension larger than 256." + + h = k.new_empty(B, NT, H, V, K) + final_state = ( + k.new_empty(N, H, V, K, dtype=torch.float32) if output_final_state else None + ) + + v_new = torch.empty_like(u) if save_new_value else None + + def grid(meta): + return (triton.cdiv(V, meta["BV"]), N * H) + + chunk_gated_delta_rule_fwd_kernel_h_blockdim64[grid]( + k=k, + v=u, + w=w, + v_new=v_new, + g=g, + gk=gk, + h=h, + h0=initial_state, + ht=final_state, + cu_seqlens=cu_seqlens, + chunk_offsets=chunk_offsets, + T=T, + H=H, + Hg=Hg, + K=K, + V=V, + BT=BT, + USE_EXP2=use_exp2, + ) + return h, v_new, final_state diff --git a/python/freetoken/kernel/fla/solve_tril.py b/python/freetoken/kernel/fla/solve_tril.py new file mode 100644 index 0000000000..a8cad05842 --- /dev/null +++ b/python/freetoken/kernel/fla/solve_tril.py @@ -0,0 +1,563 @@ +# Vendored from vLLM's third_party/flash_linear_attention (PR #53906, commit 933876c3), +# itself copied from the flash-linear-attention project (MIT, (c) 2023-2025 Songlin Yang, +# Yu Zhang). Imports are remapped onto freetoken.kernel.fla's shared helpers; keep this +# file in sync with upstream when pulling KDA kernel fixes. +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# SPDX-FileCopyrightText: Songlin Yang, Yu Zhang +# +# This file contains code copied from the flash-linear-attention project. +# The original source code was licensed under the MIT license and included +# the following copyright notice: +# Copyright (c) 2023-2025, Songlin Yang, Yu Zhang +# ruff: noqa: E501 + +import os + +import torch + +import triton +import triton.language as tl + +from .index import prepare_chunk_indices +from .op import make_tensor_descriptor +from .utils import input_guard, is_amd, is_tma_supported + +FLA_TRIL_PRECISION = os.environ.get("FLA_TRIL_PRECISION", "ieee") +ALLOWED_TRIL_PRECISIONS = ["ieee", "tf32"] if is_amd else ["ieee", "tf32", "tf32x3"] +assert FLA_TRIL_PRECISION in ALLOWED_TRIL_PRECISIONS, ( + f"FLA_TRIL_PRECISION must be one of {ALLOWED_TRIL_PRECISIONS}, but got {FLA_TRIL_PRECISION}" +) + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=num_warps, num_stages=num_stages) + for num_warps in [1, 2, 4, 8] + for num_stages in [2, 3, 4, 5] + ], + key=["BT"], +) +@triton.jit(do_not_specialize=["T"]) +def solve_tril_16x16_kernel( + A, + Ai, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + BT: tl.constexpr, + USE_TMA: tl.constexpr, + IS_VARLEN: tl.constexpr, + DOT_PRECISION: tl.constexpr, +): + i_t, i_bh = tl.program_id(0), tl.program_id(1) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = ( + tl.load(chunk_indices + i_t * 2).to(tl.int32), + tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32), + ) + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + o_i = tl.arange(0, 16) + m_A = o_i[:, None] > o_i[None, :] + m_I = o_i[:, None] == o_i[None, :] + + A = A + (bos * H + i_h) * BT + Ai = Ai + (bos * H + i_h) * 16 + + offset = (i_t * 16) % BT + if not USE_TMA: + p_A = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * 16, offset), (16, 16), (1, 0) + ) + # [16, 16] + b_A = tl.load(p_A, boundary_check=(0, 1)).to(tl.float32) + else: + desc = make_tensor_descriptor(A, [T, BT], [H * BT, 1], [16, 16]) + desc_o = make_tensor_descriptor(Ai, [T, 16], [H * 16, 1], [16, 16]) + b_A = desc.load([i_t * 16, offset]).to(tl.float32) + b_A = -tl.where(m_A, b_A, 0) + + for i in range(2, min(16, T - i_t * 16)): + # [16] + b_a = -tl.load(A + (i_t * 16 + i) * H * BT + o_i + offset) + b_a = b_a + tl.sum(b_a[:, None] * b_A, 0) + b_A = tl.where((o_i == i)[:, None], b_a, b_A) + b_A += m_I + if not USE_TMA: + p_Ai = tl.make_block_ptr( + Ai, (T, 16), (H * 16, 1), (i_t * 16, 0), (16, 16), (1, 0) + ) + tl.store( + p_Ai, + b_A.to(p_Ai.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + else: + desc_o.store([i_t * 16, 0], b_A.to(desc_o.dtype, fp_downcast_rounding="rtne")) + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=num_warps, num_stages=num_stages) + for num_warps in [1, 2, 4, 8] + for num_stages in [2, 3, 4, 5] + ], + key=["H", "BT", "IS_VARLEN"], +) +@triton.jit(do_not_specialize=["T"]) +def merge_16x16_to_32x32_inverse_kernel( + A, + Ai, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + BT: tl.constexpr, + USE_TMA: tl.constexpr, + IS_VARLEN: tl.constexpr, + DOT_PRECISION: tl.constexpr, +): + i_t, i_bh = tl.program_id(0), tl.program_id(1) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = ( + tl.load(chunk_indices + i_t * 2).to(tl.int32), + tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32), + ) + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + + o_i = tl.arange(0, 16) + m_A = o_i[:, None] > o_i[None, :] + m_I = o_i[:, None] == o_i[None, :] + A += (bos * H + i_h) * BT + Ai += (bos * H + i_h) * BT + + if not USE_TMA: + p_A_11 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT, 0), (16, 16), (1, 0) + ) + p_A_22 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + 16, 16), (16, 16), (1, 0) + ) + b_Ai_11 = tl.load(p_A_11, boundary_check=(0, 1)).to(tl.float32) + b_Ai_22 = tl.load(p_A_22, boundary_check=(0, 1)).to(tl.float32) + else: + desc = make_tensor_descriptor(A, [T, BT], [H * BT, 1], [16, 16]) + desc_o = make_tensor_descriptor(Ai, [T, BT], [H * BT, 1], [16, 16]) + b_Ai_11 = desc.load([i_t * BT + 0, 0]).to(tl.float32) + b_Ai_22 = desc.load([i_t * BT + 16, 16]).to(tl.float32) + + # [16, 16] + b_Ai_11 = -tl.where(m_A, b_Ai_11, 0) + b_Ai_22 = -tl.where(m_A, b_Ai_22, 0) + + for i in range(2, min(16, T - i_t * BT)): + b_a_11 = -tl.load(A + (i_t * BT + i) * H * BT + o_i) + b_a_11 += tl.sum(b_a_11[:, None] * b_Ai_11, 0) + b_Ai_11 = tl.where((o_i == i)[:, None], b_a_11, b_Ai_11) + for i in range(16 + 2, min(32, T - i_t * BT)): + b_a_22 = -tl.load(A + (i_t * BT + i) * H * BT + o_i + 16) + b_a_22 += tl.sum(b_a_22[:, None] * b_Ai_22, 0) + b_Ai_22 = tl.where((o_i == i - 16)[:, None], b_a_22, b_Ai_22) + + b_Ai_11 += m_I + b_Ai_22 += m_I + + if not USE_TMA: + p_A_21 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + 16, 0), (16, 16), (1, 0) + ) + b_A_21 = tl.load(p_A_21, boundary_check=(0, 1)).to(tl.float32) + else: + b_A_21 = desc.load([i_t * BT + 16, 0]).to(tl.float32) + + b_Ai_21 = -tl.dot( + tl.dot(b_Ai_22, b_A_21, input_precision=DOT_PRECISION), + b_Ai_11, + input_precision=DOT_PRECISION, + ) + + if not USE_TMA: + p_Ai_11 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT, 0), (16, 16), (1, 0) + ) + p_Ai_21 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT + 16, 0), (16, 16), (1, 0) + ) + p_Ai_22 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT + 16, 16), (16, 16), (1, 0) + ) + tl.store( + p_Ai_11, + b_Ai_11.to(p_Ai_11.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_22, + b_Ai_22.to(p_Ai_22.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_21, + b_Ai_21.to(p_Ai_21.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + else: + desc_o.store( + [i_t * BT + 0, 0], b_Ai_11.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + desc_o.store( + [i_t * BT + 16, 0], b_Ai_21.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + desc_o.store( + [i_t * BT + 16, 16], b_Ai_22.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + + +@triton.heuristics({"IS_VARLEN": lambda args: args["cu_seqlens"] is not None}) +@triton.autotune( + configs=[ + triton.Config({}, num_warps=num_warps, num_stages=num_stages) + for num_warps in [2, 4, 8] + for num_stages in [2, 3, 4, 5] + ], + key=["H", "BT", "IS_VARLEN"], +) +@triton.jit(do_not_specialize=["T"]) +def merge_16x16_to_64x64_inverse_kernel( + A, + Ai, + cu_seqlens, + chunk_indices, + T, + H: tl.constexpr, + BT: tl.constexpr, + USE_TMA: tl.constexpr, + IS_VARLEN: tl.constexpr, + DOT_PRECISION: tl.constexpr, +): + i_t, i_bh = tl.program_id(0), tl.program_id(1) + i_b, i_h = i_bh // H, i_bh % H + if IS_VARLEN: + i_n, i_t = ( + tl.load(chunk_indices + i_t * 2).to(tl.int32), + tl.load(chunk_indices + i_t * 2 + 1).to(tl.int32), + ) + bos, eos = ( + tl.load(cu_seqlens + i_n).to(tl.int32), + tl.load(cu_seqlens + i_n + 1).to(tl.int32), + ) + T = eos - bos + else: + bos, eos = i_b * T, i_b * T + T + + o_i = tl.arange(0, 16) + m_A = o_i[:, None] > o_i[None, :] + m_I = o_i[:, None] == o_i[None, :] + A += (bos * H + i_h) * BT + Ai += (bos * H + i_h) * BT + + if not USE_TMA: + p_A_11 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT, 0), (16, 16), (1, 0) + ) + p_A_22 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + 16, 16), (16, 16), (1, 0) + ) + p_A_33 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + 32, 32), (16, 16), (1, 0) + ) + p_A_44 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + 48, 48), (16, 16), (1, 0) + ) + b_Ai_11 = tl.load(p_A_11, boundary_check=(0, 1)).to(tl.float32) + b_Ai_22 = tl.load(p_A_22, boundary_check=(0, 1)).to(tl.float32) + b_Ai_33 = tl.load(p_A_33, boundary_check=(0, 1)).to(tl.float32) + b_Ai_44 = tl.load(p_A_44, boundary_check=(0, 1)).to(tl.float32) + else: + desc = make_tensor_descriptor(A, [T, BT], [H * BT, 1], [16, 16]) + desc_o = make_tensor_descriptor(Ai, [T, BT], [H * BT, 1], [16, 16]) + b_Ai_11 = desc.load([i_t * BT + 0, 0]).to(tl.float32) + b_Ai_22 = desc.load([i_t * BT + 16, 16]).to(tl.float32) + b_Ai_33 = desc.load([i_t * BT + 32, 32]).to(tl.float32) + b_Ai_44 = desc.load([i_t * BT + 48, 48]).to(tl.float32) + + # [16, 16] + b_Ai_11 = -tl.where(m_A, b_Ai_11, 0) + b_Ai_22 = -tl.where(m_A, b_Ai_22, 0) + b_Ai_33 = -tl.where(m_A, b_Ai_33, 0) + b_Ai_44 = -tl.where(m_A, b_Ai_44, 0) + + for i in range(2, min(16, T - i_t * BT)): + b_a_11 = -tl.load(A + (i_t * BT + i) * H * BT + o_i) + b_a_11 += tl.sum(b_a_11[:, None] * b_Ai_11, 0) + b_Ai_11 = tl.where((o_i == i)[:, None], b_a_11, b_Ai_11) + for i in range(16 + 2, min(32, T - i_t * BT)): + b_a_22 = -tl.load(A + (i_t * BT + i) * H * BT + o_i + 16) + b_a_22 += tl.sum(b_a_22[:, None] * b_Ai_22, 0) + b_Ai_22 = tl.where((o_i == i - 16)[:, None], b_a_22, b_Ai_22) + for i in range(32 + 2, min(48, T - i_t * BT)): + b_a_33 = -tl.load(A + (i_t * BT + i) * H * BT + o_i + 32) + b_a_33 += tl.sum(b_a_33[:, None] * b_Ai_33, 0) + b_Ai_33 = tl.where((o_i == i - 32)[:, None], b_a_33, b_Ai_33) + for i in range(48 + 2, min(64, T - i_t * BT)): + b_a_44 = -tl.load(A + (i_t * BT + i) * H * BT + o_i + 48) + b_a_44 += tl.sum(b_a_44[:, None] * b_Ai_44, 0) + b_Ai_44 = tl.where((o_i == i - 48)[:, None], b_a_44, b_Ai_44) + b_Ai_11 += m_I + b_Ai_22 += m_I + b_Ai_33 += m_I + b_Ai_44 += m_I + + if not USE_TMA: + p_A_21 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + 16, 0), (16, 16), (1, 0) + ) + p_A_31 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + 32, 0), (16, 16), (1, 0) + ) + p_A_32 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + 32, 16), (16, 16), (1, 0) + ) + p_A_41 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + 48, 0), (16, 16), (1, 0) + ) + p_A_42 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + 48, 16), (16, 16), (1, 0) + ) + p_A_43 = tl.make_block_ptr( + A, (T, BT), (H * BT, 1), (i_t * BT + 48, 32), (16, 16), (1, 0) + ) + b_A_21 = tl.load(p_A_21, boundary_check=(0, 1)).to(tl.float32) + b_A_31 = tl.load(p_A_31, boundary_check=(0, 1)).to(tl.float32) + b_A_32 = tl.load(p_A_32, boundary_check=(0, 1)).to(tl.float32) + b_A_41 = tl.load(p_A_41, boundary_check=(0, 1)).to(tl.float32) + b_A_42 = tl.load(p_A_42, boundary_check=(0, 1)).to(tl.float32) + b_A_43 = tl.load(p_A_43, boundary_check=(0, 1)).to(tl.float32) + else: + b_A_21 = desc.load([i_t * BT + 16, 0]).to(tl.float32) + b_A_31 = desc.load([i_t * BT + 32, 0]).to(tl.float32) + b_A_32 = desc.load([i_t * BT + 32, 16]).to(tl.float32) + b_A_41 = desc.load([i_t * BT + 48, 0]).to(tl.float32) + b_A_42 = desc.load([i_t * BT + 48, 16]).to(tl.float32) + b_A_43 = desc.load([i_t * BT + 48, 32]).to(tl.float32) + + b_Ai_21 = -tl.dot( + tl.dot(b_Ai_22, b_A_21, input_precision=DOT_PRECISION), + b_Ai_11, + input_precision=DOT_PRECISION, + ) + b_Ai_32 = -tl.dot( + tl.dot(b_Ai_33, b_A_32, input_precision=DOT_PRECISION), + b_Ai_22, + input_precision=DOT_PRECISION, + ) + b_Ai_43 = -tl.dot( + tl.dot(b_Ai_44, b_A_43, input_precision=DOT_PRECISION), + b_Ai_33, + input_precision=DOT_PRECISION, + ) + + b_Ai_31 = -tl.dot( + b_Ai_33, + tl.dot(b_A_31, b_Ai_11, input_precision=DOT_PRECISION) + + tl.dot(b_A_32, b_Ai_21, input_precision=DOT_PRECISION), + input_precision=DOT_PRECISION, + ) + b_Ai_42 = -tl.dot( + b_Ai_44, + tl.dot(b_A_42, b_Ai_22, input_precision=DOT_PRECISION) + + tl.dot(b_A_43, b_Ai_32, input_precision=DOT_PRECISION), + input_precision=DOT_PRECISION, + ) + b_Ai_41 = -tl.dot( + b_Ai_44, + tl.dot(b_A_41, b_Ai_11, input_precision=DOT_PRECISION) + + tl.dot(b_A_42, b_Ai_21, input_precision=DOT_PRECISION) + + tl.dot(b_A_43, b_Ai_31, input_precision=DOT_PRECISION), + input_precision=DOT_PRECISION, + ) + + if not USE_TMA: + p_Ai_11 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT, 0), (16, 16), (1, 0) + ) + p_Ai_22 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT + 16, 16), (16, 16), (1, 0) + ) + p_Ai_33 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT + 32, 32), (16, 16), (1, 0) + ) + p_Ai_44 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT + 48, 48), (16, 16), (1, 0) + ) + p_Ai_21 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT + 16, 0), (16, 16), (1, 0) + ) + p_Ai_31 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT + 32, 0), (16, 16), (1, 0) + ) + p_Ai_32 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT + 32, 16), (16, 16), (1, 0) + ) + p_Ai_41 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT + 48, 0), (16, 16), (1, 0) + ) + p_Ai_42 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT + 48, 16), (16, 16), (1, 0) + ) + p_Ai_43 = tl.make_block_ptr( + Ai, (T, BT), (H * BT, 1), (i_t * BT + 48, 32), (16, 16), (1, 0) + ) + tl.store( + p_Ai_11, + b_Ai_11.to(p_Ai_11.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_22, + b_Ai_22.to(p_Ai_22.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_33, + b_Ai_33.to(p_Ai_33.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_44, + b_Ai_44.to(p_Ai_44.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_21, + b_Ai_21.to(p_Ai_21.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_31, + b_Ai_31.to(p_Ai_31.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_32, + b_Ai_32.to(p_Ai_32.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_41, + b_Ai_41.to(p_Ai_41.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_42, + b_Ai_42.to(p_Ai_42.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + tl.store( + p_Ai_43, + b_Ai_43.to(p_Ai_43.dtype.element_ty, fp_downcast_rounding="rtne"), + boundary_check=(0, 1), + ) + else: + desc_o.store( + [i_t * BT + 0, 0], b_Ai_11.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + desc_o.store( + [i_t * BT + 16, 16], b_Ai_22.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + desc_o.store( + [i_t * BT + 32, 32], b_Ai_33.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + desc_o.store( + [i_t * BT + 48, 48], b_Ai_44.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + desc_o.store( + [i_t * BT + 16, 0], b_Ai_21.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + desc_o.store( + [i_t * BT + 32, 0], b_Ai_31.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + desc_o.store( + [i_t * BT + 32, 16], b_Ai_32.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + desc_o.store( + [i_t * BT + 48, 0], b_Ai_41.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + desc_o.store( + [i_t * BT + 48, 16], b_Ai_42.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + desc_o.store( + [i_t * BT + 48, 32], b_Ai_43.to(desc_o.dtype, fp_downcast_rounding="rtne") + ) + + +@input_guard +def solve_tril( + A: torch.Tensor, + cu_seqlens: torch.Tensor | None = None, + chunk_indices: torch.Tensor | None = None, + output_dtype: torch.dtype = torch.float, +) -> torch.Tensor: + """ + Compute the inverse of the matrix I + A + A should be strictly lower triangular, i.e., A.triu() == 0. + + Args: + A (torch.Tensor): + [B, T, H, BT], where BT should only be 16, 32, or 64. + cu_seqlens (torch.Tensor): + The cumulative sequence lengths of the input tensor. Default: `None`. + chunk_indices (torch.Tensor): + Pre-computed chunk indices. Default: `None`. + output_dtype (torch.dtype): + The dtype of the output tensor. Default: `torch.float`. + If `None`, the output dtype will be the same as the input dtype. + + Returns: + (I + A)^-1 with the same shape as A + """ + assert A.shape[-1] in [16, 32, 64] + output_dtype = A.dtype if output_dtype is None else output_dtype + + B, T, H, BT = A.shape + if chunk_indices is None and cu_seqlens is not None: + chunk_indices = prepare_chunk_indices(cu_seqlens, BT) + NT = len(chunk_indices) if cu_seqlens is not None else triton.cdiv(T, BT) + + Ai = torch.zeros_like(A, dtype=output_dtype) + if BT == 16: + merge_fn = solve_tril_16x16_kernel + elif BT == 32: + merge_fn = merge_16x16_to_32x32_inverse_kernel + elif BT == 64: + merge_fn = merge_16x16_to_64x64_inverse_kernel + + merge_fn[NT, B * H]( + A=A, + Ai=Ai, + cu_seqlens=cu_seqlens, + chunk_indices=chunk_indices, + T=T, + H=H, + BT=BT, + USE_TMA=is_tma_supported, + DOT_PRECISION=FLA_TRIL_PRECISION, + ) + return Ai diff --git a/python/freetoken/kernel/fla/utils.py b/python/freetoken/kernel/fla/utils.py index e9e0b69740..a09993119e 100644 --- a/python/freetoken/kernel/fla/utils.py +++ b/python/freetoken/kernel/fla/utils.py @@ -280,6 +280,20 @@ def _check_platform() -> Literal["nvidia", "amd", "intel", "musa"]: is_tf32_supported = is_nvidia and torch.cuda.get_device_capability()[0] >= 8 is_gather_supported = hasattr(triton.language, "gather") +# Shared chunk length of the fla chunked kernels (KDA vendor reads it from here). +FLA_CHUNK_SIZE = 64 + +# TMA descriptors (solve_tril fast path): Hopper+, opt-in via FLA_USE_TMA=1, and only +# when this triton exposes a make_tensor_descriptor (see kernel/fla/op.py). +is_tma_supported = ( + is_nvidia_hopper + and os.getenv("FLA_USE_TMA", "0") == "1" + and ( + hasattr(triton.language, "_experimental_make_tensor_descriptor") + or hasattr(triton.language, "make_tensor_descriptor") + ) +) + def get_all_max_shared_mem(): try: diff --git a/python/freetoken/kernel/gguf.py b/python/freetoken/kernel/gguf.py index 04a1656098..a7eb4f74da 100644 --- a/python/freetoken/kernel/gguf.py +++ b/python/freetoken/kernel/gguf.py @@ -20,6 +20,100 @@ import torch _CSRC = pathlib.Path(__file__).parent / "csrc" / "gguf" +# This optional switch selects only the output-row grouping of the vendored +# GGUF MMV kernels. It is intentionally limited to the reviewed values below +# because arbitrary workgroup shapes require separate kernel review. +_HIP_GGUF_MMV_Y_ENV = "FREETOKEN_GGUF_MMV_Y" + + +def _hip_target_arch() -> str | None: + """Return the active AMD GPU target in ``gfxNNNN`` form when HIP exposes it. + + PyTorch's extension builder otherwise emits code for every visible AMD target. + A one-GPU serving process only needs the active target, so preserving an explicit + user selection or deriving the target from the active device avoids unnecessary + JIT work and records the architecture in the extension build key. + """ + explicit = os.environ.get("PYTORCH_ROCM_ARCH", "").strip() + if explicit: + return explicit.split(";", 1)[0].strip() + if not torch.cuda.is_available(): + return None + arch = getattr(torch.cuda.get_device_properties(0), "gcnArchName", "") + return str(arch).split(":", 1)[0] or None + + +def _hip_gguf_cflags() -> list[str]: + """Build conservative HIP GGUF flags for the active AMD GPU target. + + The architecture environment variable is set before PyTorch asks hipcc to + compile, which makes the cache target-specific without overriding a deployment's + explicit multi-target configuration. Keep floating-point flags conservative: + the native GGUF kernels must preserve model output, and unsupported aggressive + math flags belong only in isolated benchmark experiments. + """ + target = _hip_target_arch() + if target and not os.environ.get("PYTORCH_ROCM_ARCH"): + os.environ["PYTORCH_ROCM_ARCH"] = target + # Default to the established one-row configuration. The two-row, + # four-row, and eight-row candidates are permitted only for separately + # recorded builds. + # Neither option can affect a serving configuration unless it passes model + # quality and repeatable performance gates on the target AMD GPU. + mmv_y = os.environ.get(_HIP_GGUF_MMV_Y_ENV, "1").strip() + if mmv_y not in {"1", "2", "4", "8"}: + raise RuntimeError( + f"{_HIP_GGUF_MMV_Y_ENV} must be 1, 2, 4, or 8, got {mmv_y!r}" + ) + return ["-O3", f"-DGGML_CUDA_MMV_Y={mmv_y}"] + + +def _hip_thrust_include() -> str | None: + """Return a ROCm developer include directory that exposes ``thrust/complex.h``. + + The PyTorch ROCm wheel bundles hipcc but may omit the header-only Thrust + dependency required by libtorch's HIP complex header. Prefer explicitly + configured ROCm homes, then inspect the standard versioned installation + layout. Returning ``None`` leaves hosts with a complete wheel toolchain + unchanged. + """ + candidates = [ + os.environ.get("ROCM_HOME"), + os.environ.get("ROCM_PATH"), + "/opt/rocm", + ] + candidates.extend(str(path) for path in sorted(pathlib.Path("/opt").glob("rocm-*"), reverse=True)) + for root in candidates: + if not root: + continue + include = pathlib.Path(root) / "include" + if (include / "thrust" / "complex.h").is_file(): + return str(include) + return None + + +def _hip_runtime_library_dir() -> str | None: + """Return a ROCm library directory that can satisfy ``-lamdhip64``. + + Some PyTorch ROCm wheels ship ``libamdhip64.so.7`` but not the unversioned + linker name that ``torch.utils.cpp_extension`` emits. A regular ROCm + installation supplies that linker name under its ``lib`` directory. Keep + this discovery separate from the Thrust fallback so a host can provide one + dependency through the wheel and the other through its ROCm installation. + """ + candidates = [ + os.environ.get("ROCM_HOME"), + os.environ.get("ROCM_PATH"), + "/opt/rocm", + ] + candidates.extend(str(path) for path in sorted(pathlib.Path("/opt").glob("rocm-*"), reverse=True)) + for root in candidates: + if not root: + continue + for lib_dir in (pathlib.Path(root) / "lib", pathlib.Path(root) / "lib64"): + if (lib_dir / "libamdhip64.so").is_file(): + return str(lib_dir) + return None def _host_compiler() -> str | None: @@ -51,24 +145,50 @@ def _c_compiler_for(cxx: str) -> str: def _module(): from torch.utils.cpp_extension import load - extra_cuda_cflags = ["-O3", "--expt-relaxed-constexpr"] - host_cxx = _host_compiler() - if host_cxx is not None: - # Point both nvcc's host pass (-ccbin) and torch's C++ compile (CXX) at a - # libtorch/nvcc-compatible compiler. Force (not setdefault): the system - # default (CXX unset -> g++) can be a gcc too new for the torch headers. - cxx_path = shutil.which(host_cxx) or host_cxx - extra_cuda_cflags += ["-ccbin", cxx_path] - os.environ["CXX"] = cxx_path - os.environ["CC"] = _c_compiler_for(cxx_path) + if torch.version.hip is not None: + # Neither issue -ccbin works around applies under hipcc: it has no separate + # nvcc-style host pass (its own bundled clang IS the host compiler), and + # --expt-relaxed-constexpr is an nvcc-only flag hipcc/clang rejects outright. + extra_cuda_cflags = _hip_gguf_cflags() + # The minimal PyTorch ROCm SDK can omit Thrust while libtorch's HIP + # headers include it. Add a real system ROCm developer include only + # when present, retaining the wheel-only build on complete installs. + # This must be a compiler flag, not ``extra_include_paths``: PyTorch's + # hipify pass recursively rewrites every extension include path and + # cannot write beneath the read-only system ROCm installation. + hip_thrust_include = _hip_thrust_include() + hip_runtime_library_dir = _hip_runtime_library_dir() + extra_include_paths = [str(_CSRC)] + extra_ldflags: list[str] = [] + if hip_thrust_include is not None: + extra_cuda_cflags += ["-isystem", hip_thrust_include] + if hip_runtime_library_dir is not None: + # The extension linker uses ``-lamdhip64``. Add a real ROCm + # library directory only when the wheel SDK lacks its unversioned + # linker symlink, preserving self-contained wheel installations. + extra_ldflags += [f"-L{hip_runtime_library_dir}"] + else: + extra_cuda_cflags = ["-O3", "--expt-relaxed-constexpr"] + host_cxx = _host_compiler() + if host_cxx is not None: + # Point both nvcc's host pass (-ccbin) and torch's C++ compile (CXX) at a + # libtorch/nvcc-compatible compiler. Force (not setdefault): the system + # default (CXX unset -> g++) can be a gcc too new for the torch headers. + cxx_path = shutil.which(host_cxx) or host_cxx + extra_cuda_cflags += ["-ccbin", cxx_path] + os.environ["CXX"] = cxx_path + os.environ["CC"] = _c_compiler_for(cxx_path) + extra_include_paths = [str(_CSRC)] + extra_ldflags = [] # gguf_kernel.cu carries its own PYBIND11_MODULE (appended at the end), so a # plain `load` of the single source compiles + binds the ggml_* ops. return load( name="freetoken_gguf_kernels", sources=[str(_CSRC / "gguf_kernel.cu")], - extra_include_paths=[str(_CSRC)], + extra_include_paths=extra_include_paths, extra_cuda_cflags=extra_cuda_cflags, + extra_ldflags=extra_ldflags, verbose=True, ) diff --git a/python/freetoken/kernel/triton/activation.py b/python/freetoken/kernel/triton/activation.py index 2c38b533e6..a5c75de2be 100644 --- a/python/freetoken/kernel/triton/activation.py +++ b/python/freetoken/kernel/triton/activation.py @@ -23,6 +23,13 @@ from freetoken.utils.arch import is_sm90_supported +# _fast_tanh/_fast_ex2 below inline raw PTX text (tanh.approx.f32, ex2.approx.f32) via +# tl.inline_asm_elementwise. HIP's inline-asm path doesn't reject PTX outright -- it +# fails much later, deep in register allocation ("couldn't allocate output register +# for constraint 'f'"), since the constraint syntax is generic LLVM inline-asm but the +# instruction text is NVIDIA-ISA-only. Route ROCm through portable tl/libdevice ops. +_IS_HIP = tl.constexpr(torch.version.hip is not None) + SILU = 0 GELU = 1 GELU_TANH = 2 @@ -31,6 +38,8 @@ # variant lives in triton/mxfp4_moe.py): # y = clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1) SWIGLUOAI = 3 +# GLM-5.3 "swiglu_limit": swigluoai's clamped form WITHOUT the (up + 1) bias. +SWIGLU_CLAMP = 4 _SQRT_2_OVER_PI = 0.7978845608028654 # sqrt(2/pi) _GELU_C = 0.044715 @@ -48,6 +57,8 @@ def _pdl_supported() -> bool: @triton.jit def _fast_tanh(x): + if _IS_HIP: + return libdevice.tanh(x) # PTX tanh.approx.f32 — single HW op, matches flashinfer math::tanh. return tl.inline_asm_elementwise( "tanh.approx.f32 $0, $1;", "=f,f", [x], @@ -57,6 +68,8 @@ def _fast_tanh(x): @triton.jit def _fast_ex2(x): + if _IS_HIP: + return tl.exp2(x) # PTX ex2.approx.f32 — matches __expf fast path used by flashinfer silu. return tl.inline_asm_elementwise( "ex2.approx.f32 $0, $1;", "=f,f", [x], @@ -105,6 +118,11 @@ def _act_and_mul_kernel( up = tl.minimum(tl.maximum(up, -limit), limit) act = gate / (1.0 + _fast_ex2(-gate * alpha * _LOG2E)) y = act * (up + 1.0) + elif ACT == 4: # SWIGLU_CLAMP (GLM-5.3): swigluoai without the (up + 1) bias + gate = tl.minimum(gate, limit) + up = tl.minimum(tl.maximum(up, -limit), limit) + act = gate / (1.0 + _fast_ex2(-gate * alpha * _LOG2E)) + y = act * up else: # GELU (erf) act = 0.5 * gate * (1.0 + libdevice.erf(gate * 0.7071067811865476)) y = act * up @@ -129,12 +147,16 @@ def _act_and_mul( M = x2.shape[0] grid = lambda meta: (M, triton.cdiv(d, meta["BLOCK_D"])) pdl = _pdl_supported() + # launch_pdl is a CUDA-Hopper-only Triton launch kwarg; the AMD backend's + # arg-packer rejects it outright (KeyError) even when passed as False, so it + # is only included on the one backend/arch combination that ever sets pdl=True. + pdl_kwargs = {"launch_pdl": pdl} if pdl else {} # Fixed via H100 sweep (72-config grid; 512/w4/s3 within 11% everywhere, # 1024/w4/s2 best at rows>=4096). block_d = min(triton.next_power_of_2(d), 1024 if M >= 4096 else 512) num_stages = 2 if block_d == 1024 else 3 _act_and_mul_kernel[grid]( - o2, x2, d, alpha, limit, ACT=kind, ENABLE_PDL=pdl, launch_pdl=pdl, + o2, x2, d, alpha, limit, ACT=kind, ENABLE_PDL=pdl, **pdl_kwargs, BLOCK_D=block_d, num_warps=4, num_stages=num_stages, ) return out @@ -166,4 +188,21 @@ def swigluoai_and_mul( return _act_and_mul(SWIGLUOAI, x, out, alpha=alpha, limit=limit) -__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul", "swigluoai_and_mul"] +def swiglu_clamp_and_mul( + x: torch.Tensor, + out: torch.Tensor | None = None, + *, + alpha: float = 1.0, + limit: float = 10.0, +) -> torch.Tensor: + """GLM-5.3 clamped SwiGLU over UNINTERLEAVED halves: ``clamp(gate, max=limit) * sigmoid(alpha * gate) * clamp(up, +-limit)``.""" + return _act_and_mul(SWIGLU_CLAMP, x, out, alpha=alpha, limit=limit) + + +__all__ = [ + "silu_and_mul", + "gelu_and_mul", + "gelu_tanh_and_mul", + "swigluoai_and_mul", + "swiglu_clamp_and_mul", +] diff --git a/python/freetoken/kernel/triton/attention.py b/python/freetoken/kernel/triton/attention.py index c2358d84fa..d070f37014 100644 --- a/python/freetoken/kernel/triton/attention.py +++ b/python/freetoken/kernel/triton/attention.py @@ -364,8 +364,17 @@ def decode_paged_attention( sliding_window: int | None = None, sinks: torch.Tensor | None = None, out: torch.Tensor | None = None, + rocm_block_h_probe: int | None = None, + rocm_block_n_probe: int | None = None, + rocm_num_warps_probe: int | None = None, ) -> torch.Tensor: - """SGLang-style split-k grouped decode attention for one query per request.""" + """SGLang-style split-k grouped decode attention for one query per request. + + The ``rocm_*_probe`` arguments are benchmark-only HIP controls. They let + GMKtek EVO-X2 measure a query-head tile, KV block length, or launch warp count + without changing the serving defaults. Normal callers leave every probe + argument ``None`` and preserve the established ROCm configuration. + """ assert q.is_cuda and k_cache.is_cuda and v_cache.is_cuda assert q.dim() == 3 and k_cache.dim() == 3 and v_cache.dim() == 3 @@ -396,8 +405,35 @@ def decode_paged_attention( # (e.g. 6), where block_h rounds up and the kernel masks the extra lanes. valid_block_h = min(16, group) block_h = triton.next_power_of_2(valid_block_h) + if rocm_block_h_probe is not None: + if torch.version.hip is None: + raise ValueError("rocm_block_h_probe is only valid for HIP builds") + if rocm_block_h_probe < valid_block_h or rocm_block_h_probe & (rocm_block_h_probe - 1): + raise ValueError("rocm_block_h_probe must be a power of two at least valid_block_h") + block_h = rocm_block_h_probe + elif torch.version.hip is not None: + # RDNA WMMA has no matrix-core instruction below a 16x16 tile, so a decode + # GQA group smaller than 16 (e.g. 4 here) leaves tl.dot's M dim too small to + # lower on this backend. The kernel already masks lanes >= VALID_BLOCK_H + # (it does this for non-power-of-two groups too), so padding BLOCK_H up to + # 16 is safe -- it only adds masked-out, discarded head lanes. + block_h = max(block_h, 16) block_d = triton.next_power_of_2(head_dim) block_dv = triton.next_power_of_2(head_dim) + block_n = 32 + num_warps = 4 + if rocm_block_n_probe is not None: + if torch.version.hip is None: + raise ValueError("rocm_block_n_probe is only valid for HIP builds") + if rocm_block_n_probe < 16 or rocm_block_n_probe & (rocm_block_n_probe - 1): + raise ValueError("rocm_block_n_probe must be a power of two at least 16") + block_n = rocm_block_n_probe + if rocm_num_warps_probe is not None: + if torch.version.hip is None: + raise ValueError("rocm_num_warps_probe is only valid for HIP builds") + if rocm_num_warps_probe not in (1, 2, 4, 8): + raise ValueError("rocm_num_warps_probe must be one of 1, 2, 4, or 8") + num_warps = rocm_num_warps_probe _decode_grouped_stage1_kernel[ (batch, triton.cdiv(num_q_heads, valid_block_h), max_kv_splits) @@ -428,14 +464,14 @@ def decode_paged_attention( NUM_Q_HEADS=num_q_heads, BLOCK_D=block_d, BLOCK_DV=block_dv, - BLOCK_N=32, + BLOCK_N=block_n, BLOCK_H=block_h, VALID_BLOCK_H=valid_block_h, MIN_BLOCK_KV=_MIN_BLOCK_KV, D=head_dim, DV=head_dim, SLIDING_WINDOW=sliding_window or 0, - num_warps=4, + num_warps=num_warps, num_stages=2, ) _decode_stage2_kernel[(batch, num_q_heads)]( @@ -597,6 +633,7 @@ def _extend_attention_split_kernel( kv_indptr_ptr, kv_indices_ptr, prefix_lens_ptr, + image_group_ids_ptr, sm_scale, sinks_ptr, stride_qt, @@ -619,6 +656,7 @@ def _extend_attention_split_kernel( BLOCK_N: tl.constexpr, SLIDING_WINDOW: tl.constexpr, HAS_SINKS: tl.constexpr, + HAS_IMAGE_GROUPS: tl.constexpr, ): seq_id = tl.program_id(0) q_head = tl.program_id(1) @@ -701,12 +739,25 @@ def _extend_attention_split_kernel( l_i = l_i * alpha + tl.sum(p, axis=1) m_i = m_new - current_end = tl.minimum(q_len, (block_m_id + 1) * BLOCK_M) + # Causal attention normally needs only keys through this query tile. Gemma + # 4 image soft tokens are the exception: every token in one image group can + # see the group's future tokens. Iterate over the full current extension + # only when a batch carries those group ids. + current_end = q_len if HAS_IMAGE_GROUPS else tl.minimum(q_len, (block_m_id + 1) * BLOCK_M) for start_n in tl.range(0, current_end, BLOCK_N): local_kv_offsets = start_n + offs_n mask_n = local_kv_offsets < current_end local_q_pos = offs_m causal_mask = local_kv_offsets[None, :] <= local_q_pos[:, None] + if HAS_IMAGE_GROUPS: + q_groups = tl.load(image_group_ids_ptr + q_start + offs_m, mask=mask_m, other=-1) + k_groups = tl.load( + image_group_ids_ptr + q_start + local_kv_offsets, mask=mask_n, other=-1 + ) + same_image_group = ( + (q_groups[:, None] >= 0) & (q_groups[:, None] == k_groups[None, :]) + ) + causal_mask = causal_mask | same_image_group if SLIDING_WINDOW > 0: causal_mask = causal_mask & ( (local_kv_offsets[None, :] + SLIDING_WINDOW) > local_q_pos[:, None] @@ -773,8 +824,14 @@ def extend_paged_attention( out: torch.Tensor | None = None, k_extend: torch.Tensor | None = None, v_extend: torch.Tensor | None = None, + image_group_ids: torch.Tensor | None = None, ) -> torch.Tensor: - """Block-tiled causal prefill/extend attention over paged KV cache.""" + """Block-tiled prefill attention over paged KV cache. + + Normal tokens use causal attention. When ``image_group_ids`` is supplied, + equal non-negative ids receive Gemma 4's bidirectional image-block + exception during this prefill only. + """ assert q.is_cuda and k_cache.is_cuda and v_cache.is_cuda assert q.dim() == 3 and k_cache.dim() == 3 and v_cache.dim() == 3 @@ -790,9 +847,15 @@ def extend_paged_attention( assert sinks.dim() == 1 assert sinks.numel() >= num_q_heads sinks = sinks.contiguous() + if image_group_ids is not None: + assert image_group_ids.is_cuda + assert image_group_ids.dim() == 1 + assert image_group_ids.numel() == num_q_tokens + image_group_ids = image_group_ids.contiguous() o = out if out is not None else torch.empty_like(q) sinks_arg = sinks if sinks is not None else q + image_groups_arg = image_group_ids if image_group_ids is not None else q block_d = triton.next_power_of_2(head_dim) block_dv = triton.next_power_of_2(head_dim) # Tile size is shared-memory bound: keep the fast (large) tiles on GPUs whose opt-in @@ -820,6 +883,7 @@ def extend_paged_attention( kv_indptr, kv_indices, prefix_lens, + image_groups_arg, sm_scale, sinks_arg, q.stride(0), @@ -842,6 +906,7 @@ def extend_paged_attention( BLOCK_N=block_n, SLIDING_WINDOW=sliding_window or 0, HAS_SINKS=sinks is not None, + HAS_IMAGE_GROUPS=image_group_ids is not None, num_warps=8, num_stages=1, ) diff --git a/python/freetoken/kernel/triton/e4m3_compat.py b/python/freetoken/kernel/triton/e4m3_compat.py index 1d9f744ce1..1c75ff2428 100644 --- a/python/freetoken/kernel/triton/e4m3_compat.py +++ b/python/freetoken/kernel/triton/e4m3_compat.py @@ -59,6 +59,14 @@ def e4m3_native() -> bool: if _native is None: if FORCE_EMU: _native = False + elif torch.version.hip is not None: + # torch.cuda.get_device_capability() on a HIP build returns the gfx/RDNA + # generation number (e.g. (11, 5) for gfx1150), not a CUDA compute + # capability -- comparing it against (8, 9) below is a tuple comparison + # over two unrelated numbering schemes and can false-positive (11 > 8). + # No AMD GPU has this fp8e4nv unit; e4m3_native_cx() (Triton's own, + # backend-aware check) already agrees this must be False. + _native = False else: from freetoken.gpu_select import assigned_visible_gpu @@ -88,6 +96,21 @@ def e4m3_native_cx(): return not FORCE_EMU and target_info.cuda_capability_geq(8, 9) +@jit +def e4m3_u8_to_f16(v): + """Decode an e4m3 byte to the exact fp16 value divided by 256. + + The e4m3 exponent and mantissa fit losslessly in fp16 after the bit-field + placement below. Callers that can move the compensating power-of-two scale + onto an activation use this primitive to avoid multiplying every decoded + weight by 256. The caller must preserve FP32 accumulation and apply the + reciprocal scaling exactly once, otherwise this is not numerically + equivalent to :func:`e4m3_u8_to_f32`. + """ + h = ((v & 0x80).to(tl.uint16) << 8) | ((v & 0x7F).to(tl.uint16) << 7) + return h.to(tl.float16, bitcast=True) + + @jit def e4m3_u8_to_f32(v): """Decode e4m3 bits (uint8) to fp32: place exp+mantissa in the fp16 field diff --git a/python/freetoken/kernel/triton/fp8_pertensor_linear.py b/python/freetoken/kernel/triton/fp8_pertensor_linear.py index 28a54c94e4..79b78fe6d1 100644 --- a/python/freetoken/kernel/triton/fp8_pertensor_linear.py +++ b/python/freetoken/kernel/triton/fp8_pertensor_linear.py @@ -19,7 +19,9 @@ from __future__ import annotations +import functools import os +import re import torch import triton @@ -31,6 +33,7 @@ e4m3_kernel_view, e4m3_native, e4m3_native_cx, + e4m3_u8_to_f16, e4m3_u8_to_f32, ) @@ -41,6 +44,100 @@ # (numeric reference / A-B debugging). Evaluated once; the kernels are the default. _USE_REF = os.environ.get("FREETOKEN_DEBUG_FP8_REF") == "1" +# The baseline emits sixteen independent output rows per split-K CTA. Radeon +# gfx1151 executes Wave32, so a thirty-two-row CTA is an occupancy candidate. +# The launch tile must not influence the split-K policy: otherwise changing +# output rows per CTA would silently change the K partition and numerical +# reduction grouping. Keep the compact allowlist deliberately +# narrow because arbitrary tile sizes would create undocumented kernels and +# make performance evidence impossible to compare across runs. The setting is +# read once at module import, which is safe because it is a compile-time Triton +# specialization and a server has one immutable runtime policy. +_GEMV_BLOCK_N = int(os.environ.get("FREETOKEN_FP8_GEMV_BLOCK_N", "16")) +if _GEMV_BLOCK_N not in (16, 32): + raise ValueError( + "FREETOKEN_FP8_GEMV_BLOCK_N must be 16 (validated baseline) or 32 " + "(quality-gated gfx1151 candidate)" + ) + +# One Wave32 is the validated baseline. Two and four waves are deliberately +# bounded candidates because they can improve memory-level parallelism on +# gfx1151 without changing the output tile or split-K policy. They still have to +# pass the same raw-output and model-level gates because Triton may lower a +# reduction differently when the launch wave count changes. +_GEMV_NUM_WARPS = int(os.environ.get("FREETOKEN_FP8_GEMV_NUM_WARPS", "1")) +if _GEMV_NUM_WARPS not in (1, 2, 4): + raise ValueError( + "FREETOKEN_FP8_GEMV_NUM_WARPS must be 1 (validated baseline), 2, or 4 " + "(quality-gated gfx1151 candidates)" + ) + +# ROCm must emulate e4m3 weight conversion. The optional candidate decodes a +# weight as its exact fp16 value divided by 256 and applies the compensating +# exact power-of-two scale once to the BF16 activation. This reduces repeated +# weight-side scale operations without changing the FP32 accumulator contract. +# It is off by default because compiler lowering must be verified by raw-output +# hashes and the full deterministic model-quality gate. +_GEMV_SCALE_ACTIVATION = os.environ.get("FREETOKEN_FP8_GEMV_SCALE_ACTIVATION") == "1" + + +# Row-wise _scaled_mm on sm_89 with torch < 2.12 launches its CUTLASS stream-K kernel off the +# current stream (pytorch/pytorch#177651, fixed by pytorch/pytorch@252bb4a; #182/#72/#220), and +# some builds (Windows) have no row-wise kernel (#227). Fallback: one tensor-wise GEMM per part. +def _torch_version() -> tuple[int, int]: + m = re.match(r"(\d+)\.(\d+)", torch.__version__) + return (int(m.group(1)), int(m.group(2))) if m else (0, 0) + + +@functools.cache +def rowwise_scaled_mm_ok() -> bool: + """Whether row-wise ``torch._scaled_mm`` may be issued from a side stream on this GPU. + Decided once per process, at load (never under graph capture). ``FREETOKEN_FP8_ROWWISE_MM=0/1`` + forces the answer.""" + forced = os.environ.get("FREETOKEN_FP8_ROWWISE_MM") + if forced in ("0", "1"): + return forced == "1" + if not torch.cuda.is_available(): + return True + from freetoken.gpu_select import assigned_visible_gpu + + idx = assigned_visible_gpu() + dev = torch.device("cuda", torch.cuda.current_device() if idx is None else idx) + if torch.cuda.get_device_capability(dev) == (8, 9) and _torch_version() < (2, 12): + return False + # Probe on the default stream (safe even where the launch ignores the current stream); a + # build without the row-wise kernel raises here instead of at the first forward. + try: + with torch.cuda.device(dev), torch.cuda.stream(torch.cuda.default_stream(dev)): + a = torch.zeros(16, 32, dtype=FP8, device=dev) + b = torch.zeros(32, 32, dtype=FP8, device=dev) + torch._scaled_mm( + a, b.t(), scale_a=torch.ones(16, 1, device=dev), + scale_b=torch.ones(1, 32, device=dev), out_dtype=torch.bfloat16, + ) + torch.cuda.synchronize(dev) + except RuntimeError: + return False + return True + + +def weight_scale_segments(weight_scale: torch.Tensor) -> list[tuple[int, int]]: + """``[start, end)`` row ranges over which ``weight_scale`` is constant (the fused parts). + Syncs; call at load.""" + s = weight_scale.detach().reshape(-1).float().cpu() + change = (torch.nonzero(s[1:] != s[:-1]).flatten() + 1).tolist() + bounds = [0, *change, s.numel()] + return list(zip(bounds[:-1], bounds[1:])) + + +_MAX_SEGMENTS = 8 # q/k/v = 3, GDN qkv|z = 2; a genuine per-row scale stays W8A16 instead + + +def _segments_w8a8_ok(segments: list[tuple[int, int]]) -> bool: + """cuBLASLt needs 16-row aligned fp8 operands; more parts than a fused projection has + means a genuine per-row scale.""" + return 0 < len(segments) <= _MAX_SEGMENTS and all((e - s) % 16 == 0 for s, e in segments) + # ====================================================================================== # Decode (M==1) split-K GEMV: raw fp8 x bf16 reduction in fp32, per-row scale at reduce. @@ -49,7 +146,7 @@ def _gemv_splitk_kernel( a_ptr, w_ptr, part_ptr, N, K, n_kb, kb_per, stride_ak, stride_wn, stride_wk, stride_pk, stride_pn, - BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr, + BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr, SCALE_ACTIVATION: tl.constexpr, ): """Each (pid_n, pid_k) computes the partial sum over ``kb_per`` BLOCK_K chunks for a BLOCK_N slice of outputs. ``kb_per`` ceil-tiles K so K only needs to be a multiple of @@ -66,16 +163,25 @@ def _gemv_splitk_kernel( offs_k = kb * BLOCK_K + tl.arange(0, BLOCK_K) k_mask = offs_k < K a = tl.load(a_ptr + offs_k * stride_ak, mask=k_mask, other=0.0).to(tl.float32) + # Scaling BF16 inputs by 256 is an exact exponent adjustment in + # fp32. Do it once per K element only for the quality-gated ROCm + # candidate; CUDA native e4m3 uses its normal direct conversion. + if SCALE_ACTIVATION and not e4m3_native_cx(): + a *= 256.0 if e4m3_native_cx(): w = tl.load( w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk, mask=n_mask[:, None] & k_mask[None, :], other=0.0, ).to(tl.float32) else: - w = e4m3_u8_to_f32(tl.load( + raw_w = tl.load( w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk, mask=n_mask[:, None] & k_mask[None, :], other=0, - )) + ) + if SCALE_ACTIVATION: + w = e4m3_u8_to_f16(raw_w).to(tl.float32) + else: + w = e4m3_u8_to_f32(raw_w) acc += tl.sum(w * a[None, :], axis=1) tl.store(part_ptr + pid_k * stride_pk + offs_n * stride_pn, acc, mask=n_mask) @@ -100,16 +206,23 @@ def _gemv(a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor, N, K = weight.shape BLOCK_K = 128 n_kb = triton.cdiv(K, BLOCK_K) - BLOCK_N = 16 + # This output-row tile may alter hardware occupancy and memory-transaction + # coalescing. The reference tile below deliberately holds split-K fixed, + # so it cannot also alter a row's K partition or final reduction order. + # All non-baseline values still require the full deterministic model-quality + # gate before they can become a default. + BLOCK_N = _GEMV_BLOCK_N n_tiles = triton.cdiv(N, BLOCK_N) - split_k = max(1, min(1536 // n_tiles, n_kb)) + baseline_n_tiles = triton.cdiv(N, 16) + split_k = max(1, min(1536 // baseline_n_tiles, n_kb)) split_k = 1 << (split_k.bit_length() - 1) # pow2 -> stable reduction order kb_per = triton.cdiv(n_kb, split_k) part = torch.empty((split_k, N), dtype=torch.float32, device=a.device) _gemv_splitk_kernel[(n_tiles, split_k)]( a, weight, part, N, K, n_kb, kb_per, a.stride(0), weight.stride(0), weight.stride(1), part.stride(0), part.stride(1), - BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K, num_warps=1, + BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K, SCALE_ACTIVATION=_GEMV_SCALE_ACTIVATION, + num_warps=_GEMV_NUM_WARPS, ) out = torch.empty(N, dtype=out_dtype, device=a.device) _splitk_reduce_kernel[(triton.cdiv(N, 256),)]( @@ -222,6 +335,7 @@ def _static_quant(a: torch.Tensor, input_scale: torch.Tensor) -> torch.Tensor: def _scaled_mm( a: torch.Tensor, weight: torch.Tensor, weight_scale: torch.Tensor, input_scale: torch.Tensor, uniform_scale: bool, out_dtype: torch.dtype, + scale_segments: list[tuple[int, int]] | None = None, ) -> torch.Tensor: """``a @ (weight_fp8 * weight_scale)^T`` as a W8A8 cuBLASLt GEMM. @@ -233,14 +347,26 @@ def _scaled_mm( tensor-wise path. A fused projection, whose ``weight_scale`` is piecewise-constant because each part carries its own scalar, takes the row-wise path -- that keeps every part's scale exact, where vLLM/SGLang instead requantize the parts onto a shared maximum - and eat the precision loss. Row-wise costs ~4% here (5.56 ms vs 5.39 ms per step).""" + and eat the precision loss. Row-wise costs ~4% here (5.56 ms vs 5.39 ms per step). + + Where row-wise is unsafe (:func:`rowwise_scaled_mm_ok`) ``scale_segments`` is passed and + each part runs its own tensor-wise GEMM over ``weight[s:e]`` (still stride-only), outputs + concatenated: the same W8A8 scheme, not bit-identical (accumulation order differs).""" qa = _static_quant(a, input_scale) wt = weight.t() # [N, K] row-major -> [K, N] column-major, stride-only + sa = input_scale.reshape(()) if uniform_scale: return torch._scaled_mm( - qa, wt, scale_a=input_scale.reshape(()), scale_b=weight_scale[0].reshape(()), - out_dtype=out_dtype, + qa, wt, scale_a=sa, scale_b=weight_scale[0].reshape(()), out_dtype=out_dtype, ) + if scale_segments is not None: + return torch.cat([ + torch._scaled_mm( + qa, weight[s:e].t(), scale_a=sa, scale_b=weight_scale[s].reshape(()), + out_dtype=out_dtype, + ) + for s, e in scale_segments + ], dim=1) return torch._scaled_mm( qa, wt, scale_a=input_scale.reshape(1, 1).expand(a.shape[0], 1).contiguous(), @@ -254,9 +380,11 @@ def fp8_pertensor_linear( bias: torch.Tensor | None = None, input_scale: torch.Tensor | None = None, uniform_scale: bool = False, + scale_segments: list[tuple[int, int]] | None = None, ) -> torch.Tensor: """``y = x @ (weight_fp8 * weight_scale)^T``. ``weight`` [N, K] fp8-e4m3, ``weight_scale`` - [N] fp32 (per output row). + [N] fp32 (per output row). ``scale_segments``: the fused parts' row ranges, precomputed at + load by the layer; derived here (with a sync) when omitted and needed. Whether the activation is quantized is a property of the *deployment*, never of the batch: with ``input_scale`` on sm_89+ every M runs W8A8, otherwise every M runs W8A16 (split-K @@ -266,12 +394,19 @@ def fp8_pertensor_linear( SGLang likewise run one scheme across all M on any GPU with FP8 tensor cores.""" *lead, K = x.shape N = weight.shape[0] + w8a8 = input_scale is not None and e4m3_native() + segments = None + if w8a8 and not uniform_scale and not rowwise_scaled_mm_ok(): + segments = scale_segments if scale_segments is not None else weight_scale_segments(weight_scale) + if not _segments_w8a8_ok(segments): + w8a8 = False # W8A16 below is exact for any per-row scale and never calls _scaled_mm if _USE_REF: # numeric-reference fallback (debug / A-B) w = weight.to(x.dtype) * weight_scale.to(x.dtype)[:, None] out = (x.reshape(-1, K) @ w.t()).reshape(*lead, N) - elif input_scale is not None and e4m3_native(): + elif w8a8: out = _scaled_mm( x.reshape(-1, K), weight, weight_scale, input_scale, uniform_scale, x.dtype, + scale_segments=segments, ).reshape(*lead, N) elif x.numel() // K == 1: out = _gemv(x.reshape(K), e4m3_kernel_view(weight), weight_scale, x.dtype).reshape(*lead, N) @@ -306,6 +441,7 @@ def __init__(self, in_features: int, out_features: int, has_bias: bool = False): # reflective state_dict/load_state_dict skip it entirely on checkpoints without one. self.input_scale: torch.Tensor | None = None self._uniform_scale = False + self._scale_segments: list[tuple[int, int]] | None = None def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = False) -> None: # Taken out before BaseOP's reflective pass (so it is not an "unexpected key") and @@ -318,11 +454,15 @@ def load_state_dict(self, state_dict, *, prefix: str = "", _internal: bool = Fal # only piecewise-constant, so decide once here rather than syncing on every forward. scale = self.weight_scale self._uniform_scale = bool((scale == scale[0]).all().item()) + # Segments for the per-part path; decide row-wise safety now, not under graph capture. + self._scale_segments = None if self._uniform_scale else weight_scale_segments(scale) + if self.input_scale is not None and not self._uniform_scale: + rowwise_scaled_mm_ok() def forward(self, x: torch.Tensor) -> torch.Tensor: return fp8_pertensor_linear( x, self.weight, self.weight_scale, self.bias, - self.input_scale, self._uniform_scale, + self.input_scale, self._uniform_scale, scale_segments=self._scale_segments, ) @@ -341,4 +481,6 @@ def __init__(self, in_features: int, output_sizes: list[int], has_bias: bool = F "Fp8PerTensorLinear", "Fp8PerTensorColMerged", "fp8_pertensor_linear", + "rowwise_scaled_mm_ok", + "weight_scale_segments", ] diff --git a/python/freetoken/kernel/triton/glm_dsa_sparse.py b/python/freetoken/kernel/triton/glm_dsa_sparse.py index d3219f950f..8aaeeb9ba9 100644 --- a/python/freetoken/kernel/triton/glm_dsa_sparse.py +++ b/python/freetoken/kernel/triton/glm_dsa_sparse.py @@ -49,6 +49,7 @@ def _glm_dsa_sparse_kernel( BLOCK_H: tl.constexpr, BLOCK_T: tl.constexpr, HAS_COUNTS: tl.constexpr, + HAS_ROPE: tl.constexpr, ): pid_m = tl.program_id(0) pid_b = tl.program_id(1) @@ -57,11 +58,14 @@ def _glm_dsa_sparse_kernel( offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) h_mask = offs_h < H offs_v = tl.arange(0, D_V) - offs_r = tl.arange(0, D_R) q_base = q_ptr + pid_b * stride_qb + pid_m * stride_qm + offs_h[:, None] * stride_qh q_v = tl.load(q_base + offs_v[None, :] * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32) - q_r = tl.load(q_base + (D_V + offs_r[None, :]) * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32) + if HAS_ROPE: + # NoPE checkpoints (glm5_next) have D_R == 0: tl.arange needs a non-empty + # span, so the whole rope half is compiled out on the constexpr. + offs_r = tl.arange(0, D_R) + q_r = tl.load(q_base + (D_V + offs_r[None, :]) * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32) m_i = tl.full((BLOCK_H,), -float("inf"), dtype=tl.float32) l_i = tl.zeros((BLOCK_H,), dtype=tl.float32) @@ -79,9 +83,12 @@ def _glm_dsa_sparse_kernel( valid = idxs >= 0 kv_base = pool_ptr + idxs[:, None] * stride_pn kv_v = tl.load(kv_base + offs_v[None, :] * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32) - kv_r = tl.load(kv_base + (D_V + offs_r[None, :]) * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32) - scores = (tl.dot(q_v, tl.trans(kv_v)) + tl.dot(q_r, tl.trans(kv_r))) * scale + scores = tl.dot(q_v, tl.trans(kv_v)) + if HAS_ROPE: + kv_r = tl.load(kv_base + (D_V + offs_r[None, :]) * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32) + scores += tl.dot(q_r, tl.trans(kv_r)) + scores = scores * scale scores = tl.where(valid[None, :], scores, -float("inf")) m_new = tl.maximum(m_i, tl.max(scores, axis=1)) @@ -207,6 +214,7 @@ def _glm_dsa_splitk_kernel( BLOCK_H: tl.constexpr, BLOCK_T: tl.constexpr, HAS_COUNTS: tl.constexpr, + HAS_ROPE: tl.constexpr, NUM_SPLITS: tl.constexpr, ): """Stage 1 (decode flash-decoding): each program reduces one BLOCK_T-aligned slice of @@ -221,7 +229,6 @@ def _glm_dsa_splitk_kernel( offs_h = pid_h * BLOCK_H + tl.arange(0, BLOCK_H) h_mask = offs_h < H offs_v = tl.arange(0, D_V) - offs_r = tl.arange(0, D_R) n_active = TOPK if HAS_COUNTS: @@ -238,7 +245,9 @@ def _glm_dsa_splitk_kernel( if split_end > split_start: q_base = q_ptr + pid_b * stride_qb + pid_m * stride_qm + offs_h[:, None] * stride_qh q_v = tl.load(q_base + offs_v[None, :] * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32) - q_r = tl.load(q_base + (D_V + offs_r[None, :]) * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32) + if HAS_ROPE: + offs_r = tl.arange(0, D_R) + q_r = tl.load(q_base + (D_V + offs_r[None, :]) * stride_qd, mask=h_mask[:, None], other=0.0).to(tl.float32) idx_base = idx_ptr + pid_b * stride_ib + pid_m * stride_im for start in range(split_start, split_end, BLOCK_T): @@ -248,9 +257,12 @@ def _glm_dsa_splitk_kernel( valid = idxs >= 0 kv_base = pool_ptr + idxs[:, None] * stride_pn kv_v = tl.load(kv_base + offs_v[None, :] * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32) - kv_r = tl.load(kv_base + (D_V + offs_r[None, :]) * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32) - scores = (tl.dot(q_v, tl.trans(kv_v)) + tl.dot(q_r, tl.trans(kv_r))) * scale + scores = tl.dot(q_v, tl.trans(kv_v)) + if HAS_ROPE: + kv_r = tl.load(kv_base + (D_V + offs_r[None, :]) * stride_pd, mask=valid[:, None], other=0.0).to(tl.float32) + scores += tl.dot(q_r, tl.trans(kv_r)) + scores = scores * scale scores = tl.where(valid[None, :], scores, -float("inf")) m_new = tl.maximum(m_i, tl.max(scores, axis=1)) @@ -384,7 +396,7 @@ def glm_dsa_sparse_attn( stride_nb, stride_nm, D_V=d_v, D_R=d_r, BLOCK_H=BLOCK_H, BLOCK_T=BLOCK_T, - HAS_COUNTS=has_counts, NUM_SPLITS=n_splits, + HAS_COUNTS=has_counts, HAS_ROPE=d_r > 0, NUM_SPLITS=n_splits, num_warps=4, num_stages=2, ) grid2 = (m, b, h) @@ -410,7 +422,7 @@ def glm_dsa_sparse_attn( stride_nb, stride_nm, D_V=d_v, D_R=d_r, BLOCK_H=BLOCK_H, BLOCK_T=BLOCK_T, - HAS_COUNTS=has_counts, + HAS_COUNTS=has_counts, HAS_ROPE=d_r > 0, num_warps=4, num_stages=2, ) return o diff --git a/python/freetoken/kernel/triton/hc.py b/python/freetoken/kernel/triton/hc.py new file mode 100644 index 0000000000..45d71bd731 --- /dev/null +++ b/python/freetoken/kernel/triton/hc.py @@ -0,0 +1,395 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/hc.py) +"""NVIDIA HyperConnection kernels for Qwen4Exp.""" + +from __future__ import annotations + +import functools + +import torch +import triton +import triton.language as tl + +from freetoken.utils.arch import is_sm90_supported + + +@functools.cache +def _pdl_supported() -> bool: + return is_sm90_supported() + + +@triton.jit +def _grouped_gemma_rmsnorm_kernel( + x_ptr, + w_ptr, + y_ptr, + stride_x, + stride_y, + DIM: tl.constexpr, + NUM_GROUPS: tl.constexpr, + W_SHARED: tl.constexpr, + EPS: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + GROUP_DIM: tl.constexpr = DIM // NUM_GROUPS + BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(GROUP_DIM) + + pid = tl.program_id(0) + group_id = pid % NUM_GROUPS + # row * stride can overflow int32 for large token counts. + row = (pid // NUM_GROUPS).to(tl.int64) + + offs_g = tl.arange(0, BLOCK_SIZE) + offsets = group_id * GROUP_DIM + offs_g + mask = offs_g < GROUP_DIM + # A [GROUP_DIM] affine is shared; a [DIM] affine follows the grouped + # checkpoint layout. + w_offs = offs_g if W_SHARED else offsets + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0).to(tl.float32) + w = tl.load(w_ptr + w_offs, mask, other=0.0) + + rrms = tl.rsqrt(tl.sum(x * x) / GROUP_DIM + EPS) + # Gemma's (1 + w) affine is written this way to lower to an FMA. + y = x * rrms + y += y * w.to(tl.float32) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(y_ptr + row * stride_y + offsets, y, mask) + + +def grouped_gemma_rmsnorm( + x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int +) -> torch.Tensor: + N, DIM = x.shape + assert x.stride(1) == 1, "grouped Gemma RMSNorm requires unit inner stride" + assert weight.is_contiguous(), "grouped Gemma RMSNorm weight must be contiguous" + assert DIM % num_groups == 0 + group_dim = DIM // num_groups + assert weight.numel() in (group_dim, DIM) + + y = x.new_empty(x.shape) + _grouped_gemma_rmsnorm_kernel[(N * num_groups,)]( + x, + weight, + y, + x.stride(0), + y.stride(0), + DIM, + num_groups, + W_SHARED=weight.numel() == group_dim, + EPS=eps, + launch_pdl=_pdl_supported(), + ) + return y + + +@triton.jit +def _hc_silu_kernel( + x_ptr, + y_ptr, + stride_x, + stride_y, + DIM: tl.constexpr, + HC: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(DIM) + + row = tl.program_id(0).to(tl.int64) + offs = tl.arange(0, BLOCK_SIZE) + mask = offs < DIM + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + x = tl.load(x_ptr + row * stride_x + offs, mask).to(tl.float32) / HC + y = x * tl.sigmoid(x) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(y_ptr + row * stride_y + offs, y, mask) + + +def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor: + num_tokens, DIM = x.shape + assert x.stride(1) == 1 + + output = x.new_empty(x.shape) + _hc_silu_kernel[(num_tokens,)]( + x, + output, + x.stride(0), + output.stride(0), + DIM=DIM, + HC=hc_count, + launch_pdl=_pdl_supported(), + ) + return output + + +@triton.jit +def _hc_gate_mix_kernel( + x_ptr, + g_ptr, + y_ptr, + stride_x, + stride_g, + stride_y, + DIM: tl.constexpr, + HC: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + HC_DIM: tl.constexpr = DIM // HC + + row = tl.program_id(0).to(tl.int64) + tile_id = tl.program_id(1) + offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = offs_inner < HC_DIM + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + # The constexpr loop is unrolled and keeps one stream live at a time. + # Materializing [HC, BLOCK_SIZE] more than doubles latency at large M. + acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32) + for stream in tl.static_range(HC): + offsets = stream * HC_DIM + offs_inner + g = tl.load(g_ptr + row * stride_g + offsets, mask, other=0.0) + x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0) + acc += tl.sigmoid(g.to(tl.float32)) * x.to(tl.float32) + acc /= HC + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(y_ptr + row * stride_y + offs_inner, acc, mask) + + +def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor: + N, DIM = gate.shape + assert x.shape == gate.shape + assert DIM % hc_count == 0 + assert x.stride(1) == 1 + assert gate.stride(1) == 1 + + HC_DIM = DIM // hc_count + out = x.new_empty(N, HC_DIM) + BLOCK_SIZE = 512 + _hc_gate_mix_kernel[(N, triton.cdiv(HC_DIM, BLOCK_SIZE))]( + x, + gate, + out, + x.stride(0), + gate.stride(0), + out.stride(0), + DIM, + hc_count, + BLOCK_SIZE, + launch_pdl=_pdl_supported(), + ) + return out + + +@triton.jit +def _hc_combine_kernel( + block_ptr, + res_ptr, + inj_ptr, + out_ptr, + stride_block, + stride_res, + stride_inj, + stride_out, + HC_DIM: tl.constexpr, + HC: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + HC_PAD: tl.constexpr = triton.next_power_of_2(HC) + + row = tl.program_id(0).to(tl.int64) + tile_id = tl.program_id(1) + + offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask_inner = offs_inner < HC_DIM + offs_hc = tl.arange(0, HC_PAD) + mask_hc = offs_hc < HC + offs = offs_hc[:, None] * HC_DIM + offs_inner[None, :] + mask = mask_hc[:, None] & mask_inner[None, :] + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0) + block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0) + res = tl.load(res_ptr + row * stride_res + offs, mask, other=0.0) + + # Keeping HC as a broadcast dimension is faster here than four separate + # residual load/store sequences. + inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC) + out = res.to(tl.float32) + block.to(tl.float32)[None, :] * inj[:, None] + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + tl.store(out_ptr + row * stride_out + offs, out, mask=mask) + + +def hc_combine( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + hc_count: int, +) -> torch.Tensor: + N, DIM = residual.shape + assert DIM % hc_count == 0 + hc_dim = DIM // hc_count + assert block_output.shape == (N, hc_dim) + assert injection_logits.shape == (N, hc_count) + assert residual.stride(1) == 1 + assert block_output.stride(1) == 1 + assert injection_logits.stride(1) == 1 + + out = residual.new_empty(residual.shape) + BLOCK_SIZE = 512 + _hc_combine_kernel[(N, triton.cdiv(hc_dim, BLOCK_SIZE))]( + block_output, + residual, + injection_logits, + out, + block_output.stride(0), + residual.stride(0), + injection_logits.stride(0), + out.stride(0), + hc_dim, + hc_count, + BLOCK_SIZE, + launch_pdl=_pdl_supported(), + ) + return out + + +@triton.jit +def _hc_combine_norm_kernel( + block_ptr, + res_ptr, + inj_ptr, + w_ptr, + out_ptr, + y_ptr, + stride_block, + stride_res, + stride_inj, + stride_out, + stride_y, + HC_DIM: tl.constexpr, + HC: tl.constexpr, + W_SHARED: tl.constexpr, + EPS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + HC_PAD: tl.constexpr = triton.next_power_of_2(HC) + NUM_TILES: tl.constexpr = triton.cdiv(HC_DIM, BLOCK_SIZE) + NUM_TILES_PAD: tl.constexpr = triton.next_power_of_2(NUM_TILES) + + row = tl.program_id(0).to(tl.int64) + stream = tl.program_id(1) + offs_hc = tl.arange(0, HC_PAD) + mask_hc = offs_hc < HC + tile_ids = tl.arange(0, NUM_TILES_PAD) + offs_inner = tile_ids[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :] + mask_inner = offs_inner < HC_DIM + offs = stream * HC_DIM + offs_inner + # Shared norm weights repeat across streams; per-branch weights use the + # same flattened HC layout as the residual. + w_offs = offs_inner if W_SHARED else offs + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + # Start the uncached residual load first, then issue the other combine + # loads before consuming any of them. + res = tl.load(res_ptr + row * stride_res + offs, mask_inner, other=0.0) + inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0) + block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0) + inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC) + inj = tl.sum(tl.where(offs_hc == stream, inj, 0.0)) + # Round the materialized combine result before normalization. This matches + # the unfused combine -> RMSNorm boundary. + out = (res.to(tl.float32) + block.to(tl.float32) * inj).to(out_ptr.dtype.element_ty) + tl.store(out_ptr + row * stride_out + offs, out, mask=mask_inner) + + out = out.to(tl.float32) + # Keep the two-axis reduction: flattening the padded tile is ~40% slower + # at decode sizes. + sum_sq = tl.sum(tl.sum(out * out, axis=1), axis=0) + rrms = tl.rsqrt(sum_sq / HC_DIM + EPS) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + + # Loading the weight earlier helps decode but keeps the tile live across + # the reduction and regresses larger batches, so defer it to the norm. + w = tl.load(w_ptr + w_offs, mask_inner, other=0.0) + y = out * rrms + y += y * w.to(tl.float32) + tl.store(y_ptr + row * stride_y + offs, y, mask_inner) + + +def hc_combine_norm( + residual: torch.Tensor, + block_output: torch.Tensor, + injection_logits: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + hc_count: int, +) -> tuple[torch.Tensor, torch.Tensor]: + N, DIM = residual.shape + assert DIM % hc_count == 0 + hc_dim = DIM // hc_count + assert block_output.shape == (N, hc_dim) + assert injection_logits.shape == (N, hc_count) + assert residual.stride(1) == 1 + assert block_output.stride(1) == 1 + assert injection_logits.stride(1) == 1 + assert norm_weight.is_contiguous() + assert norm_weight.numel() in (hc_dim, DIM) + + out = residual.new_empty(residual.shape) + y = residual.new_empty(residual.shape) + BLOCK_SIZE = 512 + _hc_combine_norm_kernel[(N, hc_count)]( + block_output, + residual, + injection_logits, + norm_weight, + out, + y, + block_output.stride(0), + residual.stride(0), + injection_logits.stride(0), + out.stride(0), + y.stride(0), + hc_dim, + hc_count, + W_SHARED=norm_weight.numel() == hc_dim, + EPS=eps, + BLOCK_SIZE=BLOCK_SIZE, + launch_pdl=_pdl_supported(), + ) + return out, y + + +__all__ = [ + "grouped_gemma_rmsnorm", + "hc_combine", + "hc_combine_norm", + "hc_gate_mix", + "hc_silu", +] diff --git a/python/freetoken/kernel/triton/kpool_compress.py b/python/freetoken/kernel/triton/kpool_compress.py new file mode 100644 index 0000000000..61cacb1f91 --- /dev/null +++ b/python/freetoken/kernel/triton/kpool_compress.py @@ -0,0 +1,146 @@ +"""Pool one token row's group into ``slab[cmp_rows[row]]``: members inside this +forward read the raw K/gate rows, older members read the per-request tail ring. +Adapted from ``qsa/compress.py`` with the mean replaced by a per-channel +softmax(gate + APE) weighted sum (hence the gate stream/ring and the APE input). +Non-closing rows land on a scratch row that scoring never reads.""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _compress_kpool_groups_kernel( + raw_k_ptr, + raw_g_ptr, + ring_k_ptr, + ring_g_ptr, + ape_ptr, + ring_slots_ptr, + token_to_req_ptr, + query_start_loc_ptr, + positions_ptr, + slab_ptr, + cmp_rows_ptr, + stride_raw_k_row, + stride_raw_g_row, + stride_ring_row, + stride_slab_row, + num_rows, + num_ring_rows, + num_requests, + RATIO: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, +) -> None: + row = tl.program_id(0) + dims = tl.arange(0, BLOCK_D) + in_dim = dims < HEAD_DIM + + request = tl.load(token_to_req_ptr + row, mask=row < num_rows, other=-1) + end_position = tl.load(positions_ptr + row, mask=row < num_rows, other=0).to(tl.int64) + valid_request = (request >= 0) & (request < num_requests) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + query_row_start = tl.load( + query_start_loc_ptr + safe_request, mask=valid_request, other=0 + ).to(tl.int64) + chunk_start_position = end_position - (row - query_row_start) + ring_slot = tl.load(ring_slots_ptr + safe_request, mask=valid_request, other=0).to( + tl.int64 + ) + # A row whose group has members before position 0 (end_position < RATIO - 1) + # can never close; keep its loads masked off entirely -- member positions would + # be negative and C-style % would produce NEGATIVE ring rows (illegal address). + valid_row = (row < num_rows) & valid_request & (end_position >= RATIO - 1) + + # Two-pass per-channel softmax over the RATIO group members (static loop; the + # doubled loads are 4 x 512B and free next to the fp32 math). + m = tl.full((BLOCK_D,), float("-inf"), tl.float32) + for off in tl.static_range(RATIO): + position = end_position - (RATIO - 1 - off) + use_raw = position >= chunk_start_position + raw_row = query_row_start + position - chunk_start_position + ring_row = ring_slot * RATIO + position % RATIO + g_raw = tl.load( + raw_g_ptr + raw_row * stride_raw_g_row + dims, + mask=valid_row & use_raw & (raw_row >= 0) & (raw_row < num_rows) & in_dim, + other=0.0, + ).to(tl.float32) + g_ring = tl.load( + ring_g_ptr + tl.maximum(ring_row, 0) * stride_ring_row + dims, + mask=valid_row & (~use_raw) & (ring_row >= 0) & (ring_row < num_ring_rows) & in_dim, + other=0.0, + ).to(tl.float32) + ape = tl.load(ape_ptr + off * HEAD_DIM + dims, mask=in_dim, other=0.0) + m = tl.maximum(m, tl.where(use_raw, g_raw, g_ring) + ape) + + s = tl.zeros((BLOCK_D,), dtype=tl.float32) + acc = tl.zeros((BLOCK_D,), dtype=tl.float32) + for off in tl.static_range(RATIO): + position = end_position - (RATIO - 1 - off) + use_raw = position >= chunk_start_position + raw_row = query_row_start + position - chunk_start_position + ring_row = ring_slot * RATIO + position % RATIO + raw_mask = valid_row & use_raw & (raw_row >= 0) & (raw_row < num_rows) & in_dim + ring_mask = ( + valid_row & (~use_raw) & (ring_row >= 0) & (ring_row < num_ring_rows) & in_dim + ) + g_raw = tl.load( + raw_g_ptr + raw_row * stride_raw_g_row + dims, mask=raw_mask, other=0.0 + ).to(tl.float32) + g_ring = tl.load( + ring_g_ptr + tl.maximum(ring_row, 0) * stride_ring_row + dims, + mask=ring_mask, other=0.0, + ).to(tl.float32) + k_raw = tl.load( + raw_k_ptr + raw_row * stride_raw_k_row + dims, mask=raw_mask, other=0.0 + ).to(tl.float32) + k_ring = tl.load( + ring_k_ptr + tl.maximum(ring_row, 0) * stride_ring_row + dims, + mask=ring_mask, other=0.0, + ).to(tl.float32) + ape = tl.load(ape_ptr + off * HEAD_DIM + dims, mask=in_dim, other=0.0) + e = tl.exp(tl.where(use_raw, g_raw, g_ring) + ape - m) + s += e + acc += e * tl.where(use_raw, k_raw, k_ring) + + pooled = acc / s + dest = tl.load(cmp_rows_ptr + row, mask=row < num_rows, other=0).to(tl.int64) + tl.store( + slab_ptr + dest * stride_slab_row + dims, + pooled.to(slab_ptr.dtype.element_ty), + mask=(row < num_rows) & in_dim, + ) + + +def kpool_compress_store( + k: torch.Tensor, # [T, D] raw index keys (this forward) + gate: torch.Tensor, # [T, D] raw gate scores + ring_k: torch.Tensor, # [slots * ratio, D] flat tail ring (keys) + ring_g: torch.Tensor, # [slots * ratio, D] flat tail ring (gates) + ape: torch.Tensor, # [ratio, D] fp32 model parameter + ring_slots: torch.Tensor, # [n_req] Req.table_idx + token_to_req: torch.Tensor, # [T] + cu_seqlens: torch.Tensor, # [n_req + 1] + positions: torch.Tensor, # [T] + slab: torch.Tensor, # [shadow_rows + scratch, D] + cmp_rows: torch.Tensor, # [T] shadow row (closing) or scratch row + ratio: int, +) -> None: + t, d = k.shape + if t == 0: + return + assert ape.dtype == torch.float32 and ape.shape == (ratio, d) + _compress_kpool_groups_kernel[(t,)]( + k, gate, ring_k, ring_g, ape, + ring_slots, token_to_req, cu_seqlens, positions, + slab, cmp_rows, + k.stride(0), gate.stride(0), ring_k.stride(0), slab.stride(0), + t, ring_k.shape[0], ring_slots.numel(), + RATIO=ratio, HEAD_DIM=d, BLOCK_D=triton.next_power_of_2(d), + ) + + +__all__ = ["kpool_compress_store"] diff --git a/python/freetoken/kernel/triton/mhc.py b/python/freetoken/kernel/triton/mhc.py new file mode 100644 index 0000000000..0782efe37d --- /dev/null +++ b/python/freetoken/kernel/triton/mhc.py @@ -0,0 +1,232 @@ +"""Fused mHC (Manifold-Constrained Hyper-Connections) triton kernels. + +One program per token fuses the sublayer-boundary mix: apply the previous +sublayer's hc_post (comb^T @ res + post * x), then this sublayer's hc_pre -- +the fn GEMV over the flattened streams, flat-RMS normalization, and the three +gates (sigmoid pre, sigmoid*mult post, row-softmax + Sinkhorn comb, all on an +n x n held in registers) -- and the pre-mixed layer input. + +Semantics are defined by layers/mhc.py's torch reference (bit-comparable in +fp32 up to reduction order); tests/layers/test_mhc.py pins the parity. N +(hc_mult) is a constexpr; only N == 4 is exercised. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _mhc_stage1_kernel( + x_ptr, res_ptr, post_ptr, comb_ptr, fn_ptr, + res_out_ptr, + sq_part_ptr, # [T, NS] fp32 + mix_part_ptr, # [T, NS, BLK_MIX] fp32 + H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr, + SPLIT: tl.constexpr, # hidden elems per split (multiple of BLOCK_H) + BLOCK_H: tl.constexpr, + NS: tl.constexpr, + HAS_POST: tl.constexpr, +): + """Split-K stage of the fused mHC: each program owns one hidden slice of one + token -- applies hc_post there, stores the updated streams, and reduces its + partial sq-sum + fn-GEMV contribution over NS splits.""" + t = tl.program_id(0).to(tl.int64) + s = tl.program_id(1) + offs_mix = tl.arange(0, BLK_MIX) + mix_mask = offs_mix < MIX + offs_n = tl.arange(0, N) + + if HAS_POST: + b_post = tl.load(post_ptr + t * N + offs_n) + b_comb = tl.load(comb_ptr + t * N * N + offs_n[:, None] * N + offs_n[None, :]) + + sqsum = 0.0 + acc = tl.zeros([BLK_MIX], dtype=tl.float32) + for h0 in range(s * SPLIT, tl.minimum((s + 1) * SPLIT, H), BLOCK_H): + offs_h = h0 + tl.arange(0, BLOCK_H) + h_mask = offs_h < H + if HAS_POST: + b_x = tl.load(x_ptr + t * H + offs_h, mask=h_mask, other=0.0).to(tl.float32) + for n in tl.static_range(N): + if HAS_POST: + r_new = tl.zeros([BLOCK_H], dtype=tl.float32) + for i in tl.static_range(N): + r_i = tl.load(res_ptr + (t * N + i) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32) + c_in = tl.sum(tl.where((offs_n == i)[:, None] & (offs_n == n)[None, :], b_comb, 0.0)) + r_new += c_in * r_i + p_n = tl.sum(tl.where(offs_n == n, b_post, 0.0)) + r_new += p_n * b_x + # Round through the STORAGE dtype before the sq-sum and fn-GEMV: + # the torch reference reads back what it stored (dtype-generic -- + # fp16/fp32 residuals must not be silently bf16-rounded). + r_new = r_new.to(res_out_ptr.dtype.element_ty).to(tl.float32) + tl.store(res_out_ptr + (t * N + n) * H + offs_h, r_new.to(res_out_ptr.dtype.element_ty), mask=h_mask) + else: + r_new = tl.load(res_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32) + tl.store(res_out_ptr + (t * N + n) * H + offs_h, r_new.to(res_out_ptr.dtype.element_ty), mask=h_mask) + sqsum += tl.sum(r_new * r_new) + fn_tile = tl.load( + fn_ptr + offs_mix[:, None] * (N * H) + (n * H + offs_h)[None, :], + mask=mix_mask[:, None] & h_mask[None, :], other=0.0, + ) + acc += tl.sum(fn_tile * r_new[None, :], axis=1) + + tl.store(sq_part_ptr + t * NS + s, sqsum) + tl.store(mix_part_ptr + (t * NS + s) * BLK_MIX + offs_mix, acc) + + +@triton.jit +def _mhc_stage2_kernel( + sq_part_ptr, mix_part_ptr, scale_ptr, base_ptr, + post_out_ptr, comb_out_ptr, pre_out_ptr, + rms_eps, hc_eps, post_mult, + SINKHORN: tl.constexpr, + H: tl.constexpr, N: tl.constexpr, MIX: tl.constexpr, BLK_MIX: tl.constexpr, + NS: tl.constexpr, +): + """Reduce the split partials and run the tiny gate math (sigmoid gates, + row-softmax + in-register 4x4 Sinkhorn); emits pre gates for stage 3.""" + t = tl.program_id(0).to(tl.int64) + offs_mix = tl.arange(0, BLK_MIX) + mix_mask = offs_mix < MIX + offs_s = tl.arange(0, NS) + + sqsum = tl.sum(tl.load(sq_part_ptr + t * NS + offs_s)) + acc = tl.sum( + tl.load(mix_part_ptr + (t * NS + offs_s)[:, None] * BLK_MIX + offs_mix[None, :]), + axis=0, + ) + inv_rms = tl.math.rsqrt(sqsum / (N * H) + rms_eps) + mixes = acc * inv_rms + s0 = tl.load(scale_ptr + 0) + s1 = tl.load(scale_ptr + 1) + s2 = tl.load(scale_ptr + 2) + b_base = tl.load(base_ptr + offs_mix, mask=mix_mask, other=0.0) + + is_pre = offs_mix < N + is_post = (offs_mix >= N) & (offs_mix < 2 * N) + gate_scale = tl.where(is_pre, s0, tl.where(is_post, s1, s2)) + logits = mixes * gate_scale + b_base + pre = tl.sigmoid(logits) + hc_eps + post_new = tl.sigmoid(logits) * post_mult + + offs_n2 = tl.arange(0, N) + comb_logits = tl.zeros([N, N], dtype=tl.float32) + for r in tl.static_range(N): + for c in tl.static_range(N): + lane = 2 * N + r * N + c + v = tl.sum(tl.where(offs_mix == lane, logits, 0.0)) + comb_logits += tl.where( + (offs_n2 == r)[:, None] & (offs_n2 == c)[None, :], v, 0.0 + ) + row_max = tl.max(comb_logits, axis=1) + e = tl.exp(comb_logits - row_max[:, None]) + comb = e / tl.sum(e, axis=1)[:, None] + hc_eps + comb = comb / (tl.sum(comb, axis=0)[None, :] + hc_eps) + for _ in range(SINKHORN - 1): + comb = comb / (tl.sum(comb, axis=1)[:, None] + hc_eps) + comb = comb / (tl.sum(comb, axis=0)[None, :] + hc_eps) + + post_g = tl.sum( + tl.where((offs_mix[None, :] - N) == offs_n2[:, None], post_new[None, :], 0.0), + axis=1, + ) + pre_g = tl.sum( + tl.where(offs_mix[None, :] == offs_n2[:, None], pre[None, :], 0.0), axis=1 + ) + tl.store(post_out_ptr + t * N + offs_n2, post_g) + tl.store(pre_out_ptr + t * N + offs_n2, pre_g) + tl.store(comb_out_ptr + t * N * N + offs_n2[:, None] * N + offs_n2[None, :], comb) + + +@triton.jit +def _mhc_stage3_kernel( + res_out_ptr, pre_ptr, li_out_ptr, + H: tl.constexpr, N: tl.constexpr, BLOCK_H: tl.constexpr, +): + """layer_input = sum_n pre_n * res_new_n, parallel over hidden chunks.""" + t = tl.program_id(0).to(tl.int64) + hb = tl.program_id(1) + offs_h = hb * BLOCK_H + tl.arange(0, BLOCK_H) + h_mask = offs_h < H + offs_n = tl.arange(0, N) + pre = tl.load(pre_ptr + t * N + offs_n) + li = tl.zeros([BLOCK_H], dtype=tl.float32) + for n in tl.static_range(N): + r = tl.load(res_out_ptr + (t * N + n) * H + offs_h, mask=h_mask, other=0.0).to(tl.float32) + li += tl.sum(tl.where(offs_n == n, pre, 0.0)) * r + tl.store(li_out_ptr + t * H + offs_h, li.to(li_out_ptr.dtype.element_ty), mask=h_mask) + + +def mhc_fused_post_pre_triton( + x: torch.Tensor, + residual: torch.Tensor, + post_mix: torch.Tensor | None, + comb_mix: torch.Tensor | None, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_eps: float, + post_mult: float, + sinkhorn_repeat: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Fused hc_post (skipped when ``post_mix is None``) + hc_pre. Three-stage + split-K: the GEMV/sq-sum reduction fans out over NS hidden slices. Returns + (residual_new [T,N,H] bf16, post [T,N,1] fp32, comb [T,N,N] fp32, + layer_input [T,H] bf16).""" + t, n, h = residual.shape + mix = 2 * n + n * n + assert fn.shape == (mix, n * h) and fn.dtype == torch.float32 + residual = residual.contiguous() + has_post = post_mix is not None + dev = residual.device + + res_out = torch.empty_like(residual) + post_out = torch.empty(t, n, dtype=torch.float32, device=dev) + comb_out = torch.empty(t, n, n, dtype=torch.float32, device=dev) + li_out = torch.empty(t, h, dtype=residual.dtype, device=dev) + + block_h = min(512, triton.next_power_of_2(h)) + # NS feeds a tl.arange in stage 2 -> keep it a power of two. + ns = 1 + while ns * 2 <= min(16, h // block_h): + ns *= 2 + split = triton.cdiv(triton.cdiv(h, ns), block_h) * block_h + ns = triton.cdiv(h, split) + blk_mix = triton.next_power_of_2(mix) + sq_part = torch.empty(t, ns, dtype=torch.float32, device=dev) + mix_part = torch.empty(t, ns, blk_mix, dtype=torch.float32, device=dev) + pre_out = torch.empty(t, n, dtype=torch.float32, device=dev) + + _mhc_stage1_kernel[(t, ns)]( + x.contiguous() if has_post else residual, # dummy ptr when unused + residual, + post_mix.contiguous().view(t, n) if has_post else post_out, + comb_mix.contiguous() if has_post else comb_out, + fn, res_out, sq_part, mix_part, + H=h, N=n, MIX=mix, BLK_MIX=blk_mix, + SPLIT=split, BLOCK_H=block_h, NS=ns, + HAS_POST=has_post, + num_warps=4, num_stages=2, + ) + _mhc_stage2_kernel[(t,)]( + sq_part, mix_part, hc_scale, hc_base, + post_out, comb_out, pre_out, + rms_eps, hc_eps, post_mult, + SINKHORN=sinkhorn_repeat, + H=h, N=n, MIX=mix, BLK_MIX=blk_mix, NS=ns, + num_warps=1, + ) + _mhc_stage3_kernel[(t, triton.cdiv(h, 1024))]( + res_out, pre_out, li_out, + H=h, N=n, BLOCK_H=min(1024, triton.next_power_of_2(h)), + num_warps=4, + ) + return res_out, post_out.view(t, n, 1), comb_out, li_out + + +__all__ = ["mhc_fused_post_pre_triton"] diff --git a/python/freetoken/kernel/triton/moe_router.py b/python/freetoken/kernel/triton/moe_router.py new file mode 100644 index 0000000000..54ecfe326a --- /dev/null +++ b/python/freetoken/kernel/triton/moe_router.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the SGLang project +# Adapted from SGLang (kernels/ops/moe/moe_fused_gate.py) +"""Fused softmax top-k MoE router (bias-free, ungrouped experts). + +The ``num_token_non_padded`` row mask reads the device tensor, so it survives CUDA-graph capture. +""" + +from __future__ import annotations + +import functools +from typing import Tuple + +import torch +import triton +import triton.language as tl + +from freetoken.utils.arch import is_sm90_supported + + +@functools.cache +def _pdl_supported() -> bool: + return is_sm90_supported() + + +@triton.jit +def _router_triton_kernel( + scores_ptr, + out_weights_ptr, + out_indices_ptr, + num_token_non_padded_ptr, + M, + stride_sm, + stride_sn, + stride_wm, + stride_wk, + stride_im, + stride_ik, + N: tl.constexpr, + K: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + RENORMALIZE: tl.constexpr, + HAS_TOKEN_LIMIT: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + # Row-tiled: each program handles BLOCK_M rows; all reductions run along the + # expert (N) axis. Tiling rows keeps CTAs large enough to stay occupancy-bound + # rather than launch-bound at small N (many tiny 1-warp CTAs otherwise). + pid = tl.program_id(0) + offs_m = pid * BLOCK_M + tl.arange(0, BLOCK_M) + offs_n = tl.arange(0, BLOCK_N) + mask_m = offs_m < M + mask_n = offs_n < N + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + # offs_m * stride can overflow int32 for large token counts. + row_ptr = scores_ptr + offs_m[:, None].to(tl.int64) * stride_sm + offs_n[None, :] * stride_sn + mask2d = mask_m[:, None] & mask_n[None, :] + logits = tl.load(row_ptr, mask=mask2d, other=0.0).to(tl.float32) + + ranked = tl.where(mask_n[None, :], logits, -float("inf")) + row_max = tl.max(ranked, axis=1)[:, None] + exp_row = tl.where(mask_n[None, :], tl.exp(ranked - row_max), 0.0) + activated = exp_row / tl.sum(exp_row, axis=1)[:, None] + + # Map NaN -> a finite floor + ranked = tl.where(ranked == ranked, ranked, -1e30) + + offs_k = tl.arange(0, BLOCK_K) + mask_k = offs_k < K + selected_vals = tl.zeros([BLOCK_M, BLOCK_K], dtype=tl.float32) + selected_idx = tl.zeros([BLOCK_M, BLOCK_K], dtype=tl.int32) + + cur = ranked + for k in tl.static_range(K): + max_val = tl.max(cur, axis=1)[:, None] + lane_id = tl.where(cur == max_val, offs_n[None, :], N + 1) # lowest expert id wins ties + win_lane = tl.min(lane_id, axis=1)[:, None].to(tl.int32) + win_activated = tl.sum( + tl.where(offs_n[None, :] == win_lane, activated, 0.0), axis=1 + )[:, None] + slot = offs_k[None, :] == k + selected_vals = tl.where(slot, win_activated, selected_vals) + selected_idx = tl.where(slot, win_lane, selected_idx) + cur = tl.where(offs_n[None, :] == win_lane, -float("inf"), cur) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + + if RENORMALIZE: + routed_sum = tl.sum(tl.where(mask_k[None, :], selected_vals, 0.0), axis=1)[:, None] + selected_vals = selected_vals / tl.where(routed_sum > 0.0, routed_sum, 1.0) + + if HAS_TOKEN_LIMIT: + limit = tl.load(num_token_non_padded_ptr) + selected_idx = tl.where(offs_m[:, None] < limit, selected_idx, -1) + + out_w_ptr = out_weights_ptr + offs_m[:, None].to(tl.int64) * stride_wm + offs_k[None, :] * stride_wk + out_i_ptr = out_indices_ptr + offs_m[:, None].to(tl.int64) * stride_im + offs_k[None, :] * stride_ik + store_mask = mask_m[:, None] & mask_k[None, :] + tl.store(out_w_ptr, selected_vals, mask=store_mask) + tl.store(out_i_ptr, selected_idx, mask=store_mask) + + +def fused_topk_softmax( + gating_output: torch.Tensor, + topk: int, + renormalize: bool, + num_token_non_padded: torch.Tensor | None = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Softmax over all experts, top-k, then renormalize; ties keep the lowest expert id. + + ``num_token_non_padded`` is a device scalar; rows at or past it get expert id -1. + """ + assert gating_output.ndim == 2, "gating_output must be 2D" + M, N = gating_output.shape + weights = torch.empty((M, topk), dtype=torch.float32, device=gating_output.device) + indices = torch.empty((M, topk), dtype=torch.int32, device=gating_output.device) + + BLOCK_N = triton.next_power_of_2(N) + BLOCK_K = triton.next_power_of_2(topk) + # Single warp per program keeps the per-row top-k reductions on cheap warp + # shuffles; pack a few rows per program only when N is small so tiny launches + # stay occupancy-bound. Swept on H100/B200; larger tiles / more warps regress + # (register pressure). + BLOCK_M = max(1, min(4, 256 // BLOCK_N)) + # For wide rows the K sequential argmax passes dominate and benefit from more + # warps despite the cross-warp reduction cost. + num_warps = 1 if BLOCK_N <= 512 else 4 + + _router_triton_kernel[(triton.cdiv(M, BLOCK_M),)]( + gating_output, + weights, + indices, + num_token_non_padded, + M, + gating_output.stride(0), + gating_output.stride(1), + weights.stride(0), + weights.stride(1), + indices.stride(0), + indices.stride(1), + N=N, + K=topk, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + BLOCK_K=BLOCK_K, + RENORMALIZE=renormalize, + HAS_TOKEN_LIMIT=num_token_non_padded is not None, + launch_pdl=_pdl_supported(), + num_warps=num_warps, + ) + return weights, indices + + +__all__ = ["fused_topk_softmax"] diff --git a/python/freetoken/kernel/triton/moe_shared_gate.py b/python/freetoken/kernel/triton/moe_shared_gate.py new file mode 100644 index 0000000000..0615c2fa69 --- /dev/null +++ b/python/freetoken/kernel/triton/moe_shared_gate.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the SGLang project +# Adapted from SGLang (kernels/ops/elementwise.py, ``_fused_gate_sigmoid_mul_add``) +"""Gated shared-expert epilogue: the gate reduction and the sigmoid-mul-add. + +The routed experts may write into ``hidden_states`` in place, so the gate reduction runs +before them and the mul-add after. +""" + +from __future__ import annotations + +import functools + +import torch +import triton +import triton.language as tl + +from freetoken.utils.arch import is_sm90_supported + + +@functools.cache +def _pdl_supported() -> bool: + return is_sm90_supported() + + +def _reduction_warps(hidden_dim: int, num_tokens: int) -> int: + warps = max(min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), 32), 4) + return min(warps, 8) if num_tokens >= 1024 else warps + + +@triton.jit +def _gate_sigmoid_kernel( + hidden_ptr, + weight_ptr, + gate_ptr, + stride_h, + HIDDEN: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + # row * stride can overflow int32 for large token counts. + row = tl.program_id(0).to(tl.int64) + offs = tl.arange(0, BLOCK_SIZE) + mask = offs < HIDDEN + + w = tl.load(weight_ptr + offs, mask=mask, other=0.0).to(tl.float32) + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + h = tl.load(hidden_ptr + row * stride_h + offs, mask=mask, other=0.0).to(tl.float32) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + + tl.store(gate_ptr + row, tl.sigmoid(tl.sum(h * w, axis=0))) + + +def shared_gate_sigmoid(hidden_states: torch.Tensor, gate_weight: torch.Tensor) -> torch.Tensor: + """Per-token ``sigmoid(hidden_states @ gate_weight)`` as fp32 [num_tokens].""" + num_tokens, hidden_dim = hidden_states.shape + assert hidden_states.stride(1) == 1, "shared gate requires unit inner stride" + assert gate_weight.shape == (hidden_dim,) and gate_weight.is_contiguous() + + gate = torch.empty(num_tokens, dtype=torch.float32, device=hidden_states.device) + _gate_sigmoid_kernel[(num_tokens,)]( + hidden_states, + gate_weight, + gate, + hidden_states.stride(0), + HIDDEN=hidden_dim, + BLOCK_SIZE=triton.next_power_of_2(hidden_dim), + launch_pdl=_pdl_supported(), + num_warps=_reduction_warps(hidden_dim, num_tokens), + ) + return gate + + +@triton.jit +def _gate_mul_add_kernel( + routed_ptr, + shared_ptr, + gate_ptr, + out_ptr, + stride_r, + stride_s, + stride_o, + HIDDEN: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + launch_pdl: tl.constexpr, +) -> None: + row = tl.program_id(0).to(tl.int64) + block = tl.program_id(1) + offs = block * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + mask = offs < HIDDEN + + if launch_pdl: + tl.extra.cuda.gdc_wait() + + gate = tl.load(gate_ptr + row) + routed = tl.load(routed_ptr + row * stride_r + offs, mask=mask, other=0.0).to(tl.float32) + shared = tl.load(shared_ptr + row * stride_s + offs, mask=mask, other=0.0).to(tl.float32) + + if launch_pdl: + tl.extra.cuda.gdc_launch_dependents() + + tl.store(out_ptr + row * stride_o + offs, routed + gate * shared, mask=mask) + + +def shared_gate_mul_add( + routed: torch.Tensor, shared: torch.Tensor, gate: torch.Tensor +) -> torch.Tensor: + """``routed + gate[:, None] * shared`` into a fresh tensor.""" + num_tokens, hidden_dim = routed.shape + assert shared.shape == routed.shape + assert routed.stride(1) == 1 and shared.stride(1) == 1 + assert gate.shape == (num_tokens,) + + out = torch.empty_like(routed) + block_size = min(triton.next_power_of_2(hidden_dim), 2048) + _gate_mul_add_kernel[(num_tokens, triton.cdiv(hidden_dim, block_size))]( + routed, + shared, + gate, + out, + routed.stride(0), + shared.stride(0), + out.stride(0), + HIDDEN=hidden_dim, + BLOCK_SIZE=block_size, + launch_pdl=_pdl_supported(), + num_warps=4, + ) + return out + + +__all__ = ["shared_gate_mul_add", "shared_gate_sigmoid"] diff --git a/python/freetoken/kernel/triton/norm.py b/python/freetoken/kernel/triton/norm.py index 3f95c29f0d..62bd63824d 100644 --- a/python/freetoken/kernel/triton/norm.py +++ b/python/freetoken/kernel/triton/norm.py @@ -142,9 +142,13 @@ def _rmsnorm(input, weight, eps, out, gemma: bool): # PDL only on the contiguous (decode-replay) path: on the strided qk-norm's # 32k-CTA prefill grids the per-CTA gdc_wait poll costs more than it hides. pdl = contig and is_sm90_supported() + # launch_pdl is a CUDA-Hopper-only Triton launch kwarg; the AMD backend's + # arg-packer rejects it outright (KeyError) even when passed as False, so it + # is only included on the one backend/arch combination that ever sets pdl=True. + pdl_kwargs = {"launch_pdl": pdl} if pdl else {} _rmsnorm_kernel[(A, B)]( out, input, weight, eps, H, sxa, sxb, soa, sob, - CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl, GEMMA=gemma, + CONTIG=contig, ENABLE_PDL=pdl, GEMMA=gemma, **pdl_kwargs, num_warps=_num_warps(A * B), num_stages=1, ) return out @@ -170,9 +174,10 @@ def _fused_add_rmsnorm(input, residual, weight, eps, gemma: bool): _, _, sra, srb = _leading(residual) contig = input.ndim == 2 and input.is_contiguous() and residual.is_contiguous() pdl = contig and is_sm90_supported() + pdl_kwargs = {"launch_pdl": pdl} if pdl else {} _fused_add_rmsnorm_kernel[(A, B)]( input, residual, weight, eps, H, sxa, sxb, sra, srb, - CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl, GEMMA=gemma, + CONTIG=contig, ENABLE_PDL=pdl, GEMMA=gemma, **pdl_kwargs, num_warps=_num_warps(A * B), num_stages=1, ) diff --git a/python/freetoken/kernel/triton/ple.py b/python/freetoken/kernel/triton/ple.py new file mode 100644 index 0000000000..2314e94e72 --- /dev/null +++ b/python/freetoken/kernel/triton/ple.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the SGLang project +# Adapted from SGLang (python/sglang/srt/models/qwen4_exp.py) +"""UVA row gather for the Qwen3.8-Flash-Next PLE n-gram table. + +The table (320,001,536 rows x 160, FP8-e4m3 + one scalar scale = 47.7 GiB) stays in pinned +host memory and the GPU dereferences it in place over PCIe -- at its host VA on Linux/UVA, at +the mapped device address on WDDM (``kernel/pinned.device_ptr``). One program per requested +row: read the row, widen to fp32, apply the per-tensor scale, store bf16. + +Ids outside the table store zeros. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +from freetoken.kernel.triton.e4m3_compat import e4m3_native_cx, e4m3_u8_to_f32 + +# Latency-bound over PCIe, so keep the block small and let many of them be in flight. +_NUM_WARPS = 1 + + +@triton.jit +def _ple_gather_kernel( + table_ptr, + ids_ptr, + out_ptr, + scale, + num_rows, + EMB_DIM: tl.constexpr, + IS_FP8: tl.constexpr, + BLOCK_D: tl.constexpr, +): + row = tl.program_id(0) + idx = tl.load(ids_ptr + row).to(tl.int64) + in_range = (idx >= 0) & (idx < num_rows) + idx = tl.where(in_range, idx, 0) + offsets = tl.arange(0, BLOCK_D) + mask = offsets < EMB_DIM + # the table is a host allocation: rebuild the typed pointer from the raw address + if IS_FP8: + if e4m3_native_cx(): + base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.float8e4nv)) + values = tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0.0).to(tl.float32) + else: + # pre-sm_89 has no fp8e4nv type: load raw bytes and decode in software + base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.uint8)) + values = e4m3_u8_to_f32(tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0)) + else: + base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.bfloat16)) + values = tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0.0).to(tl.float32) + values = tl.where(in_range, values * scale, 0.0) + tl.store( + out_ptr + row * EMB_DIM + offsets, + values.to(out_ptr.dtype.element_ty), + mask=mask, + ) + + +def ple_gather_rows( + table_ptr: int, + num_rows: int, + embed_dim: int, + row_ids: torch.Tensor, + out: torch.Tensor, + scale: float = 1.0, + is_fp8: bool = True, +) -> torch.Tensor: + """Gather ``row_ids`` from the host-resident table at ``table_ptr`` into ``out``. + + ``row_ids`` is a flat device int tensor; ``out`` is ``[row_ids.numel(), embed_dim]`` + bf16 on the same device. ``table_ptr`` is the address the GPU must dereference + (``kernel/pinned.device_ptr``), not necessarily the host ``data_ptr``. + """ + n = row_ids.numel() + assert out.shape == (n, embed_dim) and out.is_contiguous(), out.shape + if n: + _ple_gather_kernel[(n,)]( + table_ptr, + row_ids, + out, + float(scale), + num_rows, + EMB_DIM=embed_dim, + IS_FP8=is_fp8, + BLOCK_D=triton.next_power_of_2(embed_dim), + num_warps=_NUM_WARPS, + ) + return out + + +__all__ = ["ple_gather_rows"] diff --git a/python/freetoken/kernel/triton/qsa/__init__.py b/python/freetoken/kernel/triton/qsa/__init__.py new file mode 100644 index 0000000000..91753b7922 --- /dev/null +++ b/python/freetoken/kernel/triton/qsa/__init__.py @@ -0,0 +1,18 @@ +"""Triton kernels for Qwen3.8-Flash-Next QSA sparse attention.""" + +from .attend import qsa_sparse_paged_attention +from .compress import qsa_compress_groups, qsa_index_norm_rope, qsa_store_rows +from .expand import expand_qsa_block_indices +from .score import qsa_mqa_paged +from .topk import qsa_block_topk, qsa_block_topk_scratch_width + +__all__ = [ + "expand_qsa_block_indices", + "qsa_block_topk", + "qsa_block_topk_scratch_width", + "qsa_compress_groups", + "qsa_index_norm_rope", + "qsa_mqa_paged", + "qsa_sparse_paged_attention", + "qsa_store_rows", +] diff --git a/python/freetoken/kernel/triton/qsa/attend.py b/python/freetoken/kernel/triton/qsa/attend.py new file mode 100644 index 0000000000..541e27c680 --- /dev/null +++ b/python/freetoken/kernel/triton/qsa/attend.py @@ -0,0 +1,360 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/qsa.py) +"""Sparse paged GQA over the QSA selection.""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _qsa_sparse_paged_gqa_splitk_kernel( + q_ptr, + k_cache_ptr, + v_cache_ptr, + indices_ptr, + block_table_ptr, + token_to_req_ptr, + partial_output_ptr, + partial_lse_ptr, + output_ptr, + stride_q_row, + stride_q_head, + stride_k_block, + stride_k_token, + stride_k_head, + stride_v_block, + stride_v_token, + stride_v_head, + stride_indices_row, + stride_table_req, + stride_output_row, + stride_output_head, + num_rows, + num_cache_blocks, + num_requests, + TOPK: tl.constexpr, + PAGE_SIZE: tl.constexpr, + PAGE_TABLE_WIDTH: tl.constexpr, + GROUP_SIZE: tl.constexpr, + HEAD_DIM: tl.constexpr, + NUM_QUERY_HEADS: tl.constexpr, + NUM_SPLITS: tl.constexpr, + NUM_TILES: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +) -> None: + # row * stride can overflow int32 for large row counts. + row = tl.program_id(0).to(tl.int64) + kv_head = tl.program_id(1) + split_id = tl.program_id(2) + request = tl.load(token_to_req_ptr + row) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + + head_offsets = tl.arange(0, BLOCK_M) + dim_offsets = tl.arange(0, HEAD_DIM) + column_offsets = tl.arange(0, BLOCK_N) + first_head = kv_head * GROUP_SIZE + query = tl.load( + q_ptr + + row * stride_q_row + + (first_head + head_offsets[:, None]) * stride_q_head + + dim_offsets[None, :], + mask=head_offsets[:, None] < GROUP_SIZE, + other=0.0, + ) + + max_value = tl.full((BLOCK_M,), -1.0e20, dtype=tl.float32) + normalizer = tl.zeros((BLOCK_M,), dtype=tl.float32) + accumulator = tl.zeros((BLOCK_M, HEAD_DIM), dtype=tl.float32) + softmax_scale_log2: tl.constexpr = (HEAD_DIM**-0.5) * 1.4426950408889634 + + # Dynamic bounds avoid padded main-loop iterations for uneven splits. + split_tile_start = split_id * NUM_TILES // NUM_SPLITS + split_tile_end = (split_id + 1) * NUM_TILES // NUM_SPLITS + for tile in range(split_tile_start, split_tile_end): + columns = tile * BLOCK_N + column_offsets + logical_token = tl.load( + indices_ptr + row * stride_indices_row + columns, + mask=columns < TOPK, + other=-1, + ) + safe_token = tl.maximum(logical_token, 0) + logical_page = safe_token // PAGE_SIZE + page_offset = safe_token % PAGE_SIZE + valid = ( + (request >= 0) + & (request < num_requests) + & (logical_token >= 0) + & (logical_page < PAGE_TABLE_WIDTH) + ) + physical_page = tl.load( + block_table_ptr + + safe_request.to(tl.int64) * stride_table_req + + tl.minimum(logical_page, PAGE_TABLE_WIDTH - 1), + mask=valid, + other=-1, + ) + valid &= (physical_page >= 0) & (physical_page < num_cache_blocks) + # physical_page * block stride can overflow int32 for large caches. + safe_page = tl.maximum(physical_page, 0).to(tl.int64) + keys = tl.load( + k_cache_ptr + + safe_page[None, :] * stride_k_block + + page_offset[None, :] * stride_k_token + + kv_head * stride_k_head + + dim_offsets[:, None], + mask=valid[None, :], + other=0.0, + ) + values = tl.load( + v_cache_ptr + + safe_page[:, None] * stride_v_block + + page_offset[:, None] * stride_v_token + + kv_head * stride_v_head + + dim_offsets[None, :], + mask=valid[:, None], + other=0.0, + ) + scores = tl.dot(query, keys) + # Scaling scores avoids re-quantizing a scaled query to BF16. + scores *= softmax_scale_log2 + scores = tl.where(valid[None, :], scores, -1.0e20) + next_max = tl.maximum(max_value, tl.max(scores, axis=1)) + alpha = tl.math.exp2(max_value - next_max) + probabilities = tl.where( + valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0 + ) + accumulator = tl.dot( + probabilities.to(values.dtype), + values, + acc=accumulator * alpha[:, None], + ) + normalizer = normalizer * alpha + tl.sum(probabilities, axis=1) + max_value = next_max + + has_values = normalizer > 0 + normalized_output = tl.where( + has_values[:, None], + accumulator / tl.maximum(normalizer[:, None], 1.0e-20), + 0.0, + ) + output_mask = head_offsets[:, None] < GROUP_SIZE + if NUM_SPLITS == 1: + tl.store( + output_ptr + + row * stride_output_row + + (first_head + head_offsets[:, None]) * stride_output_head + + dim_offsets[None, :], + normalized_output, + mask=output_mask, + ) + else: + partial_lse = tl.where( + has_values, + max_value + tl.math.log2(tl.maximum(normalizer, 1.0e-20)), + -float("inf"), + ) + tl.store( + partial_output_ptr + + ( + (split_id.to(tl.int64) * num_rows + row) * NUM_QUERY_HEADS + + first_head + + head_offsets[:, None] + ) + * HEAD_DIM + + dim_offsets[None, :], + normalized_output, + mask=output_mask, + ) + tl.store( + partial_lse_ptr + + (split_id.to(tl.int64) * num_rows + row) * NUM_QUERY_HEADS + + first_head + + head_offsets, + partial_lse, + mask=head_offsets < GROUP_SIZE, + ) + + +@triton.jit +def _qsa_merge_splitk_kernel( + partial_output_ptr, + partial_lse_ptr, + output_ptr, + stride_output_row, + stride_output_head, + num_rows, + HEAD_DIM: tl.constexpr, + NUM_QUERY_HEADS: tl.constexpr, + NUM_SPLITS: tl.constexpr, + BLOCK_SPLITS: tl.constexpr, +) -> None: + row = tl.program_id(0).to(tl.int64) + head = tl.program_id(1) + split_offsets = tl.arange(0, BLOCK_SPLITS) + dim_offsets = tl.arange(0, HEAD_DIM) + split_mask = split_offsets < NUM_SPLITS + lse = tl.load( + partial_lse_ptr + (split_offsets.to(tl.int64) * num_rows + row) * NUM_QUERY_HEADS + head, + mask=split_mask, + other=-float("inf"), + ) + lse_max = tl.max(lse, axis=0) + has_values = lse_max > -float("inf") + shifted = tl.where(split_mask & has_values, lse - lse_max, -float("inf")) + weights = tl.math.exp2(shifted) + denominator = tl.sum(weights, axis=0) + partial_output = tl.load( + partial_output_ptr + + ((split_offsets[:, None].to(tl.int64) * num_rows + row) * NUM_QUERY_HEADS + head) + * HEAD_DIM + + dim_offsets[None, :], + mask=split_mask[:, None], + other=0.0, + ) + merged = tl.sum(partial_output * weights[:, None], axis=0) + merged = tl.where(denominator > 0, merged / denominator, 0.0) + tl.store( + output_ptr + row * stride_output_row + head * stride_output_head + dim_offsets, + merged, + ) + + +def qsa_sparse_paged_attention( + q: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + logical_indices: torch.Tensor, + block_table: torch.Tensor, + token_to_req: torch.Tensor, + out: torch.Tensor | None = None, +) -> torch.Tensor: + """Run sparse GQA directly over paged BF16 K/V caches.""" + + if q.ndim != 3 or k_cache.ndim != 4 or v_cache.shape != k_cache.shape: + raise ValueError("QSA sparse attention received invalid Q/K/V shapes") + if logical_indices.ndim != 2 or logical_indices.shape[0] != q.shape[0]: + raise ValueError("QSA indices must have one row per query") + if token_to_req.shape != (q.shape[0],) or block_table.ndim != 2: + raise ValueError("QSA sparse attention metadata has invalid shapes") + if logical_indices.shape[1] <= 0: + raise ValueError("QSA sparse attention requires a positive selection width") + if q.shape[2] != k_cache.shape[3] or q.shape[1] % k_cache.shape[2]: + raise ValueError("QSA sparse attention requires valid grouped-query heads") + head_dim = q.shape[2] + assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0 + assert q.dtype == k_cache.dtype == v_cache.dtype + assert logical_indices.dtype == block_table.dtype == torch.int32 + assert token_to_req.dtype == torch.int32 + assert q.stride(2) == k_cache.stride(3) == v_cache.stride(3) == 1 + assert logical_indices.stride(1) == block_table.stride(1) == 1 + assert token_to_req.stride(0) == 1 + if out is None: + out = torch.empty_like(q) + assert out.shape == q.shape and out.dtype == q.dtype and out.stride(2) == 1 + if not q.shape[0]: + return out + + group_size = q.shape[1] // k_cache.shape[2] + block_m = triton.next_power_of_2(group_size) + base_programs = q.shape[0] * k_cache.shape[2] + small_profile_limit = 8 if block_m <= 8 else 4 + + # Tuned on GB300 for the Qwen-Air TP1, TP2, and TP4 attention shapes. + # Narrow tiles favor decode; wide tiles improve throughput for prefill. + if base_programs <= small_profile_limit: + block_n, target_splits, partial_warps = 16, 64, 4 + elif base_programs < 32: + block_n, target_splits, partial_warps = 16, 32, 4 + elif base_programs <= 256: + block_n, target_splits, partial_warps = 64, 8, 2 + elif base_programs <= 512: + block_n, target_splits, partial_warps = 64, 4, 2 + else: + block_n, target_splits, partial_warps = 64, 1, 2 + + num_tiles = triton.cdiv(logical_indices.shape[1], block_n) + # Avoid empty splits when the selection width is smaller than the profile. + max_useful_splits = 1 << (num_tiles.bit_length() - 1) + num_splits = min(max_useful_splits, target_splits) + + # Split=1 writes output directly and compiles out all workspace accesses. + if num_splits == 1: + partial_output = out + partial_lse = out + else: + # FP32 partials preserve accuracy when merging independently normalized + # splits. + partial_output = torch.empty( + (num_splits, *q.shape), dtype=torch.float32, device=q.device + ) + partial_lse = torch.empty( + (num_splits, q.shape[0], q.shape[1]), + dtype=torch.float32, + device=q.device, + ) + + partial_grid = (q.shape[0], k_cache.shape[2], num_splits) + _qsa_sparse_paged_gqa_splitk_kernel[partial_grid]( + q, + k_cache, + v_cache, + logical_indices, + block_table, + token_to_req, + partial_output, + partial_lse, + out, + q.stride(0), + q.stride(1), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(2), + v_cache.stride(0), + v_cache.stride(1), + v_cache.stride(2), + logical_indices.stride(0), + block_table.stride(0), + out.stride(0), + out.stride(1), + q.shape[0], + k_cache.shape[0], + block_table.shape[0], + TOPK=logical_indices.shape[1], + PAGE_SIZE=k_cache.shape[1], + PAGE_TABLE_WIDTH=block_table.shape[1], + GROUP_SIZE=group_size, + HEAD_DIM=q.shape[2], + NUM_QUERY_HEADS=q.shape[1], + NUM_SPLITS=num_splits, + NUM_TILES=num_tiles, + BLOCK_M=block_m, + BLOCK_N=block_n, + num_warps=partial_warps, + num_stages=2, + ) + if num_splits == 1: + return out + + _qsa_merge_splitk_kernel[(q.shape[0], q.shape[1])]( + partial_output, + partial_lse, + out, + out.stride(0), + out.stride(1), + q.shape[0], + HEAD_DIM=q.shape[2], + NUM_QUERY_HEADS=q.shape[1], + NUM_SPLITS=num_splits, + BLOCK_SPLITS=triton.next_power_of_2(num_splits), + num_warps=2, + num_stages=1, + ) + return out + + +__all__ = ["qsa_sparse_paged_attention"] diff --git a/python/freetoken/kernel/triton/qsa/compress.py b/python/freetoken/kernel/triton/qsa/compress.py new file mode 100644 index 0000000000..1f19ed1eac --- /dev/null +++ b/python/freetoken/kernel/triton/qsa/compress.py @@ -0,0 +1,331 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/qsa.py and ops/qsa_pre_indexer.py) +"""QSA index-key compression, indexer norm+rope, and fixed-width row stores. + +The pending ring is one row per (request slot, ring position) keyed by ``Req.table_idx``, +the caches are row-flat, and the fused (1+w) RMSNorm + partial NeoX rope takes the rotary +width as a parameter. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _compress_qsa_groups_kernel( + raw_keys_ptr, + ring_ptr, + ring_slots_ptr, + token_to_req_ptr, + query_start_loc_ptr, + logical_positions_ptr, + pooled_ptr, + first_positions_ptr, + stride_raw_row, + stride_ring_slot, + stride_ring_row, + stride_pooled_row, + num_rows, + num_ring_slots, + num_requests, + RING_CAPACITY: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, +) -> None: + row = tl.program_id(0) + dims = tl.arange(0, BLOCK_D) + request = tl.load(token_to_req_ptr + row) + end_position = tl.load(logical_positions_ptr + row) + valid_request = (request >= 0) & (request < num_requests) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + query_row_start = tl.load( + query_start_loc_ptr + safe_request, mask=valid_request, other=0 + ) + query_row_end = tl.load( + query_start_loc_ptr + safe_request + 1, mask=valid_request, other=0 + ) + chunk_start_position = end_position - (row - query_row_start) + ring_slot = tl.load(ring_slots_ptr + safe_request, mask=valid_request, other=-1) + valid_ring_slot = (ring_slot >= 0) & (ring_slot < num_ring_slots) + valid_row = ( + (row < num_rows) + & valid_request + & (row >= query_row_start) + & (row < query_row_end) + & (end_position >= COMPRESS_RATIO - 1) + ) + accumulator = tl.zeros((BLOCK_D,), dtype=tl.float32) + + # A group can span the pending ring (older members) and this step's raw rows + # (members at positions >= chunk_start_position). + for group_offset in tl.range(0, COMPRESS_RATIO): + position = end_position - (COMPRESS_RATIO - 1 - group_offset) + use_raw = position >= chunk_start_position + raw_row = query_row_start + position - chunk_start_position + raw_values = tl.load( + raw_keys_ptr + raw_row * stride_raw_row + dims, + mask=valid_row + & use_raw + & (raw_row >= query_row_start) + & (raw_row < query_row_end) + & (raw_row < num_rows) + & (dims < HEAD_DIM), + other=0.0, + ).to(tl.float32) + ring_values = tl.load( + ring_ptr + + tl.maximum(ring_slot, 0).to(tl.int64) * stride_ring_slot + + (position % RING_CAPACITY) * stride_ring_row + + dims, + mask=valid_row + & ~use_raw + & valid_ring_slot + & (dims < HEAD_DIM), + other=0.0, + ).to(tl.float32) + accumulator += tl.where(use_raw, raw_values, ring_values) + + tl.store( + pooled_ptr + row * stride_pooled_row + dims, + accumulator / COMPRESS_RATIO, + mask=(row < num_rows) & (dims < HEAD_DIM), + ) + first_position = end_position - COMPRESS_RATIO + 1 + tl.store( + first_positions_ptr + row, + tl.where(valid_row, first_position, 0), + mask=row < num_rows, + ) + + +@triton.jit +def _index_norm_rope_kernel( + x_ptr, + positions_ptr, + cos_sin_ptr, + weight_ptr, + out_ptr, + dest_rows_ptr, + stride_x_row, + stride_out_row, + stride_cos_sin_row, + num_rows, + eps, + HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + ROTARY_HALF: tl.constexpr, + BLOCK_R: tl.constexpr, + BLOCK_D: tl.constexpr, + HAS_DEST_ROWS: tl.constexpr, +) -> None: + rows = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R) + live = rows < num_rows + dims = tl.arange(0, BLOCK_D) + in_dim = dims < HEAD_DIM + in_rotary = dims < 2 * ROTARY_HALF + # NeoX pairs dim d with d + rotary_dim/2; both halves are read so the rotation needs + # no cross-lane shuffle. + pair = dims % ROTARY_HALF + partner = tl.where(dims < ROTARY_HALF, dims + ROTARY_HALF, dims - ROTARY_HALF) + partner = tl.where(in_rotary, partner, dims) + + base = x_ptr + rows[:, None].to(tl.int64) * stride_x_row + mask = live[:, None] & in_dim[None, :] + x = tl.load(base + dims[None, :], mask=mask, other=0.0).to(tl.float32) + x_partner = tl.load(base + partner[None, :], mask=mask, other=0.0).to(tl.float32) + weight = tl.load(weight_ptr + dims, mask=in_dim, other=0.0).to(tl.float32) + 1.0 + weight_partner = ( + tl.load(weight_ptr + partner, mask=in_dim, other=0.0).to(tl.float32) + 1.0 + ) + rrms = tl.rsqrt(tl.sum(x * x, axis=1) / HEAD_DIM + eps) + y = x * rrms[:, None] * weight[None, :] + y_partner = x_partner * rrms[:, None] * weight_partner[None, :] + + position = tl.load(positions_ptr + rows // HEADS, mask=live, other=0).to(tl.int64) + cos_base = cos_sin_ptr + position[:, None] * stride_cos_sin_row + rotary_mask = live[:, None] & in_rotary[None, :] + cos = tl.load(cos_base + pair[None, :], mask=rotary_mask, other=1.0) + sin = tl.load(cos_base + ROTARY_HALF + pair[None, :], mask=rotary_mask, other=0.0) + sign = tl.where(dims < ROTARY_HALF, -1.0, 1.0) + result = tl.where(in_rotary[None, :], y * cos + sign[None, :] * y_partner * sin, y) + + if HAS_DEST_ROWS: + dest = tl.load(dest_rows_ptr + rows, mask=live, other=-1) + live = live & (dest >= 0) + dest_row = tl.maximum(dest, 0).to(tl.int64) + else: + dest_row = rows.to(tl.int64) + tl.store( + out_ptr + dest_row[:, None] * stride_out_row + dims[None, :], + result.to(out_ptr.dtype.element_ty), + mask=live[:, None] & in_dim[None, :], + ) + + +@triton.jit +def _store_qsa_rows_kernel( + cache_ptr, + slots_ptr, + rows_ptr, + stride_cache_block, + stride_cache_token, + stride_rows_row, + num_rows, + num_blocks, + PAGE_SIZE: tl.constexpr, + WIDTH: tl.constexpr, + BLOCK_D: tl.constexpr, +) -> None: + row = tl.program_id(0) + dims = tl.arange(0, BLOCK_D) + slot = tl.load(slots_ptr + row) + valid = (row < num_rows) & (slot >= 0) & (slot < num_blocks * PAGE_SIZE) + block = tl.maximum(slot, 0) // PAGE_SIZE + token = tl.maximum(slot, 0) % PAGE_SIZE + values = tl.load( + rows_ptr + row * stride_rows_row + dims, + mask=valid & (dims < WIDTH), + other=0, + ) + tl.store( + cache_ptr + + block.to(tl.int64) * stride_cache_block + + token * stride_cache_token + + dims, + values, + mask=valid & (dims < WIDTH), + ) + + +def qsa_compress_groups( + raw_keys: torch.Tensor, + ring: torch.Tensor, + ring_slots: torch.Tensor, + token_to_req: torch.Tensor, + query_start_loc: torch.Tensor, + logical_positions: torch.Tensor, + compress_ratio: int, + pooled: torch.Tensor, + first_positions: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + """Pool each row's closing group from the pending ring and this step's raw rows.""" + + rows = raw_keys.shape[0] + head_dim = raw_keys.shape[1] + if ring.ndim != 3 or ring.shape[2] != head_dim: + raise ValueError("QSA pending ring must be [slots, capacity, head_dim]") + if ring.shape[1] < compress_ratio: + raise ValueError("QSA ring capacity must cover a whole group") + if raw_keys.stride(1) != 1 or ring.stride(2) != 1 or pooled.stride(1) != 1: + raise ValueError("QSA compression needs unit-stride key rows") + if not rows: + return pooled, first_positions + _compress_qsa_groups_kernel[(rows,)]( + raw_keys, + ring, + ring_slots, + token_to_req, + query_start_loc, + logical_positions, + pooled, + first_positions, + raw_keys.stride(0), + ring.stride(0), + ring.stride(1), + pooled.stride(0), + rows, + ring.shape[0], + query_start_loc.shape[0] - 1, + RING_CAPACITY=ring.shape[1], + COMPRESS_RATIO=compress_ratio, + HEAD_DIM=head_dim, + BLOCK_D=triton.next_power_of_2(head_dim), + num_warps=4, + ) + return pooled, first_positions + + +def qsa_index_norm_rope( + x: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + norm_weight: torch.Tensor, + eps: float, + out: torch.Tensor, + heads: int = 1, + dest_rows: torch.Tensor | None = None, +) -> torch.Tensor: + """Zero-centered RMSNorm then partial NeoX rope on [rows, head_dim] indexer rows.""" + + rows, head_dim = x.shape + rotary_dim = cos_sin_cache.shape[1] + if rotary_dim % 2 or rotary_dim > head_dim: + raise ValueError("QSA indexer rope needs an even rotary_dim <= head_dim") + if x.stride(1) != 1 or out.stride(1) != 1 or not cos_sin_cache.is_contiguous(): + raise ValueError("QSA indexer norm+rope needs unit-stride rows") + if rows % heads: + raise ValueError("QSA indexer rows must be a whole number of head groups") + if not rows: + return out + block_r = 8 if head_dim >= 128 else 16 + _index_norm_rope_kernel[(triton.cdiv(rows, block_r),)]( + x, + positions, + cos_sin_cache, + norm_weight, + out, + dest_rows, + x.stride(0), + out.stride(0), + cos_sin_cache.stride(0), + rows, + eps, + HEADS=heads, + HEAD_DIM=head_dim, + ROTARY_HALF=rotary_dim // 2, + BLOCK_R=block_r, + BLOCK_D=triton.next_power_of_2(head_dim), + HAS_DEST_ROWS=dest_rows is not None, + num_warps=4, + ) + return out + + +def qsa_store_rows( + cache: torch.Tensor, + slot_mapping: torch.Tensor, + rows: torch.Tensor, +) -> None: + """Scatter rows into a ``[blocks, block_size, width]`` cache at ``block * block_size + + offset``; negative slots are dropped. The cache may be a strided per-layer view.""" + + if cache.ndim != 3 or rows.ndim != 2 or rows.shape[1] != cache.shape[2]: + raise ValueError("QSA row store needs a [blocks, block_size, width] cache") + if cache.stride(2) != 1 or rows.stride(1) != 1: + raise ValueError("QSA row store needs unit-stride rows") + if rows.shape[0] != slot_mapping.numel(): + raise ValueError("QSA row store slots and rows disagree") + if not rows.shape[0]: + return + _store_qsa_rows_kernel[(rows.shape[0],)]( + cache, + slot_mapping, + rows, + cache.stride(0), + cache.stride(1), + rows.stride(0), + rows.shape[0], + cache.shape[0], + PAGE_SIZE=cache.shape[1], + WIDTH=cache.shape[2], + BLOCK_D=triton.next_power_of_2(cache.shape[2]), + num_warps=4, + ) + + +__all__ = ["qsa_compress_groups", "qsa_index_norm_rope", "qsa_store_rows"] diff --git a/python/freetoken/kernel/triton/qsa/expand.py b/python/freetoken/kernel/triton/qsa/expand.py new file mode 100644 index 0000000000..877b539185 --- /dev/null +++ b/python/freetoken/kernel/triton/qsa/expand.py @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/qsa.py) +"""Expand QSA top-k blocks into token indices.""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _expand_qsa_indices_kernel( + block_indices_ptr, + query_positions_ptr, + sequence_lengths_ptr, + token_to_req_ptr, + output_ptr, + stride_blocks_row, + stride_blocks_column, + stride_output_row, + stride_output_column, + rows, + num_requests, + BLOCK_TOPK: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, + TOKEN_TOPK: tl.constexpr, + OUTPUT_WIDTH: tl.constexpr, + COLUMN_BLOCK: tl.constexpr, +) -> None: + # row * stride can overflow int32 for large row counts. + row = tl.program_id(0).to(tl.int64) + columns = tl.program_id(1) * COLUMN_BLOCK + tl.arange(0, COLUMN_BLOCK) + query_position = tl.load(query_positions_ptr + row) + request = tl.load(token_to_req_ptr + row) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + sequence_length = tl.load( + sequence_lengths_ptr + safe_request, + mask=(request >= 0) & (request < num_requests), + other=0, + ) + complete_blocks = tl.minimum( + tl.minimum( + (query_position + 1) // COMPRESS_RATIO, + sequence_length // COMPRESS_RATIO, + ), + BLOCK_TOPK, + ) + expanded_count = complete_blocks * COMPRESS_RATIO + tail_start = ((query_position + 1) // COMPRESS_RATIO) * COMPRESS_RATIO + tail_count = (query_position + 1) - tail_start + + is_expanded = columns < expanded_count + block_rank = columns // COMPRESS_RATIO + offset = columns % COMPRESS_RATIO + safe_rank = tl.minimum(block_rank, BLOCK_TOPK - 1) + block = tl.load( + block_indices_ptr + row * stride_blocks_row + safe_rank * stride_blocks_column, + mask=(row < rows) & is_expanded, + other=-1, + ) + expanded = block * COMPRESS_RATIO + offset + tail_offset = columns - expanded_count + is_tail = ( + (columns >= expanded_count) + & (tail_offset < tail_count) + & (tail_offset < COMPRESS_RATIO - 1) + ) + token = tl.where(is_expanded, expanded, tail_start + tail_offset) + valid = ( + (row < rows) + & (columns < OUTPUT_WIDTH) + & (is_expanded | is_tail) + & (token >= 0) + & (token < sequence_length) + ) + tl.store( + output_ptr + row * stride_output_row + columns * stride_output_column, + tl.where(valid, token, -1), + mask=(row < rows) & (columns < OUTPUT_WIDTH), + ) + + +def expand_qsa_block_indices( + block_indices: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + token_to_req: torch.Tensor, + compress_ratio: int, + token_topk: int, + out: torch.Tensor, +) -> torch.Tensor: + """Expand compressed blocks and compact the causal tail of the open group.""" + + if token_topk % compress_ratio: + raise ValueError("QSA token top-k must be divisible by compression ratio") + block_topk = token_topk // compress_ratio + output_width = token_topk + compress_ratio - 1 + if block_indices.shape != (query_positions.numel(), block_topk): + raise ValueError("QSA compressed top-k has an invalid shape") + if out.shape != (block_indices.shape[0], output_width): + raise ValueError("QSA expansion output has an invalid shape") + if not block_indices.shape[0]: + return out + column_block = 256 + _expand_qsa_indices_kernel[ + (block_indices.shape[0], triton.cdiv(output_width, column_block)) + ]( + block_indices, + query_positions, + sequence_lengths, + token_to_req, + out, + block_indices.stride(0), + block_indices.stride(1), + out.stride(0), + out.stride(1), + block_indices.shape[0], + sequence_lengths.shape[0], + BLOCK_TOPK=block_topk, + COMPRESS_RATIO=compress_ratio, + TOKEN_TOPK=token_topk, + OUTPUT_WIDTH=output_width, + COLUMN_BLOCK=column_block, + num_warps=4, + ) + return out + + +__all__ = ["expand_qsa_block_indices"] diff --git a/python/freetoken/kernel/triton/qsa/score.py b/python/freetoken/kernel/triton/qsa/score.py new file mode 100644 index 0000000000..49d7020828 --- /dev/null +++ b/python/freetoken/kernel/triton/qsa/score.py @@ -0,0 +1,191 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/qsa.py) +"""QSA block scoring over the paged compressed-key slab.""" + +from __future__ import annotations + +import math + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _qsa_mqa_paged_kernel( + q_ptr, + k_cache_ptr, + page_table_ptr, + token_to_req_ptr, + query_positions_ptr, + sequence_lengths_ptr, + visible_blocks_ptr, + logits_ptr, + stride_q_row, + stride_q_head, + stride_q_dim, + stride_cache_block, + stride_cache_token, + stride_cache_dim, + stride_table_req, + stride_table_page, + stride_logits_row, + num_rows, + num_columns, + num_pages, + num_requests, + score_divisor, + PAGE_SIZE: tl.constexpr, + PAGE_TABLE_WIDTH: tl.constexpr, + NUM_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, + TILES_PER_PROG: tl.constexpr, + STAGES: tl.constexpr, + MAX_N: tl.constexpr, + COMPRESS_RATIO: tl.constexpr, +) -> None: + row = tl.program_id(0) + dims = tl.arange(0, BLOCK_D) + heads = tl.arange(0, MAX_N) + request = tl.load(token_to_req_ptr + row) + safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1) + query_position = tl.load(query_positions_ptr + row) + sequence_length = tl.load( + sequence_lengths_ptr + safe_request, + mask=(request >= 0) & (request < num_requests), + other=0, + ) + visible = tl.minimum( + (query_position + 1) // COMPRESS_RATIO, + sequence_length // COMPRESS_RATIO, + ) + if tl.program_id(1) == 0: + tl.store(visible_blocks_ptr + row, visible) + tile_start = tl.program_id(1) * TILES_PER_PROG + # Top-k is bounded by visible_blocks, so columns beyond it need no value. + if tile_start * BLOCK_N >= visible: + return + tile_end = tl.minimum(tile_start + TILES_PER_PROG, tl.cdiv(visible, BLOCK_N)) + tile_end = tl.minimum(tile_end, tl.cdiv(num_columns, BLOCK_N)) + + # Pad the small head axis to a tensor-core-compatible N dimension. + query = tl.load( + q_ptr + + row * stride_q_row + + heads[None, :] * stride_q_head + + dims[:, None] * stride_q_dim, + mask=(heads[None, :] < NUM_HEADS) & (dims[:, None] < HEAD_DIM), + other=0.0, + ) + column_offsets = tl.arange(0, BLOCK_N) + for tile in tl.range(tile_start, tile_end, num_stages=STAGES): + columns = tile * BLOCK_N + column_offsets + live = columns < visible + logical_page = tl.minimum(columns // PAGE_SIZE, PAGE_TABLE_WIDTH - 1) + page_offset = columns % PAGE_SIZE + physical_page = tl.load( + page_table_ptr + + safe_request * stride_table_req + + logical_page * stride_table_page, + mask=live, + other=-1, + ) + page_valid = live & (physical_page >= 0) & (physical_page < num_pages) + # physical_page * block stride can overflow int32 for large caches. + safe_physical_page = tl.maximum(physical_page, 0).to(tl.int64) + keys = tl.load( + k_cache_ptr + + safe_physical_page[:, None] * stride_cache_block + + page_offset[:, None] * stride_cache_token + + dims[None, :] * stride_cache_dim, + mask=page_valid[:, None] & (dims[None, :] < HEAD_DIM), + other=0.0, + eviction_policy="evict_first", + ) + scores = tl.dot(keys, query, out_dtype=tl.float32) + scores = tl.where(heads[None, :] < NUM_HEADS, tl.maximum(scores, 0.0), 0.0) + score = tl.sum(scores, axis=1) / score_divisor + tl.store( + logits_ptr + row * stride_logits_row + columns, + tl.where(page_valid, score, -float("inf")), + mask=live & (columns < num_columns), + ) + + +def qsa_mqa_paged( + q: torch.Tensor, + k_cache: torch.Tensor, + page_table: torch.Tensor, + token_to_req: torch.Tensor, + query_positions: torch.Tensor, + sequence_lengths: torch.Tensor, + compress_ratio: int, + logits: torch.Tensor, + visible_blocks: torch.Tensor, + score_scale: float | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: + """Compute QSA scores directly from a paged compressed-key cache.""" + + if q.ndim != 3 or q.shape[1] <= 0 or q.shape[2] <= 0: + raise ValueError("QSA query must be [rows, heads, head_dim]") + if k_cache.ndim != 4 or k_cache.shape[2] != 1: + raise ValueError("QSA cache must be [pages, page_size, 1, head_dim]") + if k_cache.shape[3] != q.shape[2]: + raise ValueError("QSA query and cache dimensions must match") + if token_to_req.shape != (q.shape[0],) or query_positions.shape != (q.shape[0],): + raise ValueError("QSA request mapping and positions must match query rows") + if sequence_lengths.shape != (page_table.shape[0],): + raise ValueError("QSA sequence lengths must match page-table requests") + score_divisor = math.sqrt(q.shape[2]) if score_scale is None else score_scale + columns = logits.shape[1] + if not q.shape[0] or not columns: + return logits, visible_blocks + BLOCK_N = 64 + BLOCK_D = max(16, triton.next_power_of_2(q.shape[2])) + MAX_N = max(16, triton.next_power_of_2(q.shape[1])) + # Tuned on GB300: larger row batches provide enough parallelism to reuse Q. + tiles_per_program = 1 if q.shape[0] <= 32 else 8 + _qsa_mqa_paged_kernel[ + (q.shape[0], triton.cdiv(columns, BLOCK_N * tiles_per_program)) + ]( + q, + k_cache, + page_table, + token_to_req, + query_positions, + sequence_lengths, + visible_blocks, + logits, + q.stride(0), + q.stride(1), + q.stride(2), + k_cache.stride(0), + k_cache.stride(1), + k_cache.stride(3), + page_table.stride(0), + page_table.stride(1), + logits.stride(0), + q.shape[0], + columns, + k_cache.shape[0], + page_table.shape[0], + float(score_divisor), + PAGE_SIZE=k_cache.shape[1], + PAGE_TABLE_WIDTH=page_table.shape[1], + NUM_HEADS=q.shape[1], + HEAD_DIM=q.shape[2], + BLOCK_N=BLOCK_N, + BLOCK_D=BLOCK_D, + TILES_PER_PROG=tiles_per_program, + STAGES=2, + MAX_N=MAX_N, + COMPRESS_RATIO=compress_ratio, + num_warps=2, + ) + return logits, visible_blocks + + +__all__ = ["qsa_mqa_paged"] diff --git a/python/freetoken/kernel/triton/qsa/topk.py b/python/freetoken/kernel/triton/qsa/topk.py new file mode 100644 index 0000000000..ae57566cab --- /dev/null +++ b/python/freetoken/kernel/triton/qsa/topk.py @@ -0,0 +1,458 @@ +"""Exact block top-k over the QSA indexer scores. + +torch.topk also sorts its k winners, which the selection never uses: expand.py expands every +rank the same way and the sparse attend kernel softmaxes over the union, so only the SET +matters. One program per row runs an MSB-first radix select on the monotone uint32 image of +the fp32 scores -- ``PASSES`` ``RADIX``-bit passes, each a histogram of the columns that still +match the fixed prefix -- and one compaction pass then emits the winners in ascending column +order, -1 padded to the output width like the torch.topk path it replaces. + +A row that fits one tile is held in registers across the passes (``SINGLE_TILE``); wider rows +re-read the tile per pass, which is the price of not spilling a 256 KB row. + +One program per row stops scaling once a row runs to tens of thousands of columns: the whole +row goes through a single CTA. Wide buffers therefore split. Phase 1 gives each ``CHUNK``-wide +slice of a row its own program, which radix-selects the slice's own top-k into a candidate +workspace; phase 2 runs the same radix select over the union of those candidates. The global +top-k is a subset of that union -- a global winner beats at most k-1 columns, so it beats all +but at most k-1 of its own slice -- so the answer stays exact. ``_split_plan`` derives the +geometry from the buffer width alone, which is fixed when a graph is captured. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +# 5-bit digits beat 8-bit ones here: tl.histogram cost grows faster with the bin count than the +# extra passes cost. 35 bits of digit cover the 32-bit key, the top pass just sees zero bits. +_RADIX = 5 +_BINS = 1 << _RADIX +_PASSES = -(-32 // _RADIX) +_MAX_BLOCK_N = 4096 +# A resident tile costs ~2 ns per column against ~3.2 ns for a re-read one, so both split +# phases stay resident; past 8192 the tile spills (255 registers) and that reverses. +_MAX_RESIDENT = 8192 +_MIN_CHUNK = 4096 + + +@triton.jit +def _monotone_key(value): + """fp32 -> uint32 with the float order preserved; key 0 is reserved for a dead column.""" + bits = value.to(tl.uint32, bitcast=True) + return tl.where((bits >> 31) == 0, bits | 0x80000000, ~bits) + + +@triton.jit +def _load_keys(logits_row, columns, limit): + live = columns < limit + value = tl.load(logits_row + columns, mask=live, other=-float("inf")) + return tl.where(live & (value > -float("inf")), _monotone_key(value), 0) + + +@triton.jit +def _narrow(hist, prefix, keep, k_rem, shift, BINS: tl.constexpr): + """One radix step: pin the digit at ``shift`` and report whether the range is settled.""" + bins = tl.arange(0, BINS) + # Lowest bin whose strictly-greater bins no longer cover k_rem holds the k_rem-th key; + # `above` falls with the bin id, so the predicate is an upper set. + above = tl.sum(hist) - tl.cumsum(hist, axis=0) + inside = above < k_rem + bin_id = tl.min(tl.where(inside, bins, BINS - 1)) + hit = bins == bin_id + prefix |= bin_id.to(tl.uint32) << shift + keep |= tl.full((), BINS - 1, tl.uint32) << shift + k_rem -= tl.sum(tl.where(hit, above, 0)) + # The whole bin fits: `prefix` is a lower bound, the ties below it all win. + return prefix, keep, k_rem, k_rem == tl.sum(tl.where(hit, hist, 0)) + + +@triton.jit +def _resident_prefix(key, k_eff, BINS: tl.constexpr, RADIX: tl.constexpr, PASSES: tl.constexpr): + """Radix-select a register-resident tile; returns the winning key range and its tie budget.""" + # Invariant per pass: `k_rem` winners are still to be found inside the key range that + # `prefix` pins on the `keep` bits, and every key above that range is already a winner. + prefix = tl.zeros((), tl.uint32) + keep = tl.zeros((), tl.uint32) + k_rem = k_eff + settled = False + for step in tl.static_range(PASSES): + shift = RADIX * (PASSES - 1 - step) + if not settled: + hist = tl.histogram( + ((key >> shift) & (BINS - 1)).to(tl.int32), + BINS, + mask=(key != 0) & ((key & keep) == prefix), + ) + prefix, keep, k_rem, settled = _narrow(hist, prefix, keep, k_rem, shift, BINS) + return prefix, tl.where(settled, k_eff, k_rem) + + +@triton.jit +def _tile_ranks(key, prefix, ties, above_base, equal_base): + """Rank every winner of one tile, and carry the running counts past it. + + One packed cumsum carries both: keys above the threshold in the low half, keys equal to it + in the high half, so a winner's rank is ``above_before + min(equal_before, ties)``.""" + greater = ((key > prefix) & (key != 0)).to(tl.int32) + equal = ((key == prefix) & (key != 0)).to(tl.int32) + packed = greater | (equal << 16) + before = tl.cumsum(packed, axis=0) - packed + rank_equal = equal_base + (before >> 16) + take = (greater == 1) | ((equal == 1) & (rank_equal < ties)) + rank = above_base + (before & 0xFFFF) + tl.minimum(rank_equal, ties) + total = tl.sum(packed) + return rank, take, above_base + (total & 0xFFFF), equal_base + (total >> 16) + + +@triton.jit +def _compact( + out_row, + key, + columns, + prefix, + ties, + above_base, + equal_base, + TOP_K: tl.constexpr, + BLOCK_N: tl.constexpr, +): + """Emit one tile's winners at their global ranks; returns the running counts after it.""" + tl.static_assert(BLOCK_N <= 0xFFFF, "packed cumsum keeps 16 bits per half") + rank, take, above, equal = _tile_ranks(key, prefix, ties, above_base, equal_base) + tl.store(out_row + rank, columns.to(tl.int32), mask=take & (rank < TOP_K)) + return above, equal + + +@triton.jit +def _qsa_block_topk_kernel( + logits_ptr, + visible_ptr, + out_ptr, + stride_logits_row, + stride_out_row, + num_columns, + TOP_K: tl.constexpr, + PAD_K: tl.constexpr, + BLOCK_N: tl.constexpr, + SINGLE_TILE: tl.constexpr, + BINS: tl.constexpr, + RADIX: tl.constexpr, + PASSES: tl.constexpr, +) -> None: + row = tl.program_id(0) + limit = tl.maximum(tl.minimum(tl.load(visible_ptr + row), num_columns), 0) + k_eff = tl.minimum(limit, TOP_K) + logits_row = logits_ptr + row.to(tl.int64) * stride_logits_row + out_row = out_ptr + row.to(tl.int64) * stride_out_row + offsets = tl.arange(0, BLOCK_N) + tiles = tl.cdiv(limit, BLOCK_N) + if SINGLE_TILE: + resident = _load_keys(logits_row, offsets, limit) + + emitted = 0 + if k_eff > 0: + if SINGLE_TILE: + prefix, ties = _resident_prefix(resident, k_eff, BINS, RADIX, PASSES) + else: + prefix = tl.zeros((), tl.uint32) + keep = tl.zeros((), tl.uint32) + k_rem = k_eff + settled = False + for step in tl.static_range(PASSES): + shift = RADIX * (PASSES - 1 - step) + if not settled: + hist = tl.zeros((BINS,), tl.int32) + for tile in range(tiles): + key = _load_keys(logits_row, tile * BLOCK_N + offsets, limit) + hist += tl.histogram( + ((key >> shift) & (BINS - 1)).to(tl.int32), + BINS, + mask=(key != 0) & ((key & keep) == prefix), + ) + prefix, keep, k_rem, settled = _narrow( + hist, prefix, keep, k_rem, shift, BINS + ) + ties = tl.where(settled, k_eff, k_rem) + + above_base = 0 + equal_base = 0 + if SINGLE_TILE: + above_base, equal_base = _compact( + out_row, resident, offsets, prefix, ties, above_base, equal_base, TOP_K, BLOCK_N + ) + else: + for tile in range(tiles): + columns = tile * BLOCK_N + offsets + key = _load_keys(logits_row, columns, limit) + above_base, equal_base = _compact( + out_row, key, columns, prefix, ties, above_base, equal_base, TOP_K, BLOCK_N + ) + emitted = above_base + tl.minimum(equal_base, ties) + + pad = tl.arange(0, PAD_K) + tl.store(out_row + pad, -1, mask=(pad >= emitted) & (pad < TOP_K)) + + +@triton.jit +def _qsa_topk_split_kernel( + logits_ptr, + visible_ptr, + key_ptr, + col_ptr, + stride_logits_row, + stride_scratch_row, + num_columns, + TOP_K: tl.constexpr, + PAD_K: tl.constexpr, + CHUNK: tl.constexpr, + BINS: tl.constexpr, + RADIX: tl.constexpr, + PASSES: tl.constexpr, +) -> None: + """Phase 1: one program per (row, chunk), writing the chunk's own top-k as candidates.""" + tl.static_assert(CHUNK <= 0xFFFF, "packed cumsum keeps 16 bits per half") + row = tl.program_id(0) + split = tl.program_id(1) + base = split * CHUNK + visible = tl.maximum(tl.minimum(tl.load(visible_ptr + row), num_columns), 0) + limit = tl.minimum(tl.maximum(visible - base, 0), CHUNK) + k_eff = tl.minimum(limit, TOP_K) + slot = row.to(tl.int64) * stride_scratch_row + split * TOP_K + + emitted = 0 + if k_eff > 0: + offsets = tl.arange(0, CHUNK) + key = _load_keys(logits_ptr + row.to(tl.int64) * stride_logits_row + base, offsets, limit) + prefix, ties = _resident_prefix(key, k_eff, BINS, RADIX, PASSES) + rank, take, above, equal = _tile_ranks(key, prefix, ties, 0, 0) + write = take & (rank < TOP_K) + tl.store(key_ptr + slot + rank, key.to(tl.int32, bitcast=True), mask=write) + tl.store(col_ptr + slot + rank, (base + offsets).to(tl.int32), mask=write) + emitted = above + tl.minimum(equal, ties) + # A key of 0 is the dead-column sentinel, so the merge needs no separate count per slot. + pad = tl.arange(0, PAD_K) + tl.store(key_ptr + slot + pad, 0, mask=(pad >= emitted) & (pad < TOP_K)) + + +@triton.jit +def _merge_tile( + key_row, + col_row, + out_row, + candidates, + k_eff, + TOP_K: tl.constexpr, + BLOCK: tl.constexpr, + BINS: tl.constexpr, + RADIX: tl.constexpr, + PASSES: tl.constexpr, +): + """Top-k of one resident tile of candidates; returns how many winners it wrote.""" + tl.static_assert(BLOCK <= 0xFFFF, "packed cumsum keeps 16 bits per half") + offsets = tl.arange(0, BLOCK) + live = offsets < candidates + key = tl.load(key_row + offsets, mask=live, other=0).to(tl.uint32, bitcast=True) + prefix, ties = _resident_prefix(key, k_eff, BINS, RADIX, PASSES) + rank, take, above, equal = _tile_ranks(key, prefix, ties, 0, 0) + column = tl.load(col_row + offsets, mask=live, other=-1) + tl.store(out_row + rank, column, mask=take & (rank < TOP_K)) + return above + tl.minimum(equal, ties) + + +@triton.jit +def _qsa_topk_merge_kernel( + visible_ptr, + key_ptr, + col_ptr, + out_ptr, + stride_scratch_row, + stride_out_row, + num_columns, + TOP_K: tl.constexpr, + PAD_K: tl.constexpr, + CHUNK: tl.constexpr, + N_SPLITS: tl.constexpr, + BLOCK_SMALL: tl.constexpr, + BLOCK_MID: tl.constexpr, + BLOCK_FULL: tl.constexpr, + BINS: tl.constexpr, + RADIX: tl.constexpr, + PASSES: tl.constexpr, +) -> None: + """Phase 2: one program per row over the candidates phase 1 left behind.""" + row = tl.program_id(0) + limit = tl.maximum(tl.minimum(tl.load(visible_ptr + row), num_columns), 0) + k_eff = tl.minimum(limit, TOP_K) + # Candidates sit chunk-major, so the splits past the visible tail are one skipped suffix + # and the merge keeps costing what the live part of the row costs. + candidates = tl.minimum(tl.cdiv(limit, CHUNK), N_SPLITS) * TOP_K + key_row = key_ptr + row.to(tl.int64) * stride_scratch_row + col_row = col_ptr + row.to(tl.int64) * stride_scratch_row + out_row = out_ptr + row.to(tl.int64) * stride_out_row + + emitted = 0 + if k_eff > 0: + if candidates <= TOP_K: + # One live split: its own top-k is already the row's, ranked and packed. + slot = tl.arange(0, PAD_K) + inside = slot < TOP_K + key = tl.load(key_row + slot, mask=inside, other=0) + alive = inside & (key != 0) + column = tl.load(col_row + slot, mask=alive, other=-1) + tl.store(out_row + slot, column, mask=inside) + emitted = tl.sum(alive.to(tl.int32)) + # Register residency costs the whole tile even when few candidates are live, so a + # short row takes a narrower tile. + elif candidates <= BLOCK_SMALL: + emitted = _merge_tile( + key_row, col_row, out_row, candidates, k_eff, + TOP_K, BLOCK_SMALL, BINS, RADIX, PASSES, + ) + elif candidates <= BLOCK_MID: + emitted = _merge_tile( + key_row, col_row, out_row, candidates, k_eff, + TOP_K, BLOCK_MID, BINS, RADIX, PASSES, + ) + else: + emitted = _merge_tile( + key_row, col_row, out_row, candidates, k_eff, + TOP_K, BLOCK_FULL, BINS, RADIX, PASSES, + ) + + pad = tl.arange(0, PAD_K) + tl.store(out_row + pad, -1, mask=(pad >= emitted) & (pad < TOP_K)) + + +def _split_plan(columns: int, top_k: int) -> tuple[int, int] | None: + """``(chunk, n_splits)`` for the split+merge path, or None to keep the one-program path.""" + if top_k <= 0 or columns <= _MIN_CHUNK: + return None + max_splits = _MAX_RESIDENT // triton.next_power_of_2(top_k) + if max_splits < 2: + return None + chunk = max(_MIN_CHUNK, triton.next_power_of_2(-(-columns // max_splits))) + if chunk > _MAX_RESIDENT: + return None + n_splits = -(-columns // chunk) + # Merging n_splits*top_k candidates has to be cheaper than scanning the row once. + if n_splits < 2 or n_splits * top_k >= columns: + return None + return chunk, n_splits + + +def qsa_block_topk_scratch_width(columns: int, top_k: int) -> int: + """int32 columns of scratch ``qsa_block_topk`` wants per row; 0 when it needs none.""" + plan = _split_plan(columns, top_k) + return 0 if plan is None else 2 * plan[1] * top_k + + +def qsa_block_topk( + logits: torch.Tensor, + visible: torch.Tensor, + out: torch.Tensor, + scratch: torch.Tensor | None = None, +) -> torch.Tensor: + """Top ``out.shape[1]`` columns of every ``logits`` row below ``visible``, -1 padded. + + Winners come out in ascending column order, packed at the front of the row; a row with + fewer live columns than the width, and any column scored -inf, leaves -1 in the tail. + ``scratch`` is the candidate workspace of the split path (see + ``qsa_block_topk_scratch_width``); it is allocated per call when the caller passes none.""" + + if logits.ndim != 2 or out.ndim != 2: + raise ValueError("QSA block top-k takes 2-D logits and output") + if out.shape[0] != logits.shape[0] or visible.shape != (logits.shape[0],): + raise ValueError("QSA block top-k needs one visible count and one output row per row") + if logits.dtype != torch.float32 or out.dtype != torch.int32: + raise ValueError("QSA block top-k takes fp32 logits and an int32 output") + if logits.stride(1) != 1 or out.stride(1) != 1 or visible.stride(0) != 1: + raise ValueError("QSA block top-k needs row-contiguous logits, output and counts") + rows, columns = logits.shape + top_k = out.shape[1] + if not rows or not top_k: + return out + pad_k = triton.next_power_of_2(top_k) + plan = _split_plan(columns, top_k) + if plan is None: + block_n = min(_MAX_BLOCK_N, triton.next_power_of_2(max(columns, 1))) + _qsa_block_topk_kernel[(rows,)]( + logits, + visible, + out, + logits.stride(0), + out.stride(0), + columns, + TOP_K=top_k, + PAD_K=pad_k, + BLOCK_N=block_n, + SINGLE_TILE=columns <= block_n, + BINS=_BINS, + RADIX=_RADIX, + PASSES=_PASSES, + num_warps=8, + num_stages=1, + ) + return out + + chunk, n_splits = plan + half = n_splits * top_k + if scratch is None: + scratch = torch.empty((rows, 2 * half), dtype=torch.int32, device=logits.device) + elif ( + scratch.ndim != 2 + or scratch.shape[0] < rows + or scratch.shape[1] < 2 * half + or scratch.dtype != torch.int32 + or scratch.stride(1) != 1 + ): + raise ValueError( + f"QSA block top-k needs a row-contiguous int32 scratch of at least " + f"[{rows}, {2 * half}], got {tuple(scratch.shape)} {scratch.dtype}" + ) + keys = scratch[:rows, :half] + cols = scratch[:rows, half : 2 * half] + _qsa_topk_split_kernel[(rows, n_splits)]( + logits, + visible, + keys, + cols, + logits.stride(0), + keys.stride(0), + columns, + TOP_K=top_k, + PAD_K=pad_k, + CHUNK=chunk, + BINS=_BINS, + RADIX=_RADIX, + PASSES=_PASSES, + num_warps=8, + num_stages=1, + ) + merge_block = triton.next_power_of_2(half) + _qsa_topk_merge_kernel[(rows,)]( + visible, + keys, + cols, + out, + keys.stride(0), + out.stride(0), + columns, + TOP_K=top_k, + PAD_K=pad_k, + CHUNK=chunk, + N_SPLITS=n_splits, + BLOCK_SMALL=min(1024, merge_block), + BLOCK_MID=min(4096, merge_block), + BLOCK_FULL=merge_block, + BINS=_BINS, + RADIX=_RADIX, + PASSES=_PASSES, + num_warps=8, + num_stages=1, + ) + return out + + +__all__ = ["qsa_block_topk", "qsa_block_topk_scratch_width"] diff --git a/python/freetoken/kernel/triton/sampling.py b/python/freetoken/kernel/triton/sampling.py index 7345d65fc4..a970714406 100644 --- a/python/freetoken/kernel/triton/sampling.py +++ b/python/freetoken/kernel/triton/sampling.py @@ -1,4 +1,4 @@ -"""Multi-CTA (split-vocab) Triton sampling ops (provenance: sampling<-vllm Qrita). +"""Multi-CTA (split-vocab) Triton sampling ops. Optional pure-triton drop-in for freetoken.kernel.sampling / flashinfer.sampling (softmax / top-k / top-p / combined + draw), self-contained. @@ -6,35 +6,46 @@ Design: * Every row is split across many CTAs (``_plan`` -> G column-chunks) so bs=1 uses the whole GPU, unlike a single-block-per-row kernel that is single-SM-bound. - * softmax is a multi-CTA online softmax; top-p uses a small fixed number of - histogram-bracket refinement passes instead of a ~48-iter bisection; the draw - is a multi-CTA inverse-CDF. - * The top-k path is adapted from vLLM's Qrita kernel - (v1/sample/ops/topk_topp_triton.py::_topk_topp_kernel): gather the small set of - "outlier" candidates (probs >= rmax*FRAC) into a compact per-row buffer in ONE - full-vocab pass, then run the k-th-value search on that tiny buffer (3 full-vocab - passes total vs ~6 for a pure histogram top-k). The outlier-pivot heuristic is - swapped for the probs domain (truncate at rmax*FRAC; softmax probs are not - Gaussian). Rows overflowing CAP silently drop the smallest gathered candidates; - the refine still finds the exact k-th since every value >= threshold is kept, and - an in-kernel guard keeps everything if fewer than k finite candidates are gathered. + * softmax is a multi-CTA online softmax; the draw is a multi-CTA inverse-CDF. + * top-k and top-p each use one Triton kernel: the row's CTAs bin their chunk over + the fp32 bit pattern (order-preserving for x >= 0), meet at a per-row spin barrier, and all + redo the refine so they share the bracket. Four rounds of 256 bins bring the 2**31 range + down to one bit pattern, so the threshold is exactly the k-th largest prob (top-k, counts) + or the value where the descending cumulative mass reaches p (top-p, exact per-bin mass). + Every boundary tie is kept, matching flashinfer, then the same kernel renormalizes or + draws. No candidate buffer, data-dependent shape, or host sync is needed. Results are + exact up to fp32 atomic summation order. + * If a cooperative launch is unavailable, the same exact kernel is retried with one CTA + per row; only parallelism changes. + * deterministic, generator and check_nan exist for flashinfer signature compatibility and are + ignored; seed and offset are honored. Given a seed, top-k draws reproduce; top-p may pick a + different token on rows whose cumulative mass sits within fp32 rounding of p. """ from __future__ import annotations +import logging +from functools import cache + import torch import triton import triton.language as tl from freetoken.kernel.triton.autotune_cache import autotune_cache_kwargs -_NUM_SM = torch.cuda.get_device_properties(torch.cuda.current_device()).multi_processor_count +logger = logging.getLogger(__name__) + _MIN_CHUNK = 4096 # do not split a row finer than this -def _plan(B, V): +@cache +def _num_sm(device): + return torch.cuda.get_device_properties(device).multi_processor_count + + +def _plan(B, V, device): """Return (G, CHUNK): split each row into G column-chunks of size CHUNK.""" - g_by_sm = max(1, _NUM_SM // B) + g_by_sm = max(1, _num_sm(device) // B) g_by_chunk = max(1, triton.cdiv(V, _MIN_CHUNK)) G = min(g_by_sm, g_by_chunk) CHUNK = triton.cdiv(V, G) @@ -123,8 +134,10 @@ def _sm_finalize( def softmax(logits, temperature=None, enable_pdl=None): logits = logits.float() B, V = logits.shape + if B == 0: + return logits.clone() probs = torch.empty_like(logits) - G, CHUNK = _plan(B, V) + G, CHUNK = _plan(B, V, logits.device) if temperature is None: temperature = 1.0 if isinstance(temperature, torch.Tensor): @@ -141,13 +154,6 @@ def softmax(logits, temperature=None, enable_pdl=None): return probs -# =========================================================================== -# multi-CTA top-p via histogram-bracket refinement. Every full-vocab pass is -# split across all SMs; the sequential refine step is a tiny grid=(B,) kernel. -# =========================================================================== -_PBINS = 64 # top-p uses count-hist + bin-center mass (64**4 ~ 1.7e7) -_PR = 4 - _SR_CFGS = [ triton.Config({"BLOCK_SIZE": bs}, num_warps=w, num_stages=s) for bs in (1024, 2048, 4096) @@ -156,184 +162,9 @@ def softmax(logits, temperature=None, enable_pdl=None): ] -@triton.jit -def _rmax_pass(probs_ptr, rmax_ptr, V, G, CHUNK, row_stride, BLOCK_SIZE: tl.constexpr): - pid = tl.program_id(0) - row = pid // G - base = row * row_stride - start = (pid % G) * CHUNK - end = tl.minimum(start + CHUNK, V) - m = 0.0 - for s0 in tl.range(start, end, BLOCK_SIZE): - offs = s0 + tl.arange(0, BLOCK_SIZE) - mask = offs < end - x = tl.load(probs_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) - m = tl.maximum(m, tl.max(x, 0)) - tl.atomic_max(rmax_ptr + row, m) - - -@triton.autotune(configs=_SR_CFGS, key=["CHUNK"], reset_to_zero=["hist_ptr"], **autotune_cache_kwargs) -@triton.jit -def _count_hist_pass( - probs_ptr, lo_ptr, hi_ptr, hist_ptr, V, G, CHUNK, row_stride, - BINS: tl.constexpr, BLOCK_SIZE: tl.constexpr, -): - pid = tl.program_id(0) - row = pid // G - lo = tl.load(lo_ptr + row) - hi = tl.load(hi_ptr + row) - invw = BINS / tl.maximum(hi - lo, 1e-30) - base = row * row_stride - start = (pid % G) * CHUNK - end = tl.minimum(start + CHUNK, V) - acc = tl.zeros([BINS], tl.int32) - for s0 in tl.range(start, end, BLOCK_SIZE): - offs = s0 + tl.arange(0, BLOCK_SIZE) - mask = offs < end - x = tl.load(probs_ptr + base + offs, mask=mask, other=-1.0).to(tl.float32) - inrange = mask & (x >= lo) & (x < hi) - b = ((x - lo) * invw).to(tl.int32) - # tl.histogram does NOT cleanly drop out-of-range indices; route every - # out-of-bracket element to bin 0 and then subtract that count back out so - # the histogram holds ONLY in-[lo,hi) counts (out-of-range is tracked via - # `above`/excluded, exactly like the one-hot path). - b = tl.where(inrange, tl.maximum(0, tl.minimum(b, BINS - 1)), 0) - hcnt = tl.histogram(b, BINS) - noor = tl.sum((mask & (~inrange)).to(tl.int32)) - hcnt = hcnt - tl.where(tl.arange(0, BINS) == 0, noor, 0) - acc += hcnt - tl.atomic_add(hist_ptr + row * BINS + tl.arange(0, BINS), acc.to(tl.float32)) - - -@triton.jit -def _refine_pass(lo_ptr, hi_ptr, above_ptr, hist_ptr, target_ptr, BINS: tl.constexpr): - row = tl.program_id(0) - lo = tl.load(lo_ptr + row) - hi = tl.load(hi_ptr + row) - above = tl.load(above_ptr + row) - target = tl.load(target_ptr + row) - w = (hi - lo) / BINS - jj = tl.arange(0, BINS) - h = tl.load(hist_ptr + row * BINS + jj) - prefix = tl.cumsum(h, 0) - total = tl.sum(h, 0) - c_ge_bottom = above + total - prefix + h - ok = c_ge_bottom >= target - j = tl.max(tl.where(ok, jj, -1)) - prefix_j = tl.sum(tl.where(jj <= j, h, 0.0)) - upd = j >= 0 - tl.store(lo_ptr + row, tl.where(upd, lo + j * w, lo)) - tl.store(hi_ptr + row, tl.where(upd, lo + (j + 1) * w, hi)) - tl.store(above_ptr + row, tl.where(upd, above + total - prefix_j, above)) - # zero the row so the next iteration's atomic_add starts clean (reset_to_zero - # only fires during autotuning, not on production calls) - tl.store(hist_ptr + row * BINS + jj, 0.0) - - -@triton.jit -def _refine_mass_pass(lo_ptr, hi_ptr, above_ptr, hist_ptr, target_ptr, BINS: tl.constexpr): - # top-p refine: hist holds COUNTS; approximate per-bin MASS as count*bin_center - # (exact in the limit as the bracket narrows). target is the p mass threshold. - row = tl.program_id(0) - lo = tl.load(lo_ptr + row) - hi = tl.load(hi_ptr + row) - above = tl.load(above_ptr + row) - target = tl.load(target_ptr + row) - w = (hi - lo) / BINS - jj = tl.arange(0, BINS) - h = tl.load(hist_ptr + row * BINS + jj) - center = lo + (jj.to(tl.float32) + 0.5) * w - massbin = h * center - prefix = tl.cumsum(massbin, 0) - total = tl.sum(massbin, 0) - c_ge_bottom = above + total - prefix + massbin - ok = c_ge_bottom >= target - j = tl.max(tl.where(ok, jj, -1)) - prefix_j = tl.sum(tl.where(jj <= j, massbin, 0.0)) - upd = j >= 0 - tl.store(lo_ptr + row, tl.where(upd, lo + j * w, lo)) - tl.store(hi_ptr + row, tl.where(upd, lo + (j + 1) * w, hi)) - tl.store(above_ptr + row, tl.where(upd, above + total - prefix_j, above)) - tl.store(hist_ptr + row * BINS + jj, 0.0) - - -@triton.autotune(configs=_SR_CFGS, key=["CHUNK"], reset_to_zero=["ksum_ptr"], **autotune_cache_kwargs) -@triton.jit -def _ksum_pass(probs_ptr, thr_ptr, ksum_ptr, V, G, CHUNK, row_stride, BLOCK_SIZE: tl.constexpr): - pid = tl.program_id(0) - row = pid // G - thr = tl.load(thr_ptr + row) - base = row * row_stride - start = (pid % G) * CHUNK - end = tl.minimum(start + CHUNK, V) - s = 0.0 - for s0 in tl.range(start, end, BLOCK_SIZE): - offs = s0 + tl.arange(0, BLOCK_SIZE) - mask = offs < end - x = tl.load(probs_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) - s += tl.sum(tl.where(x >= thr, x, 0.0), 0) - tl.atomic_add(ksum_ptr + row, s) - - -@triton.autotune(configs=_SR_CFGS, key=["CHUNK"], **autotune_cache_kwargs) -@triton.jit -def _write_pass(probs_ptr, out_ptr, thr_ptr, ksum_ptr, V, G, CHUNK, row_stride, BLOCK_SIZE: tl.constexpr): - pid = tl.program_id(0) - row = pid // G - thr = tl.load(thr_ptr + row) - inv_s = 1.0 / tl.load(ksum_ptr + row) - base = row * row_stride - start = (pid % G) * CHUNK - end = tl.minimum(start + CHUNK, V) - for s0 in tl.range(start, end, BLOCK_SIZE): - offs = s0 + tl.arange(0, BLOCK_SIZE) - mask = offs < end - x = tl.load(probs_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) - tl.store(out_ptr + base + offs, tl.where(x >= thr, x * inv_s, 0.0), mask=mask) - - -def _search(probs, target, mass, R, BINS): - """Return per-row threshold: keep x >= thr, with count/mass(>=thr) ~ target.""" - B, V = probs.shape - dev = probs.device - G, CHUNK = _plan(B, V) - grid = (B * G,) - rmax = torch.zeros(B, device=dev, dtype=torch.float32) - _rmax_pass[grid](probs, rmax, V, G, CHUNK, probs.stride(0), BLOCK_SIZE=2048, num_warps=8) - lo = torch.zeros(B, device=dev, dtype=torch.float32) - hi = (rmax * 1.0000001).contiguous() - above = torch.zeros(B, device=dev, dtype=torch.float32) - hist = torch.zeros(B * BINS, device=dev, dtype=torch.float32) - for _ in range(R): - _count_hist_pass[grid](probs, lo, hi, hist, V, G, CHUNK, probs.stride(0), BINS) - if mass: - _refine_mass_pass[(B,)](lo, hi, above, hist, target, BINS) - else: - _refine_pass[(B,)](lo, hi, above, hist, target, BINS) - return lo - - -def _renorm(probs, thr): - B, V = probs.shape - dev = probs.device - G, CHUNK = _plan(B, V) - grid = (B * G,) - out = torch.empty_like(probs) - ksum = torch.zeros(B, device=dev, dtype=torch.float32) - _ksum_pass[grid](probs, thr, ksum, V, G, CHUNK, probs.stride(0)) - _write_pass[grid](probs, out, thr, ksum, V, G, CHUNK, probs.stride(0)) - return out - - def top_p_renorm_probs(probs, top_p): probs = probs.float() - B, V = probs.shape - if isinstance(top_p, torch.Tensor): - target = top_p.float().to(probs.device).contiguous() - else: - target = torch.full((B,), float(top_p), device=probs.device, dtype=torch.float32) - thr = _search(probs, target, True, _PR, _PBINS) - return _renorm(probs, thr) + return _topp(probs, _topp_target(top_p, probs.size(0), probs.device), None, False) # --------------------------------------------------------------------------- @@ -358,38 +189,47 @@ def _draw_part(probs_ptr, thr_ptr, psum_ptr, V, G, CHUNK, row_stride, BLOCK_SIZE @triton.jit -def _draw_scan(psum_ptr, choff_ptr, u_ptr, target_ptr, G, G_POW2: tl.constexpr): +def _draw_scan(psum_ptr, choff_ptr, u_ptr, target_ptr, last_ptr, G, G_POW2: tl.constexpr): row = tl.program_id(0) goff = tl.arange(0, G_POW2) gmask = goff < G ps = tl.load(psum_ptr + row * G + goff, mask=gmask, other=0.0) tl.store(choff_ptr + row * G + goff, tl.cumsum(ps, 0) - ps, mask=gmask) tl.store(target_ptr + row, tl.load(u_ptr + row) * tl.sum(ps, 0)) + tl.store(last_ptr + row, tl.max(tl.where(gmask & (ps > 0), goff, -1), 0)) @triton.autotune(configs=_SR_CFGS, key=["CHUNK"], **autotune_cache_kwargs) @triton.jit -def _draw_find(probs_ptr, thr_ptr, choff_ptr, target_ptr, out_ptr, V, G, CHUNK, row_stride, +def _draw_find(probs_ptr, thr_ptr, choff_ptr, target_ptr, psum_ptr, last_ptr, out_ptr, V, G, CHUNK, row_stride, BLOCK_SIZE: tl.constexpr): pid = tl.program_id(0) row = pid // G thr = tl.load(thr_ptr + row) target = tl.load(target_ptr + row) acc = tl.load(choff_ptr + pid) + incl = acc + tl.load(psum_ptr + pid) base = row * row_stride start = (pid % G) * CHUNK end = tl.minimum(start + CHUNK, V) + last_kept = start * 0 - 1 for s0 in tl.range(start, end, BLOCK_SIZE): offs = s0 + tl.arange(0, BLOCK_SIZE) mask = offs < end x = tl.load(probs_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) - wv = tl.where((x >= thr) & mask, x, 0.0) + kept = (x >= thr) & mask + wv = tl.where(kept, x, 0.0) cval = acc + tl.cumsum(wv, 0) idx = tl.where(cval > target, offs, V) blk_min = tl.min(idx, 0) if (blk_min < V) and (acc <= target): tl.store(out_ptr + row, blk_min) acc += tl.sum(wv, 0) + last_kept = tl.maximum(last_kept, tl.max(tl.where(kept, offs, -1), 0)) + # see _keep_tail: the CTA owning an fp rounding gap writes its last kept token + is_last_mass = tl.load(last_ptr + row) == pid % G + if (acc <= target) and (last_kept >= 0) and ((incl > target) or is_last_mass): + tl.store(out_ptr + row, last_kept) _UGEN = {} @@ -412,16 +252,19 @@ def _gen_u(B, device, seed, offset): def _draw(probs, thr, seed, offset): B, V = probs.shape dev = probs.device - G, CHUNK = _plan(B, V) + if B == 0: + return torch.empty(0, device=dev, dtype=torch.int32) + G, CHUNK = _plan(B, V, dev) grid = (B * G,) psum = torch.empty(B * G, device=dev, dtype=torch.float32) choff = torch.empty(B * G, device=dev, dtype=torch.float32) target = torch.empty(B, device=dev, dtype=torch.float32) - out = torch.empty(B, device=dev, dtype=torch.int32) + last = torch.empty(B, device=dev, dtype=torch.int32) + out = torch.zeros(B, device=dev, dtype=torch.int32) u = _gen_u(B, dev, seed, offset) _draw_part[grid](probs, thr, psum, V, G, CHUNK, probs.stride(0)) - _draw_scan[(B,)](psum, choff, u, target, G, _next_pow2(G)) - _draw_find[grid](probs, thr, choff, target, out, V, G, CHUNK, probs.stride(0)) + _draw_scan[(B,)](psum, choff, u, target, last, G, _next_pow2(G)) + _draw_find[grid](probs, thr, choff, target, psum, last, out, V, G, CHUNK, probs.stride(0)) return out @@ -442,155 +285,348 @@ def top_p_sampling_from_probs(probs, top_p, indices=None, deterministic=True, ge check_nan=False, seed=None, offset=None, return_valid=False): probs = probs.float() src = probs if indices is None else probs[indices].contiguous() - if isinstance(top_p, torch.Tensor): - target = top_p.float().to(src.device).contiguous() - else: - target = torch.full((src.size(0),), float(top_p), device=src.device, dtype=torch.float32) - thr = _search(src, target, True, _PR, _PBINS) - out = _draw(src, thr, seed, offset) + out = _topp(src, _topp_target(top_p, src.size(0), src.device), None, True, seed, offset) out = out.to(indices.dtype) if indices is not None else out return (out, torch.ones_like(out, dtype=torch.bool)) if return_valid else out # =========================================================================== -# top-k via Qrita outlier-gather + tiny-buffer refine (adapted from vLLM) -# Passes: rmax (full) + gather (full) + refine (tiny, buffer-only) + write (full) -# = 3 full-vocab passes. Keeps the multi-CTA vocab split so bs=1 uses the whole GPU. +# top-k: one cooperative kernel per call. Every CTA of a row histograms its column chunk +# over the fp32 bit pattern, the row's CTAs meet at a spin barrier, then each one redoes +# the tiny refine step so all of them hold the same bracket. Four rounds (exponent, then +# 8+8+7 mantissa bits) end on a single bit pattern, so thr is exactly the k-th largest +# prob. The same kernel then either renormalizes (DRAW=0) or draws a token (DRAW=1). # =========================================================================== -_FRAC = 0.05 # outlier gather pivot: keep probs >= rmax*FRAC -_CAP = 8192 # per-row candidate buffer capacity -_KBINS = 256 # bins per refine bracket iteration -_KR = 3 # refine iterations (256**3 ~ 1.7e7 resolution over [0, rmax]) +_KBINS = 256 +_INF_BITS = tl.constexpr(0x7F800000) +_FUSED_BLOCK = 2048 + + +def _topk_target(top_k, B, dev): + if isinstance(top_k, torch.Tensor): + return top_k.to(device=dev, dtype=torch.int32).contiguous() + return torch.full((B,), max(int(top_k), 1), device=dev, dtype=torch.int32) + + +@triton.jit +def _row_barrier(bar_ptr, need): + # the row's CTAs must be co-resident (cooperative launch), or a lone CTA (G == 1) passes at once. + # every warp's preceding atomics must be issued before thread 0 announces arrival + tl.debug_barrier() + tl.atomic_add(bar_ptr, 1) + n = tl.atomic_add(bar_ptr, 0) + while n < need: + n = tl.atomic_add(bar_ptr, 0) + + +@triton.jit +def _bits_round( + probs_ptr, base, start, end, hist_ptr, bar_ptr, target, lo, above, need, + S: tl.constexpr, WIDTH: tl.constexpr, BINS: tl.constexpr, BLOCK: tl.constexpr, +): + jj = tl.arange(0, BINS) + acc = tl.zeros([BINS], tl.int32) + for s0 in tl.range(start, end, BLOCK): + offs = s0 + tl.arange(0, BLOCK) + mask = offs < end + y = tl.load(probs_ptr + base + offs, mask=mask, other=-1.0).to(tl.float32).to(tl.int32, bitcast=True) + d = y - lo + if WIDTH == 0: + inrange = mask & (y >= lo) & (y <= _INF_BITS) + else: + inrange = mask & (y >= lo) & (d < WIDTH) & (y <= _INF_BITS) + # every out-of-bracket lane (padding included) lands in bin 0 and is subtracted back out + b = tl.where(inrange, d >> S, 0) + h = tl.histogram(b, BINS) + acc += h - tl.where(jj == 0, tl.sum((~inrange).to(tl.int32)), 0) + tl.atomic_add(hist_ptr + jj, acc) + _row_barrier(bar_ptr, need) + h = tl.load(hist_ptr + jj, cache_modifier=".cg") + prefix = tl.cumsum(h, 0) + total = tl.sum(h, 0) + ok = above + total - prefix + h >= target + j = tl.max(tl.where(ok, jj, -1)) + prefix_j = tl.sum(tl.where(jj <= j, h, 0)) + upd = j >= 0 + lo = tl.where(upd, lo + (j << S), lo) + above = tl.where(upd, above + total - prefix_j, above) + return lo, above -@triton.autotune(configs=_SR_CFGS, key=["CHUNK"], reset_to_zero=["cnt_ptr"], **autotune_cache_kwargs) @triton.jit -def _gather_pass( - probs_ptr, rmax_ptr, buf_ptr, cnt_ptr, V, G, CHUNK, row_stride, - FRAC, CAP: tl.constexpr, BLOCK_SIZE: tl.constexpr, +def _topk_fused( + probs_ptr, target_ptr, hist_ptr, bar_ptr, ksum_ptr, psum_ptr, u_ptr, out_ptr, tok_ptr, + V, G, CHUNK, row_stride, + DRAW: tl.constexpr, G_POW2: tl.constexpr, BINS: tl.constexpr, BLOCK: tl.constexpr, ): pid = tl.program_id(0) row = pid // G - thr0 = tl.load(rmax_ptr + row) * FRAC + cta = pid % G base = row * row_stride - bufbase = row * CAP - start = (pid % G) * CHUNK + start = cta * CHUNK end = tl.minimum(start + CHUNK, V) - for s0 in tl.range(start, end, BLOCK_SIZE): - offs = s0 + tl.arange(0, BLOCK_SIZE) + target = tl.maximum(tl.load(target_ptr + row), 1) + hrow = hist_ptr + row * 4 * BINS + brow = bar_ptr + row + lo = 0 + above = lo + lo, above = _bits_round(probs_ptr, base, start, end, hrow, brow, target, lo, above, G, 23, 0, BINS, BLOCK) + lo, above = _bits_round(probs_ptr, base, start, end, hrow + BINS, brow, target, lo, above, 2 * G, 15, 1 << 23, BINS, BLOCK) + lo, above = _bits_round(probs_ptr, base, start, end, hrow + 2 * BINS, brow, target, lo, above, 3 * G, 7, 1 << 15, BINS, BLOCK) + lo, above = _bits_round(probs_ptr, base, start, end, hrow + 3 * BINS, brow, target, lo, above, 4 * G, 0, 1 << 7, BINS, BLOCK) + _keep_tail(probs_ptr, base, start, end, pid, row, cta, lo.to(tl.float32, bitcast=True), brow, 5 * G, + ksum_ptr, psum_ptr, u_ptr, out_ptr, tok_ptr, V, G, DRAW, G_POW2, BLOCK) + + +@triton.jit +def _keep_tail( + probs_ptr, base, start, end, pid, row, cta, thr, bar_ptr, need, + ksum_ptr, psum_ptr, u_ptr, out_ptr, tok_ptr, V, G, + DRAW: tl.constexpr, G_POW2: tl.constexpr, BLOCK: tl.constexpr, +): + # Keep x >= thr over this chunk. This deliberately retains every boundary tie, + # matching flashinfer's top-k and top-p filtering semantics. + s = 0.0 + for s0 in tl.range(start, end, BLOCK): + offs = s0 + tl.arange(0, BLOCK) mask = offs < end x = tl.load(probs_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) - m = mask & (x >= thr0) - mi = m.to(tl.int32) - n = tl.sum(mi) - posbase = tl.atomic_add(cnt_ptr + row, n) - cpos = posbase + tl.cumsum(mi, 0) - 1 - wmask = m & (cpos < CAP) - tl.store(buf_ptr + bufbase + cpos, x, mask=wmask) + s += tl.sum(tl.where(mask & (x >= thr), x, 0.0), 0) + if DRAW: + tl.store(psum_ptr + pid, s) + _row_barrier(bar_ptr, need) + goff = tl.arange(0, G_POW2) + gmask = goff < G + ps = tl.load(psum_ptr + row * G + goff, mask=gmask, other=0.0, cache_modifier=".cg") + acc = tl.sum(tl.where(goff < cta, ps, 0.0), 0) + incl = tl.sum(tl.where(goff <= cta, ps, 0.0), 0) + tgt = tl.load(u_ptr + row) * tl.sum(ps, 0) + last_kept = start * 0 - 1 + for s0 in tl.range(start, end, BLOCK): + offs = s0 + tl.arange(0, BLOCK) + mask = offs < end + x = tl.load(probs_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) + kept = mask & (x >= thr) + wv = tl.where(kept, x, 0.0) + cval = acc + tl.cumsum(wv, 0) + idx = tl.where(cval > tgt, offs, V) + blk_min = tl.min(idx, 0) + if (blk_min < V) and (acc <= tgt): + tl.store(tok_ptr + row, blk_min) + acc += tl.sum(wv, 0) + last_kept = tl.maximum(last_kept, tl.max(tl.where(kept & (x > 0), offs, -1), 0)) + # fp rounding can leave tgt between this CTA's running sum and the next CTA's prefix, or past the total; + # the CTA that owns that gap (or the last one holding mass) writes its last kept token instead + is_last_mass = tl.sum(tl.where((goff > cta) & (ps > 0), 1, 0), 0) == 0 + if (acc <= tgt) and (last_kept >= 0) and ((incl > tgt) or is_last_mass): + tl.store(tok_ptr + row, last_kept) + else: + tl.atomic_add(ksum_ptr + row, s) + _row_barrier(bar_ptr, need) + inv = 1.0 / tl.atomic_add(ksum_ptr + row, 0.0) + for s0 in tl.range(start, end, BLOCK): + offs = s0 + tl.arange(0, BLOCK) + mask = offs < end + x = tl.load(probs_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) + tl.store(out_ptr + base + offs, tl.where(x >= thr, x * inv, 0.0), mask=mask) + + +_PMBINS = 256 @triton.jit -def _refine_topk( - buf_ptr, cnt_ptr, rmax_ptr, target_ptr, thr_ptr, ksum_ptr, - CAP: tl.constexpr, R: tl.constexpr, BINS: tl.constexpr, BLK: tl.constexpr, +def _pmass_round( + probs_ptr, base, start, end, priv_ptr, mass_ptr, bar_ptr, target, lo, above, need, + S: tl.constexpr, WIDTH: tl.constexpr, BINS: tl.constexpr, BLOCK: tl.constexpr, ): - # single-CTA-per-row refine on the tiny buffer -> (threshold, kept-sum): - # histogram-bracket the k-th largest, then sum the kept mass (all kept values - # are in the buffer since threshold >= gather pivot). - row = tl.program_id(0) - cnt = tl.load(cnt_ptr + row) - cnt = tl.minimum(cnt, CAP) - target = tl.load(target_ptr + row) - base = row * CAP + # top-p round over the bit pattern: per-bin MASS (exact up to fp32 atomic order) via scatter-add into this + # CTA's private buffer, then one reduction into the row buffer, so the bin holding the p crossing is known jj = tl.arange(0, BINS) - lo = 0.0 - hi = tl.load(rmax_ptr + row) * 1.0000001 - above = 0.0 - for _it in tl.static_range(R): - denom = tl.maximum(hi - lo, 1e-30) - w = denom / BINS - invw = BINS / denom - hc = tl.zeros([BINS], tl.int32) - for s0 in tl.range(0, cnt, BLK): - offs = s0 + tl.arange(0, BLK) - mask = offs < cnt - x = tl.load(buf_ptr + base + offs, mask=mask, other=-1.0) - inrange = mask & (x >= lo) & (x < hi) - b = ((x - lo) * invw).to(tl.int32) - b = tl.where(inrange, tl.maximum(0, tl.minimum(b, BINS - 1)), 0) - hcnt = tl.histogram(b, BINS) - noor = tl.sum((mask & (~inrange)).to(tl.int32)) - hc += hcnt - tl.where(jj == 0, noor, 0) - h = hc.to(tl.float32) - prefix = tl.cumsum(h, 0) - total = tl.sum(h, 0) - c_ge = above + total - prefix + h - ok = c_ge >= target - j = tl.max(tl.where(ok, jj, -1)) - prefix_j = tl.sum(tl.where(jj <= j, h, 0.0)) - upd = j >= 0 - new_lo = lo + j * w - new_hi = lo + (j + 1) * w - new_above = above + total - prefix_j - lo = tl.where(upd, new_lo, lo) - hi = tl.where(upd, new_hi, hi) - above = tl.where(upd, new_above, above) - thr = lo - # guard: if fewer finite candidates than k were gathered, keep everything. - if cnt < target: - thr = 0.0 - ks = 0.0 - for s0 in tl.range(0, cnt, BLK): - offs = s0 + tl.arange(0, BLK) - mask = offs < cnt - x = tl.load(buf_ptr + base + offs, mask=mask, other=0.0) - ks += tl.sum(tl.where(x >= thr, x, 0.0)) - tl.store(thr_ptr + row, thr) - tl.store(ksum_ptr + row, ks) + for s0 in tl.range(start, end, BLOCK): + offs = s0 + tl.arange(0, BLOCK) + mask = offs < end + x = tl.load(probs_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) + y = x.to(tl.int32, bitcast=True) + d = y - lo + if WIDTH == 0: + inrange = mask & (y >= lo) & (y <= _INF_BITS) + else: + inrange = mask & (y >= lo) & (d < WIDTH) & (y <= _INF_BITS) + tl.atomic_add(priv_ptr + tl.where(inrange, d >> S, 0), x, mask=inrange) + # every warp's scatter-adds must land before any thread reads the private bins back + tl.debug_barrier() + tl.atomic_add(mass_ptr + jj, tl.load(priv_ptr + jj)) + _row_barrier(bar_ptr, need) + m = tl.load(mass_ptr + jj, cache_modifier=".cg") + prefix = tl.cumsum(m, 0) + total = tl.sum(m, 0) + ok = above + total - prefix + m >= target + # p above the total mass (fp rounding at p = 1): keep the whole bracket + j = tl.maximum(tl.max(tl.where(ok, jj, -1)), 0) + prefix_j = tl.sum(tl.where(jj <= j, m, 0.0)) + return lo + (j << S), above + total - prefix_j -def _topk_target(top_k, B, dev): - if isinstance(top_k, torch.Tensor): - return top_k.float().to(dev).contiguous() - return torch.full((B,), float(int(top_k)), device=dev, dtype=torch.float32) +@triton.jit +def _topp_fused( + probs_ptr, tp_ptr, tk_ptr, hist_ptr, priv_ptr, mass_ptr, bar_ptr, ksumk_ptr, ksum_ptr, psum_ptr, u_ptr, + out_ptr, tok_ptr, V, G, CHUNK, row_stride, + TOPK: tl.constexpr, DRAW: tl.constexpr, G_POW2: tl.constexpr, KBINS: tl.constexpr, PBINS: tl.constexpr, + BLOCK: tl.constexpr, +): + # top-p, optionally after an exact top-k stage: the top-k threshold becomes the lower edge of the top-p + # bracket and the p target is scaled by the kept top-k mass, so no renormalized copy is ever written + pid = tl.program_id(0) + row = pid // G + cta = pid % G + base = row * row_stride + start = cta * CHUNK + end = tl.minimum(start + CHUNK, V) + brow = bar_ptr + row + lo = 0 + if TOPK: + tk = tl.maximum(tl.load(tk_ptr + row), 1) + hk = hist_ptr + row * 4 * KBINS + above_i = lo + lo, above_i = _bits_round(probs_ptr, base, start, end, hk, brow, tk, lo, above_i, G, 23, 0, KBINS, BLOCK) + lo, above_i = _bits_round(probs_ptr, base, start, end, hk + KBINS, brow, tk, lo, above_i, 2 * G, 15, 1 << 23, KBINS, BLOCK) + lo, above_i = _bits_round(probs_ptr, base, start, end, hk + 2 * KBINS, brow, tk, lo, above_i, 3 * G, 7, 1 << 15, KBINS, BLOCK) + lo, above_i = _bits_round(probs_ptr, base, start, end, hk + 3 * KBINS, brow, tk, lo, above_i, 4 * G, 0, 1 << 7, KBINS, BLOCK) + thr_k = lo.to(tl.float32, bitcast=True) + s = 0.0 + for s0 in tl.range(start, end, BLOCK): + offs = s0 + tl.arange(0, BLOCK) + mask = offs < end + x = tl.load(probs_ptr + base + offs, mask=mask, other=0.0).to(tl.float32) + s += tl.sum(tl.where(mask & (x >= thr_k), x, 0.0), 0) + tl.atomic_add(ksumk_ptr + row, s) + _row_barrier(brow, 5 * G) + target = tl.load(tp_ptr + row) * tl.atomic_add(ksumk_ptr + row, 0.0) + done = 5 + else: + target = tl.load(tp_ptr + row) + done = 0 + mp = mass_ptr + row * 4 * PBINS + pp = priv_ptr + pid * 4 * PBINS + above = 0.0 + lo, above = _pmass_round(probs_ptr, base, start, end, pp, mp, brow, target, lo, above, (done + 1) * G, + 23, 0, PBINS, BLOCK) + lo, above = _pmass_round(probs_ptr, base, start, end, pp + PBINS, mp + PBINS, brow, target, lo, above, + (done + 2) * G, 15, 1 << 23, PBINS, BLOCK) + lo, above = _pmass_round(probs_ptr, base, start, end, pp + 2 * PBINS, mp + 2 * PBINS, brow, target, lo, above, + (done + 3) * G, 7, 1 << 15, PBINS, BLOCK) + lo, above = _pmass_round(probs_ptr, base, start, end, pp + 3 * PBINS, mp + 3 * PBINS, brow, target, lo, above, + (done + 4) * G, 0, 1 << 7, PBINS, BLOCK) + _keep_tail(probs_ptr, base, start, end, pid, row, cta, lo.to(tl.float32, bitcast=True), brow, + (done + 5) * G, ksum_ptr, psum_ptr, u_ptr, out_ptr, tok_ptr, V, G, DRAW, G_POW2, BLOCK) + + +_COOPERATIVE_DISABLED = set() +_COOP_CTAS_PER_SM = 2 # the fused kernels use ~80 regs/thread at 8 warps; 4/SM fails the cooperative launch + + +def _fused_plan(B, V, device, force_single=False): + if force_single: + return 1, V + # the cooperative launch needs the whole grid co-resident, so cap B*G by an occupancy budget instead of _plan's one CTA per SM + g_by_sm = max(1, (_COOP_CTAS_PER_SM * _num_sm(device)) // B) + g_by_chunk = max(1, triton.cdiv(V, _MIN_CHUNK)) + G = min(g_by_sm, g_by_chunk) + return G, triton.cdiv(V, G) -def _topk_thr_ksum(probs, top_k): - """Return (threshold[B], kept_sum[B]) for a top-k keep: x >= threshold.""" +def _fused_launch(probs, kernel, tk, tp, draw, seed, offset, force_single=False): B, V = probs.shape dev = probs.device - G, CHUNK = _plan(B, V) - grid = (B * G,) - target = _topk_target(top_k, B, dev) - rmax = torch.zeros(B, device=dev, dtype=torch.float32) - _rmax_pass[grid](probs, rmax, V, G, CHUNK, probs.stride(0), BLOCK_SIZE=2048, num_warps=8) - buf = torch.empty(B * _CAP, device=dev, dtype=torch.float32) - cnt = torch.zeros(B, device=dev, dtype=torch.int32) - _gather_pass[grid](probs, rmax, buf, cnt, V, G, CHUNK, probs.stride(0), _FRAC, _CAP) - thr = torch.empty(B, device=dev, dtype=torch.float32) - ksum = torch.empty(B, device=dev, dtype=torch.float32) - _refine_topk[(B,)](buf, cnt, rmax, target, thr, ksum, _CAP, _KR, _KBINS, 2048) - return thr, ksum + G, CHUNK = _fused_plan(B, V, dev, force_single) + n_hist = 4 * _KBINS if (kernel is _topk_fused or tk is not None) else 0 + n_mass = 4 * _PMBINS if kernel is _topp_fused else 0 + # hist[B, n_hist] | mass[B, n_mass] | priv[B * G, n_mass] | bar/ksum/ksum_k/tok[B] + ws = torch.zeros(B * (n_hist + n_mass) + B * G * n_mass + 4 * B, device=dev, dtype=torch.int32) + hist = ws[:B * n_hist] + mass = ws[B * n_hist:B * (n_hist + n_mass)].view(torch.float32) + priv = ws[B * (n_hist + n_mass):B * (n_hist + n_mass) + B * G * n_mass].view(torch.float32) + tail = B * (n_hist + n_mass) + B * G * n_mass + bar = ws[tail:tail + B] + ksum = ws[tail + B:tail + 2 * B].view(torch.float32) + ksum_k = ws[tail + 2 * B:tail + 3 * B].view(torch.float32) + if draw: + psum = torch.empty(B * G, device=dev, dtype=torch.float32) + u = _gen_u(B, dev, seed, offset) + res = ws[tail + 3 * B:] + out, tok = probs, res + else: + psum, u = ksum, ksum + res = torch.empty_like(probs) + out, tok = res, bar + # a lone CTA per row in a single wave streams faster with more warps; with G > 1 the co-residency budget caps warps + wide = G == 1 and B <= _num_sm(dev) + common = dict(DRAW=draw, G_POW2=_next_pow2(G), BLOCK=8192 if wide else _FUSED_BLOCK, num_warps=32 if wide else 8, + launch_cooperative_grid=G > 1) + if kernel is _topk_fused: + _topk_fused[(B * G,)](probs, tk, hist, bar, ksum, psum, u, out, tok, V, G, CHUNK, probs.stride(0), + BINS=_KBINS, **common) + else: + _topp_fused[(B * G,)](probs, tp, tk if tk is not None else tp, hist, priv, mass, bar, ksum_k, ksum, psum, u, out, tok, + V, G, CHUNK, probs.stride(0), TOPK=tk is not None, KBINS=_KBINS, PBINS=_PMBINS, **common) + return res + + +def _cooperative_key(probs, kernel, tk, draw): + kind = "topk" if kernel is _topk_fused else "topk_topp" if tk is not None else "topp" + return probs.device, kind, draw + + +def _is_cooperative_launch_error(exc): + message = str(exc).lower() + return "cooperative" in message or "too many resources requested for launch" in message + + +def _exact_launch(probs, kernel, tk, tp, draw, seed, offset): + key = _cooperative_key(probs, kernel, tk, draw) + force_single = key in _COOPERATIVE_DISABLED + G, _ = _fused_plan(*probs.shape, probs.device, force_single) + try: + return _fused_launch(probs, kernel, tk, tp, draw, seed, offset, force_single) + except RuntimeError as exc: + if force_single or G == 1 or not _is_cooperative_launch_error(exc): + raise + _COOPERATIVE_DISABLED.add(key) + logger.warning("cooperative triton sampling unavailable on %s (%s); retrying with one CTA per row", + probs.device, exc) + return _fused_launch(probs, kernel, tk, tp, draw, seed, offset, force_single=True) + + +def _topk(probs, target, draw, seed=None, offset=None): + if probs.size(0) == 0: + return torch.empty(0, device=probs.device, dtype=torch.int32) if draw else probs.clone() + + return _exact_launch(probs, _topk_fused, target, None, draw, seed, offset) + + +def _topp(probs, tp, tk, draw, seed=None, offset=None): + if probs.size(0) == 0: + return torch.empty(0, device=probs.device, dtype=torch.int32) if draw else probs.clone() + + return _exact_launch(probs, _topp_fused, tk, tp, draw, seed, offset) + + +def _topp_target(top_p, B, dev): + if isinstance(top_p, torch.Tensor): + return top_p.float().to(dev).contiguous() + return torch.full((B,), float(top_p), device=dev, dtype=torch.float32) def top_k_renorm_probs(probs, top_k): probs = probs.float() - B, V = probs.shape - dev = probs.device - G, CHUNK = _plan(B, V) - grid = (B * G,) - thr, ksum = _topk_thr_ksum(probs, top_k) - out = torch.empty_like(probs) - _write_pass[grid](probs, out, thr, ksum, V, G, CHUNK, probs.stride(0)) - return out + return _topk(probs, _topk_target(top_k, probs.size(0), probs.device), False) def top_k_sampling_from_probs(probs, top_k, indices=None, deterministic=True, generator=None, check_nan=False, seed=None, offset=None, return_valid=False): probs = probs.float() src = probs if indices is None else probs[indices].contiguous() - r = top_k_renorm_probs(src, top_k) - out = _draw(r, _zeros_thr(src.size(0), src.device), seed, offset) + out = _topk(src, _topk_target(top_k, src.size(0), src.device), True, seed, offset) out = out.to(indices.dtype) if indices is not None else out return (out, torch.ones_like(out, dtype=torch.bool)) if return_valid else out @@ -601,13 +637,8 @@ def top_k_top_p_sampling_from_probs(probs, top_k, top_p, indices=None, return_valid=False): probs = probs.float() src = probs if indices is None else probs[indices].contiguous() - r = top_k_renorm_probs(src, top_k) - if isinstance(top_p, torch.Tensor): - target = top_p.float().to(src.device).contiguous() - else: - target = torch.full((src.size(0),), float(top_p), device=src.device, dtype=torch.float32) - thr = _search(r, target, True, _PR, _PBINS) - out = _draw(r, thr, seed, offset) + B = src.size(0) + out = _topp(src, _topp_target(top_p, B, src.device), _topk_target(top_k, B, src.device), True, seed, offset) out = out.to(indices.dtype) if indices is not None else out return (out, torch.ones_like(out, dtype=torch.bool)) if return_valid else out diff --git a/python/freetoken/kernel/utils.py b/python/freetoken/kernel/utils.py index 7a0164b59a..42f15a5b4b 100644 --- a/python/freetoken/kernel/utils.py +++ b/python/freetoken/kernel/utils.py @@ -30,7 +30,13 @@ def _cuda_cflags(extra: List[str]) -> List[str]: PTX→SASS JIT (driver-only, no CUDA toolkit). One top PTX suffices: the loader always JIT-forwards from the highest compatible PTX. When the env is unset (runtime JIT), this is a no-op and tvm-ffi targets only the local GPU.""" - flags = DEFAULT_CUDA_CFLAGS + extra + import torch + + flags = list(DEFAULT_CUDA_CFLAGS) + if torch.version.hip is not None: + # nvcc-only: hipcc/clang rejects it outright. + flags = [f for f in flags if f != "--expt-relaxed-constexpr"] + flags = flags + extra arch_list = os.getenv("TVM_FFI_CUDA_ARCH_LIST", "").split() if arch_list: def _rank(a: str) -> int: diff --git a/python/freetoken/kvcache/__init__.py b/python/freetoken/kvcache/__init__.py index 1bb352b41c..41b82f1d5d 100644 --- a/python/freetoken/kvcache/__init__.py +++ b/python/freetoken/kvcache/__init__.py @@ -40,7 +40,8 @@ def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]: from .mha_pool import MHAKVCache return MHAKVCache - types = {spec.attn_type for spec in specs_fn()} + specs = list(specs_fn()) + types = {spec.attn_type for spec in specs} if AttnType.DSV4 in types: from .dsv4_paged_pool import DSV4PagedKVCache @@ -50,6 +51,11 @@ def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]: return HybridSWAKVCache if AttnType.DSA in types: + # kpool-compressed indexer (glm5_next): shadow slab + tail rings. + if any(s.attn_type == AttnType.DSA and s.index_ratio > 1 for s in specs): + from .dsa_pool import KpoolDSAKVCache + + return KpoolDSAKVCache from .dsa_pool import DSAKVCache return DSAKVCache @@ -57,6 +63,10 @@ def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]: from .dsa_pool import MLAKVCache return MLAKVCache + if AttnType.QSA in types: + from .qsa_pool import QSAKVCache + + return QSAKVCache if AttnType.BSA in types: from .bsa_pool import BSAKVCache @@ -106,6 +116,7 @@ def create_kv_pool(config, num_pages: int, device: torch.device, dtype: torch.dt num_swa_tokens=num_swa_tokens, device=device, dtype=dtype, + num_req_slots=config.max_running_req + 1, # + 1 for the dummy request row ) @@ -116,6 +127,7 @@ def create_kvcache_pool( dtype: torch.dtype, device: torch.device, num_swa_tokens: int | None = None, + num_req_slots: int | None = None, ) -> BaseKVCachePool: if model_config.has_swa_attention: from .hybrid_swa_pool import HybridSWAKVCache @@ -132,36 +144,31 @@ def create_kvcache_pool( from .mha_pool import MHAKVCache - # Hybrid linear-attention models (e.g. Qwen3.5 GatedDeltaNet) only store paged KV - # for their full-attention layers; the linear layers keep a separate recurrent - # state. Back just those layers and remap their global ids to dense storage slots - # so we don't over-allocate slabs for the (majority) linear layers. + # Hybrid linear-attention models only store paged KV for their non-linear + # layers; the linear layers keep a separate recurrent state (LinearStatePool). + # The linear group emits no paged spec, so the remaining paged spec(s) drive + # the dispatch below; the paged pool backs JUST those layers via a global-id + # -> dense-slot remap (layer_ids) so the (majority) linear layers cost no + # slabs. layer_ids: tuple[int, ...] | None = None - num_kv_heads = model_config.num_kv_heads - head_dim = model_config.head_dim + kv_specs = [s for s in model_config.kv_cache_group_specs() if s.num_layers > 0] if model_config.has_linear_attention: - specs = [s for s in model_config.kv_cache_group_specs() if s.num_layers > 0] - assert len(specs) == 1, f"expected one paged-KV group, got {[s.name for s in specs]}" - spec = specs[0] - layer_ids = spec.layer_ids - num_kv_heads = spec.num_kv_heads - head_dim = spec.head_dim - - # Latent-KV MLA models declare it on their single full-attention group: they get - # the latent pool (one slab, V aliases K), plus the DSA index-key slab when the - # spec carries indexer dims. The same spec fields drive the KV cost model, so the - # factory and the budget can never disagree. - kv_specs = model_config.kv_cache_group_specs() - - # GQA block-sparse (MiniMax-M3): one full-attention group carrying the index dims - # with mla=False -> the MHA pool plus the index-key slab. The same spec fields - # drive the KV cost model, so the factory and the budget can never disagree. + assert len(kv_specs) == 1, ( + f"hybrid-linear models support one paged-KV group, got " + f"{[s.name for s in kv_specs]}" + ) + layer_ids = kv_specs[0].layer_ids + + # Latent-KV MLA / GQA block-sparse models declare their geometry on the single + # paged spec; the same spec fields drive the KV cost model, so the factory and + # the budget can never disagree. from freetoken.attention import AttnType as _AttnType if len(kv_specs) == 1 and kv_specs[0].attn_type == _AttnType.BSA: from .bsa_pool import BSAKVCache spec = kv_specs[0] + assert layer_ids is None, "hybrid-linear x BSA has no pool support yet" return BSAKVCache( num_kv_heads=spec.num_kv_heads, num_layers=model_config.num_layers, @@ -174,36 +181,77 @@ def create_kvcache_pool( num_index_layers=spec.num_index_layers, ) + # QSA (Qwen3.8-Flash-Next): the same GQA group, but it stores one index key per + # index_ratio tokens and adds per-request tiers sized by the concurrency, not the pages. + # layer_ids is mandatory here -- the model is hybrid-linear, and letting MHAKVCache back + # all num_layers would allocate K/V slabs for the GDN layers too. + if len(kv_specs) == 1 and kv_specs[0].attn_type == _AttnType.QSA: + from .qsa_pool import QSAKVCache + + spec = kv_specs[0] + if num_req_slots is None: + raise ValueError("QSA pools need num_req_slots (max_running_req + 1)") + return QSAKVCache( + num_kv_heads=spec.num_kv_heads, + num_layers=model_config.num_layers, + head_dim=spec.head_dim, + num_pages=num_pages, + page_size=page_size, + dtype=dtype, + device=device, + index_head_dim=spec.index_head_dim, + num_index_layers=spec.num_index_layers, + index_ratio=spec.index_ratio, + num_req_slots=num_req_slots, + layer_ids=spec.layer_ids, + ) + if len(kv_specs) == 1 and kv_specs[0].mla: - from .dsa_pool import DSAKVCache, MLAKVCache + from .dsa_pool import DSAKVCache, KpoolDSAKVCache, MLAKVCache spec = kv_specs[0] + # With a layer remap the pool allocates len(layer_ids) slabs; without one + # it backs every model layer (all-MLA models, GLM-5.2). + num_layers = model_config.num_layers if layer_ids is None else len(layer_ids) if spec.index_head_dim > 0 and spec.num_index_layers > 0: - return DSAKVCache( + common = dict( latent_dim=spec.head_dim, - num_layers=model_config.num_layers, + num_layers=num_layers, num_pages=num_pages, page_size=page_size, dtype=dtype, device=device, index_head_dim=spec.index_head_dim, num_index_layers=spec.num_index_layers, + layer_ids=layer_ids, ) + if spec.index_ratio > 1: + # kpool tail rings are keyed by Req.table_idx; + 1 covers the dummy request row. + if num_req_slots is None: + raise ValueError("kpool pools need num_req_slots (max_running_req + 1)") + return KpoolDSAKVCache( + **common, + index_ratio=spec.index_ratio, + num_req_slots=num_req_slots, + ) + return DSAKVCache(**common) return MLAKVCache( latent_dim=spec.head_dim, - num_layers=model_config.num_layers, + num_layers=num_layers, num_pages=num_pages, page_size=page_size, dtype=dtype, device=device, + layer_ids=layer_ids, ) + spec = kv_specs[0] if len(kv_specs) == 1 else None return MHAKVCache( - num_kv_heads=num_kv_heads, + num_kv_heads=spec.num_kv_heads if spec is not None else model_config.num_kv_heads, num_pages=num_pages, page_size=page_size, num_layers=model_config.num_layers, - head_dim=head_dim, + head_dim=spec.head_dim if spec is not None else model_config.head_dim, device=device, dtype=dtype, layer_ids=layer_ids, diff --git a/python/freetoken/kvcache/base.py b/python/freetoken/kvcache/base.py index ae8cf9ecb7..95669e8c81 100644 --- a/python/freetoken/kvcache/base.py +++ b/python/freetoken/kvcache/base.py @@ -21,7 +21,10 @@ def spec_kv_bytes_per_token(spec, config) -> int: x layers, plus the bf16 DSA index-key slab when the spec carries indexer dims. Pure per-spec arithmetic -- pool families compose it over THEIR OWN groups; no family branching here. (2 bytes/elem == the torch.bfloat16 dsa_pool.DSAKVCache._alloc - hardcodes; keep the two in lockstep if the slab dtype ever changes.)""" + hardcodes; keep the two in lockstep if the slab dtype ever changes.) + + ``index_ratio`` > 1 (QSA) stores one index key per token group, not per token; that slab's + ring and scratch rows are fixed-size and priced in QSAKVCache.kv_cost instead.""" per_token = ( (1 if spec.mla else 2) # MLA latent groups store one slab (V aliases K) * spec.head_dim @@ -29,7 +32,7 @@ def spec_kv_bytes_per_token(spec, config) -> int: * config.dtype.itemsize * spec.num_layers ) - return per_token + spec.index_head_dim * spec.num_index_layers * 2 + return per_token + spec.index_head_dim * spec.num_index_layers * 2 // spec.index_ratio class BaseKVCachePool(ABC): diff --git a/python/freetoken/kvcache/dsa_pool.py b/python/freetoken/kvcache/dsa_pool.py index e6a51ac936..13225cec06 100644 --- a/python/freetoken/kvcache/dsa_pool.py +++ b/python/freetoken/kvcache/dsa_pool.py @@ -5,9 +5,10 @@ separate V (``v_cache`` aliases ``k_cache``, same convention as dsv4_paged_pool's single-latent tiers). ``DSAKVCache`` extends it with the DeepSeek-Sparse-Attention index-key slab: one ``index_head_dim``-wide bf16 row per token per full-indexer -layer, addressed by the SAME physical rows as the latent slab (page_size == 1), and -``rebuild`` resizes BOTH slabs atomically so the allocator can never hand out a slot -one slab has and the other lacks. +layer, addressed by the SAME physical rows as the latent slab (GLM-5.2, page 1; +``KpoolDSAKVCache`` overrides the geometry to a 1/ratio shadow). ``rebuild`` +resizes ALL slabs atomically so the allocator can never hand out a slot one slab +has and the other lacks. Storage lives here -- not in the attention backend -- so the engine's rebuild path (``MHAKVCache.rebuild``-shaped: fresh allocation, object identity preserved, views @@ -27,6 +28,10 @@ class MLAKVCache(BaseKVCachePool): The leading singleton keeps the buffer shape-compatible with MHAKVCache's (tokens = shape[2] * shape[3]). + + ``layer_ids`` backs only a SUBSET of the model's layers (hybrid linear x + MLA/DSA) while callers keep addressing by GLOBAL layer id -- the same remap + contract as MHAKVCache; None keeps the identity mapping. """ def __init__( @@ -37,14 +42,23 @@ def __init__( page_size: int, dtype: torch.dtype, device: torch.device, + layer_ids: "tuple[int, ...] | None" = None, ) -> None: self._latent_dim = latent_dim - self._num_layers = num_layers + if layer_ids is None: + self._num_layers = num_layers + self._layer_index: dict[int, int] | None = None + else: + self._num_layers = len(layer_ids) + self._layer_index = {int(g): i for i, g in enumerate(layer_ids)} self._page_size = page_size self._dtype = dtype self._device = device self._alloc(num_pages) + def _local_layer(self, layer_id: int) -> int: + return layer_id if self._layer_index is None else self._layer_index[layer_id] + def _alloc(self, num_pages: int) -> None: self._num_pages = num_pages self._kv_buffer = torch.empty( @@ -53,10 +67,12 @@ def _alloc(self, num_pages: int) -> None: dtype=self._dtype, ) - # -- views ------------------------------------------------------------------ + # -- views (addressed by GLOBAL layer id; remapped when layer_ids was given) -- def k_cache(self, layer_id: int) -> torch.Tensor: """Paged latent view ``[num_pages, page_size, latent_dim]``.""" - return self._kv_buffer[0, layer_id].view(self._num_pages, self._page_size, -1) + return self._kv_buffer[0, self._local_layer(layer_id)].view( + self._num_pages, self._page_size, -1 + ) def v_cache(self, layer_id: int) -> torch.Tensor: # MLA: K == V (single latent); same buffer, dsv4_paged_pool precedent. @@ -64,7 +80,7 @@ def v_cache(self, layer_id: int) -> torch.Tensor: def latent_rows(self, layer_id: int) -> torch.Tensor: """Row-flat latent view ``[num_pages * page_size, latent_dim]``.""" - return self._kv_buffer[0, layer_id].view(-1, self._latent_dim) + return self._kv_buffer[0, self._local_layer(layer_id)].view(-1, self._latent_dim) # -- writes ----------------------------------------------------------------- def store_kv( @@ -77,8 +93,8 @@ def store_kv( """Scatter this forward's latent rows: ``c_kv`` [T, kv_lora_rank] and ``k_rope`` [T, qk_rope_head_dim] land in the row's two halves. - v0: two narrow ``index_put_`` scatters. TODO: generalize kernel/csrc - store.cu to a two-width fused store and route this through it. + Two narrow ``index_put_`` scatters. TODO: fuse into kernel/csrc + store.cu (two-width store). """ rows = self.latent_rows(layer_id) split = rows.shape[1] - k_rope.shape[-1] @@ -141,20 +157,29 @@ def __init__( device: torch.device, index_head_dim: int, num_index_layers: int, + layer_ids: "tuple[int, ...] | None" = None, ) -> None: self._index_head_dim = index_head_dim self._num_index_layers = num_index_layers - super().__init__(latent_dim, num_layers, num_pages, page_size, dtype, device) + super().__init__( + latent_dim, num_layers, num_pages, page_size, dtype, device, + layer_ids=layer_ids, + ) + + def _index_rows(self, num_pages: int) -> int: + """Index-slab row count: one row per token (KpoolDSAKVCache overrides to the + 1/ratio shadow + scratch layout).""" + return num_pages * self._page_size def _alloc(self, num_pages: int) -> None: - # Both slabs in one allocation step: rebuild can never leave the pool with a - # grown latent slab and a stale index slab (the OOB class this type exists for). + # Both slabs in one allocation step so rebuild can never leave one grown + # and the other stale. super()._alloc(num_pages) # bf16 == the 2 bytes/token/layer the KV cost model budgets for this slab # (cache_status._kv_cost_model); keep the two in lockstep. self._index_k_buffer = torch.zeros( self._num_index_layers, - num_pages * self._page_size, + self._index_rows(num_pages), self._index_head_dim, dtype=torch.bfloat16, device=self._device, @@ -180,4 +205,56 @@ def store_index_k(self, k: torch.Tensor, out_loc: torch.Tensor, slot: int) -> No self._index_k_buffer[slot][out_loc] = k -__all__ = ["MLAKVCache", "DSAKVCache"] +class KpoolDSAKVCache(DSAKVCache): + """DSAKVCache with the index slab as a 1/ratio SHADOW of the KV pages, plus + per-request scratch rows and tail rings. + + A pooled entry lives at ``token_slot // index_ratio`` (``page_size % + index_ratio == 0``), so compressed rows follow page sharing/eviction for + free. Rows whose pool does not close write to the request's scratch row + instead (scoring never reads it). The tail rings hold the in-progress + pool's raw K + gate at ``pos % index_ratio`` for pools straddling two + forwards; never cleared -- prefix-cache resume points are page-aligned, + so a stale ring is never read. + """ + + def __init__(self, *args, num_req_slots: int, index_ratio: int, **kwargs) -> None: + self._num_req_slots = num_req_slots + self._index_ratio = index_ratio + super().__init__(*args, **kwargs) + assert self._page_size % index_ratio == 0, ( + f"kpool needs page_size ({self._page_size}) divisible by " + f"index_ratio ({index_ratio})" + ) + + def _index_rows(self, num_pages: int) -> int: + # 1/ratio shadow of every token slot + one scratch row per request slot. + return num_pages * self._page_size // self._index_ratio + self._num_req_slots + + @property + def cmp_scratch_base(self) -> int: + """First scratch row (== shadow row count); request ``table_idx`` offsets it.""" + return self._num_pages * self._page_size // self._index_ratio + + def _alloc(self, num_pages: int) -> None: + super()._alloc(num_pages) + self._tail_k = torch.zeros( + self._num_index_layers, self._num_req_slots, self._index_ratio, + self._index_head_dim, dtype=torch.bfloat16, device=self._device, + ) + self._tail_gate = torch.zeros_like(self._tail_k) + + def rebuild(self, num_pages: int) -> None: + self._tail_k = None + self._tail_gate = None + super().rebuild(num_pages) + + def tail_k(self, slot: int) -> torch.Tensor: + """Tail raw keys for an indexer layer slot: ``[num_req_slots, ratio, head_dim]``.""" + return self._tail_k[slot] + + def tail_gate(self, slot: int) -> torch.Tensor: + return self._tail_gate[slot] + + +__all__ = ["MLAKVCache", "DSAKVCache", "KpoolDSAKVCache"] diff --git a/python/freetoken/kvcache/linear_state_pool.py b/python/freetoken/kvcache/linear_state_pool.py index 2b3b013835..ff7539410c 100644 --- a/python/freetoken/kvcache/linear_state_pool.py +++ b/python/freetoken/kvcache/linear_state_pool.py @@ -1,9 +1,11 @@ from __future__ import annotations +import math + import torch from freetoken.distributed import get_tp_info from freetoken.env import ENV -from freetoken.models.config import LinearGatedDeltaGroupConfig +from freetoken.models.config import LinearGatedDeltaGroupConfig, SlotStateSpec from freetoken.utils import div_even _SSM_DTYPES = { @@ -35,6 +37,11 @@ class LinearStatePool: Indexed by ``Req.table_idx`` (0..max_running_req), the same per-request slot the page table uses, so the scheduler's existing admit/free of ``table_idx`` covers the state's lifetime. One fixed slot per running request; no paging, no eviction. + + A model can declare extra per-request tensors on the same slots through + ``ModelConfig.slot_states`` (see ``SlotStateSpec``); they advance, snapshot, COW and + rebuild with the GDN state and are read back through ``slot_state(name, layer_id)``. + Consumers must re-read them each forward: ``rebuild`` replaces the tensors. """ def __init__( @@ -44,6 +51,7 @@ def __init__( dtype: torch.dtype, device: torch.device, tp_size: int | None = None, + slot_states: tuple[SlotStateSpec, ...] = (), ) -> None: if tp_size is None: tp_size = get_tp_info().size @@ -70,6 +78,16 @@ def __init__( ) self._local_index = {layer_id: i for i, layer_id in enumerate(group.layer_ids)} + self._slot_specs = tuple(slot_states) + names = [spec.name for spec in self._slot_specs] + if len(set(names)) != len(names): + raise ValueError(f"duplicate slot_state names: {names}") + self._state_layer_index = { + spec.name: {lid: i for i, lid in enumerate(spec.layer_ids)} + for spec in self._slot_specs + } + self.slot_states: dict[str, torch.Tensor] = self._alloc_slot_states(num_slots) + # Free-list allocator over slots 1..num_slots-1 (slot 0 reserved as a padding sink, # sglang MambaPool convention). Live working slots, ping-pong track slots, and # radix-tree-donated snapshots are all drawn from this single free-list, so memory @@ -77,6 +95,30 @@ def __init__( self.padding_slot = 0 self._free_slots: list[int] = list(range(1, num_slots)) + def _alloc_slot_states(self, num_slots: int) -> dict[str, torch.Tensor]: + return { + spec.name: torch.full( + (max(1, len(spec.layer_ids)), num_slots, *spec.shape), + spec.fill_value, + dtype=spec.dtype if spec.dtype is not None else self._conv_dtype, + device=self._device, + ) + for spec in self._slot_specs + } + + def has_slot_state(self, name: str) -> bool: + return name in self.slot_states + + def slot_state(self, name: str, layer_id: int | None = None) -> torch.Tensor: + """One declared sibling state, ``[num_slots, *shape]``; ``layer_id`` picks the layer row.""" + t = self.slot_states[name] + if layer_id is None: + assert not self._state_layer_index[name], ( + f"slot_state {name!r} is per-layer, pass layer_id" + ) + return t[0] + return t[self._state_layer_index[name][layer_id]] + @property def num_free_slots(self) -> int: return len(self._free_slots) @@ -110,6 +152,7 @@ def rebuild(self, num_slots: int) -> None: device = self._device self.conv_states = None self.recurrent_states = None + self.slot_states = {} if device.type == "cuda": torch.cuda.synchronize(device) torch.cuda.empty_cache() @@ -121,6 +164,7 @@ def rebuild(self, num_slots: int) -> None: dtype=rec_dtype, device=device, ) + self.slot_states = self._alloc_slot_states(num_slots) self._num_slots = num_slots self._free_slots = list(range(1, num_slots)) @@ -138,12 +182,16 @@ def clear_slots(self, slots) -> None: slots = torch.as_tensor(slots, dtype=torch.long, device=self._device) self.conv_states[:, slots] = 0 self.recurrent_states[:, slots] = 0 + for spec in self._slot_specs: + self.slot_states[spec.name][:, slots] = spec.fill_value def copy_from(self, src: int, dst: int) -> None: """Copy a whole-sequence snapshot (conv + recurrent, all layers) from slot ``src`` to ``dst``. Used for COW-on-restore (donated snapshot -> fresh live slot).""" self.conv_states[:, dst].copy_(self.conv_states[:, src]) self.recurrent_states[:, dst].copy_(self.recurrent_states[:, src]) + for t in self.slot_states.values(): + t[:, dst].copy_(t[:, src]) def is_linear_layer(self, layer_id: int) -> bool: return layer_id in self._local_index @@ -161,6 +209,8 @@ def reset(self, table_idx: int) -> None: """Zero a slot across all linear layers (new request takes this table_idx).""" self.conv_states[:, table_idx].zero_() self.recurrent_states[:, table_idx].zero_() + for spec in self._slot_specs: + self.slot_states[spec.name][:, table_idx] = spec.fill_value @property def num_linear_layers(self) -> int: @@ -180,6 +230,8 @@ def bytes_per_slot(self) -> int: self.conv_states[:, 0].numel() * self.conv_states.element_size() + self.recurrent_states[:, 0].numel() * self.recurrent_states.element_size() ) + for t in self.slot_states.values(): + per += t[:, 0].numel() * t.element_size() return int(per) @@ -187,15 +239,22 @@ def linear_state_bytes_per_req( group: LinearGatedDeltaGroupConfig, tp_size: int, dtype: torch.dtype, + slot_states: tuple[SlotStateSpec, ...] = (), ) -> int: - """Linear-state bytes for one request across all linear layers (TP-local).""" + """Linear-state bytes for one request across all linear layers (TP-local), plus any + declared slot_states.""" n_layers, local_conv_dim, local_v_heads = _linear_local_dims(group, tp_size) conv_elems = local_conv_dim * (group.conv_kernel_dim - 1) rec_elems = local_v_heads * group.key_head_dim * group.value_head_dim conv_bytes = conv_elems * dtype.itemsize # conv state in model dtype rec_bytes = rec_elems * ssm_state_dtype().itemsize # recurrent state (default fp32) - return int(n_layers * (conv_bytes + rec_bytes)) + total = n_layers * (conv_bytes + rec_bytes) + + for spec in slot_states: + item = (spec.dtype if spec.dtype is not None else dtype).itemsize + total += max(1, len(spec.layer_ids)) * math.prod(spec.shape) * item + return int(total) __all__ = ["LinearStatePool", "linear_state_bytes_per_req"] @@ -206,10 +265,16 @@ def state_pool_bytes(config, num_slots: int | None = None) -> int: slot count). The engine adds this to the KV family's fixed cost when budgeting -- the state pool is a sibling pool, not a KV tier.""" linear_group = config.model_config.linear_attention_group() + slot_states = getattr(config.model_config, "slot_states", ()) if linear_group is None: + if slot_states: + raise ValueError("slot_states ride the linear-state slots; model has no linear group") return 0 slots = num_slots if num_slots is not None else _linear_pool_num_slots(config) - return linear_state_bytes_per_req(linear_group, config.tp_info.size, config.dtype) * slots + per_req = linear_state_bytes_per_req( + linear_group, config.tp_info.size, config.dtype, slot_states + ) + return per_req * slots def _linear_pool_num_slots(config) -> int: diff --git a/python/freetoken/kvcache/qsa_pool.py b/python/freetoken/kvcache/qsa_pool.py new file mode 100644 index 0000000000..fddcdbd35d --- /dev/null +++ b/python/freetoken/kvcache/qsa_pool.py @@ -0,0 +1,212 @@ +"""QSA compressed-block sparse KV pool: paged GQA K/V + compressed index keys + pending ring. + +Qwen3.8-Flash-Next scores whole ``index_ratio``-token groups instead of single tokens, so +its indexer slab holds ONE compressed key row per group, addressed by ``slot // +index_ratio``. Because ``page_size % index_ratio == 0``, a group's tokens always live in one +page at consecutive slots, which makes that division well-defined: the compressed rows are a +1/ratio shadow of the K/V pages and follow page sharing and eviction for free -- no +allocator, no free, no clear (SGLang qsa_kv_pool / vLLM compressed-region precedent). + +Two tiers ride alongside the shadow slab and are NOT per-token: +- ``pending_ring``: the last ``ring_capacity`` pre-RoPE index keys of each running request (sized by ``ring_capacity_for``), indexed by ``Req.table_idx``. A group that straddles two forwards (chunked prefill, and + every decode step) reads its already-consumed members from here. Never cleared: a new + tenant of a table_idx starts at a group boundary (cached_len is 0 or a page multiple), so + its first closing group takes every member from its own forward. +- scratch rows at ``cmp_scratch_base``: one row per request slot, the write target for rows + whose group does not close in this forward, so the compress kernel scatters unconditionally + with no negative index and no cross-row conflict (DSV4 precedent). + +The slab is amortized into the per-token KV price (``unit_bytes``); the ring and scratch are +fixed and priced through ``kv_cost``'s ``fixed_cache_size``. +""" + +from __future__ import annotations + +import math +from typing import Sequence + +import torch + +from .mha_pool import MHAKVCache + +# The index tiers are always 2-byte (compute dtype); spec_kv_bytes_per_token budgets the same. +_INDEX_DTYPE_BYTES = 2 + + +class QSAKVCache(MHAKVCache): + """MHA paged pool + the compressed index-key slab + the per-request pending ring. + + ``cmp_k_cache(slot)`` is row-flat ``[num_pages * page_size // index_ratio + num_req_slots, + index_head_dim]``: row ``r < cmp_scratch_base`` holds the compressed key of the token group + whose K/V slots are ``[r * index_ratio, (r + 1) * index_ratio)``, and the rows from + ``cmp_scratch_base`` on are the per-request-slot scratch sinks. ``slot`` is the sparse + layer's order in the attention backend, same convention as BSAKVCache/DSAKVCache. + """ + + @classmethod + def ring_capacity_for(cls, index_ratio: int, num_speculative_tokens: int = 0) -> int: + """Ring depth: one row per pending position, keyed ``position % capacity``; spec decode widens by the draft depth (vLLM sizing).""" + return index_ratio * math.ceil((index_ratio + num_speculative_tokens) / index_ratio) + + def __init__( + self, + num_kv_heads: int, + num_layers: int, + head_dim: int, + num_pages: int, + page_size: int, + dtype: torch.dtype, + device: torch.device, + index_head_dim: int, + num_index_layers: int, + index_ratio: int, + num_req_slots: int, + ring_capacity: int | None = None, + layer_ids: Sequence[int] | None = None, + ) -> None: + if index_ratio < 1 or page_size % index_ratio != 0: + # slot // index_ratio only names one group when a group never straddles a page. + raise ValueError( + f"QSA needs page_size ({page_size}) divisible by index_ratio ({index_ratio})" + ) + if ring_capacity is None: + ring_capacity = self.ring_capacity_for(index_ratio) + if ring_capacity < index_ratio: + # A closing group reads up to index_ratio - 1 past members plus this forward's. + raise ValueError( + f"QSA needs ring_capacity ({ring_capacity}) >= index_ratio ({index_ratio})" + ) + # Index keys ride the compute dtype (the model's index_k is engine-dtype). The KV cost + # model budgets 2 bytes per token per index layer for the slab + # (base.spec_kv_bytes_per_token); keep the two in lockstep. + assert dtype.itemsize == _INDEX_DTYPE_BYTES, ( + f"QSA index slab budgets 2 bytes/token (spec_kv_bytes_per_token); got {dtype}" + ) + self._index_head_dim = index_head_dim + self._num_index_layers = num_index_layers + self._index_ratio = index_ratio + self._num_req_slots = num_req_slots + self._ring_capacity = ring_capacity + self._index_dtype = dtype + self._page_size = page_size + super().__init__( + num_kv_heads=num_kv_heads, + num_layers=num_layers, + head_dim=head_dim, + num_pages=num_pages, + page_size=page_size, + dtype=dtype, + device=device, + layer_ids=layer_ids, + ) + self._zero_kv_slabs() + self._alloc_index_tiers(num_pages) + + def _zero_kv_slabs(self) -> None: + # Defense-in-depth: the attend kernels pos-mask every K/V load (the real fix for + # torch.empty's recycled NaN/Inf bit patterns), but a zeroed slab keeps any future + # unmasked read finite instead of model-poisoning. One memset per (re)allocation. + self._kv_buffer.zero_() + + def _alloc_index_tiers(self, num_pages: int) -> None: + # ZERO-initialized: the score kernel reads whole rows of blocks unmasked and relies on + # never-written tail rows dotting to a finite 0. Written rows are never cleared again, + # so the kernel must clamp visible blocks to kvlen // index_ratio. + self._cmp_scratch_base = num_pages * self._page_size // self._index_ratio + self._cmp_k_buffer = torch.zeros( + self._num_index_layers, + self._cmp_scratch_base + self._num_req_slots, + self._index_head_dim, + dtype=self._index_dtype, + device=self._device, + ) + self._pending_ring = torch.zeros( + self._num_req_slots, + self._num_index_layers, + self._ring_capacity, + self._index_head_dim, + dtype=self._index_dtype, + device=self._device, + ) + + def rebuild(self, num_pages: int) -> None: + # Free the index tiers BEFORE the K/V realloc (super().rebuild frees + syncs + + # empty_cache), then re-derive them at the new page count. If the index alloc itself + # fails (OOM), null the K/V slab too and re-raise: a pool with a grown K/V slab and no + # index slab would mis-serve silently. Rebuild is idle-only, so zeroing the ring here + # cannot drop a live request's pending members. + self._cmp_k_buffer = None + self._pending_ring = None + super().rebuild(num_pages) + self._zero_kv_slabs() + try: + self._alloc_index_tiers(num_pages) + except Exception: + self._kv_buffer = None + self._k_buffer = None + self._v_buffer = None + raise + + @classmethod + def kv_cost(cls, config) -> tuple[int, int, int, int]: + from .base import spec_kv_bytes_per_token + from freetoken.attention import AttnType + + num_req_slots = config.max_running_req + 1 + per_token = 0 + fixed = 0 + for spec in config.model_config.kv_cache_group_specs(): + if spec.is_swa: + continue + per_token += spec_kv_bytes_per_token(spec, config) + if spec.attn_type is AttnType.QSA: + # One index-key row = all index layers at one position. + row = spec.index_head_dim * spec.num_index_layers * _INDEX_DTYPE_BYTES + fixed += num_req_slots * row * (cls.ring_capacity_for(spec.index_ratio) + 1) + return per_token * config.page_size, fixed, config.page_size, 0 + + def unit_bytes(self) -> tuple[int, int]: + # Only the shadow slab scales with pages, and only its non-scratch rows; the ring and + # the scratch rows are the fixed term kv_cost reports separately. + kv, swa = super().unit_bytes() + tokens = int(self._kv_buffer.shape[2]) * int(self._kv_buffer.shape[3]) + slab = ( + self._num_index_layers + * self._cmp_scratch_base + * self._index_head_dim + * self._index_dtype.itemsize + ) + return kv + slab // tokens, swa + + def cmp_k_cache(self, slot: int) -> torch.Tensor: + """Compressed index keys of one sparse layer: ``[rows, index_head_dim]``.""" + return self._cmp_k_buffer[slot] + + def pending_ring(self, slot: int) -> torch.Tensor: + """One sparse layer's pending ring: ``[num_req_slots, ring_capacity, index_head_dim]``.""" + return self._pending_ring[:, slot] + + @property + def cmp_scratch_base(self) -> int: + """First scratch row of ``cmp_k_cache``; row ``cmp_scratch_base + table_idx`` sinks a + forward whose group does not close.""" + return self._cmp_scratch_base + + @property + def index_ratio(self) -> int: + return self._index_ratio + + @property + def index_head_dim(self) -> int: + return self._index_head_dim + + @property + def ring_capacity(self) -> int: + return self._ring_capacity + + @property + def num_req_slots(self) -> int: + return self._num_req_slots + + +__all__ = ["QSAKVCache"] diff --git a/python/freetoken/layers/__init__.py b/python/freetoken/layers/__init__.py index 2dbd63732b..31c19bf9b8 100644 --- a/python/freetoken/layers/__init__.py +++ b/python/freetoken/layers/__init__.py @@ -1,4 +1,10 @@ -from .activation import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul, swigluoai_and_mul +from .activation import ( + gelu_and_mul, + gelu_tanh_and_mul, + silu_and_mul, + swiglu_clamp_and_mul, + swigluoai_and_mul, +) from .base import BaseOP, OPList, StateLessOP from .embedding import ParallelLMHead, VocabParallelEmbedding from .linear import ( @@ -23,6 +29,7 @@ "gelu_and_mul", "gelu_tanh_and_mul", "swigluoai_and_mul", + "swiglu_clamp_and_mul", "BaseOP", "StateLessOP", "OPList", diff --git a/python/freetoken/layers/activation.py b/python/freetoken/layers/activation.py index 93602b6c5d..ae1d9881e6 100644 --- a/python/freetoken/layers/activation.py +++ b/python/freetoken/layers/activation.py @@ -56,4 +56,19 @@ def swigluoai_and_mul( return swigluoai_and_mul(x, out=out, alpha=alpha, limit=limit) -__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul", "swigluoai_and_mul"] +def swiglu_clamp_and_mul( + x, out=None, *, alpha: float = 1.0, limit: float = 10.0 +): + """GLM-5.3 clamped SwiGLU over UNINTERLEAVED halves: ``clamp(gate, max=limit) * sigmoid(alpha * gate) * clamp(up, +-limit)``.""" + from freetoken.kernel.triton.activation import swiglu_clamp_and_mul + + return swiglu_clamp_and_mul(x, out=out, alpha=alpha, limit=limit) + + +__all__ = [ + "silu_and_mul", + "gelu_and_mul", + "gelu_tanh_and_mul", + "swigluoai_and_mul", + "swiglu_clamp_and_mul", +] diff --git a/python/freetoken/layers/gguf.py b/python/freetoken/layers/gguf.py index ac49b1a5be..511ec695ee 100644 --- a/python/freetoken/layers/gguf.py +++ b/python/freetoken/layers/gguf.py @@ -22,6 +22,7 @@ GGML_F32, GGML_NAME, GGML_Q4_0, + GGML_Q4_K, GGML_Q6_K, GGML_Q8_0, row_bytes, @@ -32,9 +33,9 @@ # ggml type groups for kernel dispatch (subset we build kernels for). _UNQUANTIZED = {GGML_F32, GGML_F16, GGML_BF16} # standard + k-quants: both an MMVQ (small-batch GEMV) and MMQ (large-batch) kernel exist. -_MMVQ = {GGML_Q4_0, GGML_Q8_0, GGML_Q6_K} -_MMQ = {GGML_Q4_0, GGML_Q8_0, GGML_Q6_K} -_DEQUANT = {GGML_Q4_0, GGML_Q8_0, GGML_Q6_K} +_MMVQ = {GGML_Q4_0, GGML_Q4_K, GGML_Q8_0, GGML_Q6_K} +_MMQ = {GGML_Q4_0, GGML_Q4_K, GGML_Q8_0, GGML_Q6_K} +_DEQUANT = {GGML_Q4_0, GGML_Q4_K, GGML_Q8_0, GGML_Q6_K} # Below this token count, the MMVQ GEMV kernel wins (matches vLLM's heuristic). _MMVQ_SAFE = 6 diff --git a/python/freetoken/layers/mhc.py b/python/freetoken/layers/mhc.py new file mode 100644 index 0000000000..1cd6459d7b --- /dev/null +++ b/python/freetoken/layers/mhc.py @@ -0,0 +1,150 @@ +"""mHC -- Manifold-Constrained Hyper-Connections (GLM-5.3-Flash; arXiv 2512.24880). + +The residual stream is widened to ``hc_mult`` (n) parallel streams. Around every +sublayer the streams are mixed by three learned, token-dependent maps computed +from ONE fp32 GEMM over the flattened streams (``fn [2n+n^2, n*hidden]``), +RMS-normalized over the full ``n*hidden`` vector and split into: + +* ``pre_mix [n]`` sigmoid gates: the sublayer input is ``sum_i pre_i * res_i`` +* ``post_mix [n]`` sigmoid*mult gates: how much sublayer output enters each stream +* ``comb_mix [n, n]`` softmax + Sinkhorn-projected (approximately doubly-stochastic) + stream-mixing matrix -- the "manifold constraint": mixing + neither amplifies nor loses residual mass. + +``mhc_post`` then rebuilds the streams: ``out_j = sum_i comb_ij * res_i + post_j * x``. + +Semantics match vLLM's reference (``model_executor/kernels/mhc/torch.py``, +PR #53906) bit-for-bit in fp32; the per-layer weights are ``hc_{attn,ffn}_fn`` / +``_scale`` / ``_base`` from the checkpoint. This torch implementation is the +correctness baseline; the fused triton kernel (``kernel/triton/mhc.py``, +dispatched by ``mhc_fused_post_pre`` below) replaces it on CUDA and is +validated against this file. +""" + +from __future__ import annotations + +import torch + + +def hc_expand(x: torch.Tensor, n: int) -> torch.Tensor: + """[T, hidden] -> [T, n, hidden] by replication (model entry).""" + return x.unsqueeze(1).expand(-1, n, -1).contiguous() + + +def hc_contract(x: torch.Tensor) -> torch.Tensor: + """[T, n, hidden] -> [T, hidden] by averaging (model exit).""" + return x.mean(dim=1) + + +def mhc_pre( + residual: torch.Tensor, # [T, n, hidden] bf16 + fn: torch.Tensor, # [2n + n^2, n*hidden] fp32 + hc_scale: torch.Tensor, # [3] fp32 + hc_base: torch.Tensor, # [2n + n^2] fp32 + rms_eps: float, + hc_eps: float, + post_mult: float, + sinkhorn_repeat: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Returns (post_mix [T, n, 1] fp32, comb_mix [T, n, n] fp32, + layer_input [T, hidden] bf16).""" + n, hidden = residual.shape[-2], residual.shape[-1] + t = residual.shape[0] + + x = residual.reshape(t, n * hidden).to(torch.float32) + mixes = x @ fn.t() + # RMS over the FULL flattened n*hidden vector (not per stream). + mixes = mixes * torch.rsqrt(x.square().sum(-1, keepdim=True) / (n * hidden) + rms_eps) + + pre_mix = torch.sigmoid(mixes[:, :n] * hc_scale[0] + hc_base[:n]) + hc_eps + post_mix = torch.sigmoid(mixes[:, n : 2 * n] * hc_scale[1] + hc_base[n : 2 * n]) + post_mix = post_mix * post_mult + + comb = mixes[:, 2 * n :].view(t, n, n) * hc_scale[2] + hc_base[2 * n :].view(1, n, n) + comb = torch.softmax(comb, dim=-1) + hc_eps + # Sinkhorn-Knopp projection toward the doubly-stochastic manifold: alternate + # column / row normalization, ``sinkhorn_repeat`` column steps in total. + comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_eps) + for _ in range(sinkhorn_repeat - 1): + comb = comb / (comb.sum(dim=-1, keepdim=True) + hc_eps) + comb = comb / (comb.sum(dim=-2, keepdim=True) + hc_eps) + + layer_input = ( + (pre_mix.unsqueeze(-1) * residual.to(torch.float32)).sum(dim=1).to(residual.dtype) + ) + return post_mix.view(t, n, 1), comb, layer_input + + +def mhc_post( + x: torch.Tensor, # [T, hidden] sublayer output + residual: torch.Tensor, # [T, n, hidden] + post_mix: torch.Tensor, # [T, n, 1] fp32 + comb_mix: torch.Tensor, # [T, n, n] fp32 +) -> torch.Tensor: + """out_j = sum_i comb_ij * residual_i + post_j * x; returns [T, n, hidden].""" + mixed = torch.einsum( + "tij,tih->tjh", comb_mix.to(torch.float32), residual.to(torch.float32) + ) + post = post_mix.to(torch.float32) * x.unsqueeze(-2).to(torch.float32) + return (mixed + post).to(residual.dtype) + + +def mhc_fused_post_pre_torch( + x: torch.Tensor, + residual: torch.Tensor, + post_mix: torch.Tensor, + comb_mix: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_eps: float, + post_mult: float, + sinkhorn_repeat: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Decomposed reference: hc_post then hc_pre; the fused triton kernel + replaces it on CUDA and is validated against this function.""" + residual_new = mhc_post(x, residual, post_mix, comb_mix) + post_new, comb_new, layer_input = mhc_pre( + residual_new, fn, hc_scale, hc_base, rms_eps, hc_eps, post_mult, sinkhorn_repeat + ) + return residual_new, post_new, comb_new, layer_input + + +def mhc_fused_post_pre( + x: torch.Tensor, + residual: torch.Tensor, + post_mix: torch.Tensor, + comb_mix: torch.Tensor, + fn: torch.Tensor, + hc_scale: torch.Tensor, + hc_base: torch.Tensor, + rms_eps: float, + hc_eps: float, + post_mult: float, + sinkhorn_repeat: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Apply the previous sublayer's hc_post, then this sublayer's hc_pre on the + updated streams. The fused triton kernel serves every batch size on CUDA + (deliberately no T threshold); the decomposed torch path serves CPU/tests.""" + if residual.is_cuda: + from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton + + return mhc_fused_post_pre_triton( + x, residual, post_mix, comb_mix, fn, hc_scale, hc_base, + rms_eps, hc_eps, post_mult, sinkhorn_repeat, + ) + return mhc_fused_post_pre_torch( + x, residual, post_mix, comb_mix, fn, hc_scale, hc_base, + rms_eps, hc_eps, post_mult, sinkhorn_repeat, + ) + + +__all__ = [ + "hc_expand", + "hc_contract", + "mhc_pre", + "mhc_post", + "mhc_fused_post_pre", + "mhc_fused_post_pre_torch", +] diff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.py index d68d8ded57..146783b3d0 100644 --- a/python/freetoken/layers/moe.py +++ b/python/freetoken/layers/moe.py @@ -218,6 +218,12 @@ def __init__( ) self.layer_id = layer_id self.offload_cache: OffloadMoeCache | None = None + # Qwen Q4_K_M has a tiny set of Q6_K down-projection layers. They keep + # their byte-exact rows in a separate cache because Q5_K and Q6_K have + # incompatible packed row sizes. The engine wires these only when the + # loaded checkpoint declares exceptional Q6_K layers. + self.auxiliary_offload_cache: OffloadMoeCache | None = None + self.auxiliary_layer_id: int | None = None def forward( self, @@ -303,6 +309,10 @@ def _decode_routed( ids), so no ``ensure_experts``/``copy_missing`` here.""" cache = self.offload_cache assert cache is not None + if self.auxiliary_offload_cache is not None: + return self._decode_q6_down_routed( + cache, self.auxiliary_offload_cache, hidden_states, topk_weights, topk_ids + ) if cache.is_cpu_layer(self.layer_id): executor = cache.cpu_executor assert executor is not None, "CPU MoE executor was not initialized" @@ -322,6 +332,34 @@ def _decode_routed( is_prefill=False, ) + def _decode_q6_down_routed( + self, + cache: OffloadMoeCache, + auxiliary: OffloadMoeCache, + hidden_states: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + ) -> torch.Tensor: + """Decode an exceptional Q6_K down layer through two independent caches.""" + if cache.decode_target != "gpu": + raise NotImplementedError( + "Qwen GGUF Q6_K down layers currently require the GPU offload backend" + ) + auxiliary_layer_id = self.auxiliary_layer_id + assert auxiliary_layer_id is not None + raw_ids = topk_ids.clone() + cache.ensure_experts(self.layer_id, topk_ids) + auxiliary.ensure_experts(auxiliary_layer_id, raw_ids) + cache.copy_missing() + auxiliary.copy_missing() + from freetoken.moe.fused_q4_k_q6_k import fused_experts_gguf_q4_k_q6_k + + gate_up, _unused_down = cache.bank_views() + (down,) = auxiliary.bank_views() + return fused_experts_gguf_q4_k_q6_k( + hidden_states, gate_up, down, topk_weights, topk_ids, raw_ids, self.activation + ) + def _decode_hybrid( self, cache: OffloadMoeCache, @@ -383,6 +421,10 @@ def _prefill_routed( pass through unmapped.""" cache = self.offload_cache assert cache is not None + if self.auxiliary_offload_cache is not None: + return self._prefill_q6_down_routed( + cache, self.auxiliary_offload_cache, hidden_states, topk_weights, topk_ids + ) if cache.prefill_overlap: views = self._wait_prefill_overlap(cache) out = self._expert_gemm( @@ -410,6 +452,33 @@ def _prefill_routed( is_prefill=True, ) + def _prefill_q6_down_routed( + self, + cache: OffloadMoeCache, + auxiliary: OffloadMoeCache, + hidden_states: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + ) -> torch.Tensor: + """Prefill exceptional Q6_K layers without mixed-cache overlap choreography.""" + if cache.prefill_overlap or auxiliary.prefill_overlap: + raise NotImplementedError( + "Qwen GGUF Q6_K down layers require --disable-moe-prefill-overlap" + ) + auxiliary_layer_id = self.auxiliary_layer_id + assert auxiliary_layer_id is not None + cache.materialize_layer(self.layer_id) + auxiliary.materialize_layer(auxiliary_layer_id) + cache.copy_missing() + auxiliary.copy_missing() + from freetoken.moe.fused_q4_k_q6_k import fused_experts_gguf_q4_k_q6_k + + gate_up, _unused_down = cache.bank_views(self.num_experts) + (down,) = auxiliary.bank_views(self.num_experts) + return fused_experts_gguf_q4_k_q6_k( + hidden_states, gate_up, down, topk_weights, topk_ids, topk_ids, self.activation + ) + def _wait_prefill_overlap(self, cache: OffloadMoeCache) -> tuple[torch.Tensor, ...]: """Double-buffer choreography for this layer's overlap prefill: kick off the next layer's full-layer H2D copy, then return this layer's bank views (in @@ -531,6 +600,16 @@ def _expert_gemm( return fused_experts_gguf_q4_0( hidden_states, gate_up, down, topk_weights, topk_ids, self.activation ) + if fmt == "q4_k_q5_k": + # Qwen Q4_K_M is a mixed GGUF recipe: routed gate/up rows are Q4_K + # while down rows are Q5_K. The kernel reads both packed layouts + # directly and applies the two quant dispatches in sequence. + from freetoken.moe.fused_q4_k_q5_k import fused_experts_gguf_q4_k_q5_k + + gate_up, down = views + return fused_experts_gguf_q4_k_q5_k( + hidden_states, gate_up, down, topk_weights, topk_ids, self.activation + ) if fmt == "mxfp4_triton": # gpt-oss MXFP4 experts (biased, clamped swiglu): transposed split-K GEMV # decode + grouped `_t` prefill. The swiglu scalars live on the layer diff --git a/python/freetoken/message/__init__.py b/python/freetoken/message/__init__.py index e9279f0b33..490a9b705a 100644 --- a/python/freetoken/message/__init__.py +++ b/python/freetoken/message/__init__.py @@ -3,16 +3,19 @@ BaseBackendMsg, BatchBackendMsg, CacheRebuildBackendMsg, + CacheStatsBackendMsg, ExitMsg, UserMsg, ) -from .frontend import BaseFrontendMsg, BatchFrontendMsg, CacheRebuildReply, UserReply +from .frontend import BaseFrontendMsg, BatchFrontendMsg, CacheRebuildReply, CacheStatsReply, UserReply from .tokenizer import ( AbortMsg, BaseTokenizerMsg, BatchTokenizerMsg, CacheRebuildMsg, CacheRebuildResultMsg, + CacheStatsMsg, + CacheStatsResultMsg, DetokenizeMsg, ErrorReplyMsg, PromptAdmittedMsg, @@ -25,12 +28,15 @@ "BaseBackendMsg", "BatchBackendMsg", "CacheRebuildBackendMsg", + "CacheStatsBackendMsg", "ExitMsg", "UserMsg", "BaseTokenizerMsg", "BatchTokenizerMsg", "CacheRebuildMsg", "CacheRebuildResultMsg", + "CacheStatsMsg", + "CacheStatsResultMsg", "DetokenizeMsg", "ErrorReplyMsg", "PromptAdmittedMsg", @@ -38,5 +44,6 @@ "BaseFrontendMsg", "BatchFrontendMsg", "CacheRebuildReply", + "CacheStatsReply", "UserReply", ] diff --git a/python/freetoken/message/backend.py b/python/freetoken/message/backend.py index c42ecc5a65..0bf05f6e22 100644 --- a/python/freetoken/message/backend.py +++ b/python/freetoken/message/backend.py @@ -37,6 +37,10 @@ class UserMsg(BaseBackendMsg): # Optional precomputed multimodal soft-token embeddings (GPU tensor). Only used by # the in-process offline path; remains None for the (serialized) online path. mm_embeds: torch.Tensor | None = None + # Online multimodal requests carry CPU patch tensors over the message wire. + # The scheduler encodes them on its GPU before ordinary prefill admission. + mm_pixel_values: torch.Tensor | None = None + mm_image_position_ids: torch.Tensor | None = None @dataclass @@ -53,3 +57,10 @@ class CacheRebuildBackendMsg(BaseBackendMsg): num_mamba_slots: int | None = None num_swa_pages: int | None = None mode: str = "if_idle" # only "if_idle" is supported; "drain" is deferred (rejected) + + +@dataclass +class CacheStatsBackendMsg(BaseBackendMsg): + """Request one read-only snapshot of the MoE cache counters from the scheduler.""" + + request_id: str diff --git a/python/freetoken/message/frontend.py b/python/freetoken/message/frontend.py index 24725567b9..61f82446d5 100644 --- a/python/freetoken/message/frontend.py +++ b/python/freetoken/message/frontend.py @@ -68,3 +68,11 @@ class CacheRebuildReply(BaseFrontendMsg): mamba_slots: int = 0 num_swa_pages: int = 0 error: str | None = None + + +@dataclass +class CacheStatsReply(BaseFrontendMsg): + """Scheduler-provided, read-only MoE cache-statistics response for the API server.""" + + request_id: str + stats: Dict diff --git a/python/freetoken/message/tokenizer.py b/python/freetoken/message/tokenizer.py index 33b75c785f..75d5e28ecf 100644 --- a/python/freetoken/message/tokenizer.py +++ b/python/freetoken/message/tokenizer.py @@ -72,6 +72,19 @@ class TokenizeMsg(BaseTokenizerMsg): sampling_params: SamplingParams chat_template_kwargs: Dict[str, Any] | None = None tools: List[Dict[str, Any]] | None = None + # ``None`` preserves the tokenizer's normal policy: rendered chat messages + # own their special tokens, while raw completion strings receive the model + # default. A completion caller that has already rendered a complete prompt + # can set this explicitly to avoid inserting a second BOS or template token. + add_special_tokens: bool | None = None + # OpenAI ``image_url`` values aligned to marker strings in ``text``. They + # are decoded in the tokenizer process and never sent as arbitrary URLs to + # the GPU engine. + image_urls: List[Any] | None = None + # CPU image tensors prepared by the tokenizer worker. They retain their + # native shapes through the ZMQ wire and are encoded on the scheduler GPU. + mm_pixel_values: Any | None = None + mm_image_position_ids: Any | None = None @dataclass @@ -102,6 +115,21 @@ class CacheRebuildResultMsg(BaseTokenizerMsg): error: str | None = None +@dataclass +class CacheStatsMsg(BaseTokenizerMsg): + """API-to-tokenizer passthrough for a read-only MoE cache-statistics snapshot.""" + + request_id: str + + +@dataclass +class CacheStatsResultMsg(BaseTokenizerMsg): + """Scheduler-to-tokenizer passthrough carrying an immutable cache-statistics snapshot.""" + + request_id: str + stats: Dict[str, Any] + + @dataclass class ErrorReplyMsg(BaseTokenizerMsg): # scheduler -> tokenizer/detokenizer worker -> frontend: a request the scheduler cannot diff --git a/python/freetoken/message/utils.py b/python/freetoken/message/utils.py index ee92adf5d2..6eff054982 100644 --- a/python/freetoken/message/utils.py +++ b/python/freetoken/message/utils.py @@ -32,10 +32,19 @@ def serialize_type(self) -> Dict: serialized = {} if isinstance(self, torch.Tensor): - assert self.dim() == 1, "we can only serialize 1D tensor for now" + # Backend messages cross a ZMQ boundary as JSON plus bytes. Image + # preprocessing needs 3-D patch and position tensors, while token ids + # remain 1-D. Preserve arbitrary CPU shapes explicitly instead of + # flattening and losing the vision batch contract. + assert self.device.type == "cpu", "only CPU tensors can cross a process boundary" + tensor = self.contiguous() serialized["__type__"] = "Tensor" - serialized["buffer"] = self.numpy().tobytes() - serialized["dtype"] = str(self.dtype) + serialized["shape"] = list(tensor.shape) + serialized["dtype"] = str(tensor.dtype) + # NumPy has no stable bfloat16 dtype on every supported version. Carry + # its bit pattern as uint16 and restore the original torch dtype below. + raw = tensor.view(torch.uint16) if tensor.dtype == torch.bfloat16 else tensor + serialized["buffer"] = raw.numpy().tobytes() return serialized # normal type @@ -64,14 +73,17 @@ def _deserialize_any(cls_map: Dict[str, Type], data: Any) -> Any: def deserialize_type(cls_map: Dict[str, Type], data: Dict) -> Any: type_name = data["__type__"] - # we can only serialize 1D tensor for now if type_name == "Tensor": buffer = data["buffer"] dtype_str = data["dtype"].replace("torch.", "") - np_dtype = getattr(np, dtype_str) + shape = tuple(int(dim) for dim in data.get("shape", [])) assert isinstance(buffer, bytes) - np_tensor = np.frombuffer(buffer, dtype=np_dtype) - return torch.from_numpy(np_tensor.copy()) + if dtype_str == "bfloat16": + raw = np.frombuffer(buffer, dtype=np.uint16).copy().reshape(shape) + return torch.from_numpy(raw).view(torch.bfloat16) + np_dtype = getattr(np, dtype_str) + np_tensor = np.frombuffer(buffer, dtype=np_dtype).copy().reshape(shape) + return torch.from_numpy(np_tensor) cls = cls_map.get(type_name) if cls is None: diff --git a/python/freetoken/models/blocks.py b/python/freetoken/models/blocks.py index ca5b19723c..e811ca8066 100644 --- a/python/freetoken/models/blocks.py +++ b/python/freetoken/models/blocks.py @@ -1,6 +1,7 @@ from __future__ import annotations from abc import ABC, abstractmethod +from contextlib import contextmanager from typing import TYPE_CHECKING from freetoken.layers import ( @@ -16,6 +17,8 @@ if TYPE_CHECKING: import torch + from freetoken.core import Batch + from .config import ModelConfig @@ -23,6 +26,11 @@ class BaseLLMModel(ABC, BaseOP): @abstractmethod def forward(self) -> torch.Tensor: ... + @contextmanager + def forward_host_ctx(self, batch: Batch, use_graph: bool): + """Around one forward dispatch: enter before it is enqueued, exit right after. A backend that feeds the forward from host memory overrides this.""" + yield + class GatedMLP(BaseOP): def __init__(self, config: ModelConfig): diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.py index f6105e1f8d..37dfc8faac 100644 --- a/python/freetoken/models/config.py +++ b/python/freetoken/models/config.py @@ -20,18 +20,35 @@ def vision_load_enabled() -> bool: def detect_expert_quant(hf_config: Any) -> str: """Routed-expert quantization from a checkpoint's ``quantization_config``: ``"nvfp4"`` for - a ModelOpt FP4 build, else the lowercased algo string (``"none"`` when unquantized). Models - with mixed-precision configs (e.g. qwen3_5_moe) need their own detector.""" + a ModelOpt FP4 build (``quant_algo: NVFP4``) OR an llm-compressor NVFP4 export + (``quant_method: compressed-tensors`` + ``format: nvfp4-pack-quantized``, or + ``format: mixed-precision`` with an nvfp4 config group, e.g. + RedHatAI/GLM-5.3-Flash-NVFP4), else the lowercased algo string (``"none"`` when + unquantized). Models with mixed-precision configs (e.g. qwen3_5_moe) need their + own detector.""" quant = getattr(hf_config, "quantization_config", None) if quant is None: return "none" - if isinstance(quant, dict): - algo = quant.get("quant_algo") or quant.get("quant_method") - else: - algo = getattr(quant, "quant_algo", None) or getattr(quant, "quant_method", None) + get = quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d)) + algo = get("quant_algo") or get("quant_method") if algo is None: return "none" - return "nvfp4" if "fp4" in str(algo).lower() else str(algo).lower() + if "fp4" in str(algo).lower(): + return "nvfp4" + fmt = str(get("format") or "").lower() + # exact "nvfp4" (not the "fp4" substring) so MXFP4 exports don't misroute + if "nvfp4" in fmt: + return "nvfp4" + # llm-compressor writes "mixed-precision" at the top when the groups differ (GLM-5.3-Flash: nvfp4 routed experts, fp8 MTP experts); the real format then sits in each group + if fmt == "mixed-precision": + groups = get("config_groups") or {} + groups = [g or {} for g in (groups.values() if isinstance(groups, dict) else [])] + # groups that target the experts decide; only a generic ["Linear"] group falls back to all of them + expert_groups = [g for g in groups if any("experts" in str(t) for t in (g.get("targets") or []))] + for g in expert_groups or groups: + if "nvfp4" in str(g.get("format") or "").lower(): + return "nvfp4" + return str(algo).lower() def detect_compressed_tensors_nvfp4(hf_config: Any) -> bool: @@ -97,6 +114,10 @@ class KVCacheGroupSpec: mla: bool = False index_head_dim: int = 0 num_index_layers: int = 0 + # Grouped index-key compression: one index-key row per ``index_ratio`` tokens + # (QSA groups, glm5_next kpool pools; 1 keeps the per-token BSA/DSA slab). The + # pool factory and the KV cost model divide by the same value. + index_ratio: int = 1 # Attention-type taxonomy value for this group; drives the backend capability # matrix and (with the pool factory) selects the KV pool family. attn_type: AttnType = AttnType.FULL @@ -134,6 +155,10 @@ class FullAttentionGroupConfig(BaseAttentionGroupConfig): mla: bool = False index_head_dim: int = 0 num_index_layers: int = 0 + # Grouped index-key compression ratio (see KVCacheGroupSpec.index_ratio). + # GQA + ratio > 1 -> AttnType.QSA (Qwen3.8); MLA + ratio > 1 -> the glm5_next + # kpool DSA layout (attn type stays DSA; the pool factory branches on mla). + index_ratio: int = 1 @dataclass(frozen=True) @@ -157,7 +182,11 @@ class LinearGatedDeltaGroupConfig(BaseAttentionGroupConfig): key_head_dim: int value_head_dim: int conv_kernel_dim: int - output_gate: bool + # Output-gate activation name ("silu", "sigmoid"), forwarded to rms_norm_gated. + output_gate: str + # "gdn" and "kda" share the same state geometry (one LinearStatePool serves + # both); the variant selects the kernels. + variant: Literal["gdn", "kda"] = "gdn" @dataclass(frozen=True) @@ -185,16 +214,33 @@ class on purpose: subclassing SWAAttentionGroupConfig would flip has_swa_attenti def _full_group_attn_type(group: FullAttentionGroupConfig) -> AttnType: # Mirrors the pool-factory split: mla + index slab -> DSAKVCache, mla -> MLAKVCache, - # GQA (non-mla) + index slab -> BSAKVCache (MiniMax-M3 block-sparse attention). + # GQA (non-mla) + index slab -> QSAKVCache when the index keys are compressed + # (index_ratio > 1, Qwen3.8-Flash-Next) else BSAKVCache (MiniMax-M3 block-sparse). if not group.mla: if group.index_head_dim > 0 and group.num_index_layers > 0: - return AttnType.BSA + return AttnType.QSA if group.index_ratio > 1 else AttnType.BSA return AttnType.FULL if group.index_head_dim > 0 and group.num_index_layers > 0: return AttnType.DSA return AttnType.MLA +@dataclass(frozen=True) +class SlotStateSpec: + """One extra per-request tensor riding the LinearStatePool slots. + + Allocated as ``[max(1, len(layer_ids)), num_slots, *shape]`` and advanced, snapshot, + COW'd and rebuilt with the GDN state; the owner reads it back through + ``pool.slot_state(name, layer_id)``. ``shape`` is per slot and TP-replicated. + """ + + name: str + shape: Tuple[int, ...] + layer_ids: Tuple[int, ...] = () + dtype: Any | None = None # a torch dtype; None -> the pool's compute dtype + fill_value: float = 0.0 + + @dataclass(frozen=True) class ModelConfig: num_layers: int @@ -273,6 +319,12 @@ class ModelConfig: has_attn_bias: bool = False has_router_bias: bool = False moe_weight_format: str | None = None + # Native GGUF Qwen Q4_K_M may use Q6_K down-expert rows in a small subset of + # layers while the remaining down rows are Q5_K. The parser records those + # original layer ids so the exact auxiliary Q6_K cache can be attached only + # where it is needed. + gguf_q6_down_layer_ids: Tuple[int, ...] = () + gguf_tensor_types: Tuple[Tuple[str, int], ...] = () swiglu_limit: float | None = None hidden_act_alpha: float = 1.702 # Full DeepseekV4Args payload for the DSV4-specific machinery (MLA sparse attention, @@ -283,14 +335,25 @@ class ModelConfig: # DSA indexer geometry the model module needs. Opaque to model-agnostic engine code; # None for every other model. glm_dsa_args: Any | None = None + # GLM-5.3-Flash (glm5_next) payload (Glm5NextArgs): NoPE-MLA dims, the kpool indexer + # geometry, the KDA head config, and the mHC knobs. Opaque to model-agnostic engine + # code; None for every other model. + glm5_args: Any | None = None # MiniMax-M3 (minimax_m3) payload (MiniMaxM3Args): the block-sparse indexer geometry # (index heads/dim, top-k blocks, init/local blocks, sparse layer set) plus the # swigluoai/dense-MLP scalars the model module needs. Opaque to model-agnostic engine # code; None for every other model. m3_args: Any | None = None + # Qwen3.8-Flash-Next (qwen4_exp) payload (Qwen4ExpArgs): hyper-connection widths, PLE + # n-gram embedding geometry and the QSA indexer scoring geometry the model module + # needs. Opaque to model-agnostic engine code; None for every other model. + qwen4_args: Any | None = None # Generic execution-path capability flags (set by a model's parse_config) so the engine and # factories stay model-agnostic instead of branching on dsv4_args: single_stream_only: bool = False # model runs one sequence at a time -> force bs=1 + # Extra per-request tensors riding the LinearStatePool slots (see SlotStateSpec); + # () for models without any. Requires a linear-attention group to ride on. + slot_states: Tuple[SlotStateSpec, ...] = () @property def is_moe(self) -> bool: @@ -409,6 +472,7 @@ def kv_cache_group_specs(self) -> Tuple[KVCacheGroupSpec, ...]: mla=group.mla, index_head_dim=group.index_head_dim, num_index_layers=group.num_index_layers, + index_ratio=group.index_ratio, attn_type=_full_group_attn_type(group), ) ) diff --git a/python/freetoken/models/gemma4/attention.py b/python/freetoken/models/gemma4/attention.py index 9103bf3413..9acfb0ccee 100644 --- a/python/freetoken/models/gemma4/attention.py +++ b/python/freetoken/models/gemma4/attention.py @@ -45,6 +45,7 @@ def __init__(self, config: ModelConfig, layer_id: int): self.attn_spec = AttentionSpec( sliding_window=group.sliding_window if self.is_swa else None, sm_scale=config.attn_sm_scale, + multimodal_bidirectional=self.is_swa, ) self.rotary = get_rope( head_dim=self.head_dim, diff --git a/python/freetoken/models/gemma4/gguf.py b/python/freetoken/models/gemma4/gguf.py index 437822b519..1c62adbf4f 100644 --- a/python/freetoken/models/gemma4/gguf.py +++ b/python/freetoken/models/gemma4/gguf.py @@ -12,6 +12,7 @@ from __future__ import annotations +import os from typing import TYPE_CHECKING, Iterator import torch @@ -21,7 +22,9 @@ ModelConfig, RotaryConfig, SWAAttentionGroupConfig, + vision_load_enabled, ) +from freetoken.models.gemma4.config import VisionConfig from freetoken.models.gguf.dequant import GGML_Q4_0, GGML_Q6_K, dequantize, row_bytes if TYPE_CHECKING: @@ -93,6 +96,13 @@ def g(key: str): scaling=None, ) + # llama.cpp stores Gemma 4's visual tower in a sibling ``mmproj`` GGUF rather + # than in the text GGUF. Reconstruct the vision config from that file only + # when the explicit opt-in is enabled, preserving the text-only memory budget + # by default. The values originate in the projector metadata, not guessed + # from the text checkpoint's geometry. + vision_config = _parse_gguf_vision_config(shim, hidden) + return ModelConfig( num_layers=num_layers, num_qo_heads=num_qo_heads, @@ -118,6 +128,8 @@ def g(key: str): attn_sm_scale=1.0, final_logit_softcapping=float(g("final_logit_softcapping")), embedding_scale=float(hidden) ** 0.5, + vision_config=vision_config, + image_token_id=_gemma4_image_token_id(m) if vision_config is not None else None, attention_groups=( FullAttentionGroupConfig( name="full", @@ -171,6 +183,142 @@ def _to_bf16(t) -> torch.Tensor: return flat.reshape(t.shape) +def find_gemma4_mmproj(model_path: str) -> str | None: + """Return the unique sibling Gemma4 projector GGUF, when the release supplies one. + + Text GGUF releases keep the 1.2 GiB vision tower in a separate file whose name + includes ``mmproj``. Text-only loading deliberately never calls this helper; + vision setup calls it only after the explicit ``FREETOKEN_LOAD_VISION`` opt-in. + A missing or ambiguous sibling remains an error for the caller to report with + the model path, rather than silently loading arbitrary GGUF content. + """ + directory = os.path.dirname(model_path) + candidates = sorted( + os.path.join(directory, name) + for name in os.listdir(directory) + if name.endswith(".gguf") and "mmproj" in name.lower() + ) + return candidates[0] if len(candidates) == 1 else None + + +def _gemma4_image_token_id(metadata: dict) -> int: + """Find Gemma's image placeholder in the GGUF tokenizer without a magic id. + + ``image_token_id`` is part of the original checkpoint configuration, but + GGUF retains the tokenizer rather than that JSON field. The token has had + two spellings across Gemma converters, so accept those exact spellings and + reject all other image-looking vocabulary entries rather than binding an + unrelated token silently. + """ + tokens = metadata.get("tokenizer.ggml.tokens") + if not isinstance(tokens, list): + raise ValueError("Gemma4 GGUF vision requires tokenizer.ggml.tokens") + accepted = {"<|image|>"} + matches = [index for index, token in enumerate(tokens) if str(token) in accepted] + if len(matches) != 1: + raise ValueError( + "Gemma4 GGUF vision requires exactly one image placeholder token; " + f"found {matches} among accepted spellings {sorted(accepted)}" + ) + return matches[0] + + +def _parse_gguf_vision_config(shim: "GgufConfigShim", text_hidden_size: int) -> VisionConfig | None: + """Build :class:`VisionConfig` from the sibling Gemma4 projector GGUF. + + The projector supplies its parameter dimensions. Algorithm settings not + represented in the GGUF metadata follow the official Gemma4 26B-A4B vision + contract: 10,240 position slots, 3x3 pooling, 280 soft tokens, + two-dimensional RoPE theta 100, standardization, and unclipped linears. + Validate the cross-file projection width so a mixed text/projector directory + fails during startup instead of producing corrupt image embeddings. + """ + if not vision_load_enabled(): + return None + mmproj_path = find_gemma4_mmproj(shim.model_path) + if mmproj_path is None: + raise FileNotFoundError( + f"Gemma4 vision was requested but no unique sibling mmproj GGUF exists beside " + f"{shim.model_path!r}" + ) + from freetoken.models.gguf.reader import load_gguf_metadata + + metadata = load_gguf_metadata(mmproj_path) + + def v(key: str): + value = metadata.get(f"clip.vision.{key}") + if value is None: + raise KeyError(f"missing Gemma4 projector metadata key clip.vision.{key}") + return value + + vision_hidden = int(v("embedding_length")) + projection_width = int(v("projection_dim")) + if projection_width != text_hidden_size: + raise ValueError( + "Gemma4 projector/text width mismatch: " + f"mmproj={projection_width}, text={text_hidden_size}" + ) + num_heads = int(v("attention.head_count")) + return VisionConfig( + hidden_size=vision_hidden, + num_layers=int(v("block_count")), + num_heads=num_heads, + num_kv_heads=num_heads, + head_dim=vision_hidden // num_heads, + intermediate_size=int(v("feed_forward_length")), + patch_size=int(v("patch_size")), + # llama.cpp's mmproj metadata does not serialize the learned table's + # capacity. Gemma4's released 26B config fixes it at 10 * 1024. + position_embedding_size=10_240, + pooling_kernel_size=3, + rms_norm_eps=float(v("attention.layer_norm_epsilon")), + rope_theta=100.0, + hidden_act="gelu_tanh", + standardize=True, + use_clipped_linears=False, + soft_tokens_per_image=280, + text_hidden_size=text_hidden_size, + ) + + +def gemma4_mmproj_param_name(source_name: str) -> str | None: + """Map one llama.cpp Gemma4 projector tensor name to FreeToken's module key.""" + if source_name == "mm.input_projection.weight": + return "embed_vision.embedding_projection.weight" + if source_name == "v.patch_embd.weight": + return "vision_tower.patch_embedder.input_proj.weight" + if source_name == "v.position_embd.weight": + return "vision_tower.patch_embedder.position_embedding_table" + if source_name == "v.std_bias": + return "vision_tower.std_bias" + if source_name == "v.std_scale": + return "vision_tower.std_scale" + if not source_name.startswith("v.blk."): + return None + prefix, suffix = source_name.rsplit(".", 1)[0], source_name.rsplit(".", 1)[1] + parts = prefix.split(".") + if len(parts) != 4 or parts[0] != "v" or parts[1] != "blk" or not parts[2].isdigit() or suffix != "weight": + return None + layer = parts[2] + remap = { + "ln1": "input_layernorm.weight", + "ln2": "pre_feedforward_layernorm.weight", + "attn_post_norm": "post_attention_layernorm.weight", + "ffn_post_norm": "post_feedforward_layernorm.weight", + "attn_q_norm": "self_attn.q_norm.weight", + "attn_k_norm": "self_attn.k_norm.weight", + "attn_q": "self_attn.q_proj.weight", + "attn_k": "self_attn.k_proj.weight", + "attn_v": "self_attn.v_proj.weight", + "attn_out": "self_attn.o_proj.weight", + "ffn_gate": "mlp.gate_proj.weight", + "ffn_up": "mlp.up_proj.weight", + "ffn_down": "mlp.down_proj.weight", + } + mapped = remap.get(parts[3]) + return f"vision_tower.encoder.layers.{layer}.{mapped}" if mapped else None + + def _require_tp1(what: str) -> None: """GGUF quant layers / expert banks are not sharded; reject TP>1 with a clear error instead of failing later on a confusing shape mismatch (mirrors the HF @@ -297,6 +445,24 @@ def layer_of(name: str) -> int: assert not qkv_buf, f"incomplete qkv groups: {sorted(qkv_buf)}" assert not gate_up_buf, f"incomplete gate_up groups: {sorted(gate_up_buf)}" + # Gemma's text GGUF stores the vision tower in a sibling ``*-mmproj.gguf``. + # This stays behind the explicit vision opt-in so text-only serving never + # pays the startup or memory cost of the projector. + if config.is_multimodal: + mmproj_path = find_gemma4_mmproj(model_path) + if mmproj_path is None: + raise FileNotFoundError( + f"Gemma4 vision is enabled but no unique sibling mmproj GGUF exists beside {model_path}" + ) + for t in iter_gguf_tensors(mmproj_path): + target = gemma4_mmproj_param_name(t.name) + if target is None: + raise ValueError(f"unmapped Gemma4 projector tensor: {t.name}") + tensor = _to_bf16(t) + if t.name == "v.patch_embd.weight": + tensor = tensor.flatten(1) + yield target, tensor + # -------------------------------------------------------------------------------------- # Model layer swap: dense bf16 Linear/Embedding -> native GGUF-quant ops. diff --git a/python/freetoken/models/gemma4/model.py b/python/freetoken/models/gemma4/model.py index 66fcf41235..f530d5a8b5 100644 --- a/python/freetoken/models/gemma4/model.py +++ b/python/freetoken/models/gemma4/model.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from typing import TYPE_CHECKING import torch @@ -11,7 +12,7 @@ ParallelLMHead, VocabParallelEmbedding, ) -from freetoken.utils import nvtx_annotate +from freetoken.utils import init_logger, nvtx_annotate from freetoken.models.blocks import BaseLLMModel @@ -23,6 +24,38 @@ from freetoken.models.config import ModelConfig +logger = init_logger(__name__) + + +def _log_vision_embedding_summary(label: str, tensor: torch.Tensor) -> None: + """Write a bounded numerical fingerprint for an opt-in vision parity run. + + ``FREETOKEN_GEMMA4_VISION_DEBUG=1`` enables this diagnostic while investigating + a reference mismatch. It deliberately records only shape, finite-state, + aggregate statistics, and the first sixteen scalar values: that is enough to + compare FreeToken's vision output with llama.cpp's mtmd debug output without + placing a full image embedding in a server log. The environment guard keeps + normal serving free from the device synchronization caused by ``cpu()``. + """ + if os.environ.get("FREETOKEN_GEMMA4_VISION_DEBUG") != "1": + return + values = tensor.detach().float() + sample = values.reshape(-1)[:16].cpu().tolist() + logger.info_rank0( + "Gemma4 vision debug %s: shape=%s finite=%s mean=%.8f std=%.8f " + "min=%.8f max=%.8f sum=%.8f first16=%s", + label, + tuple(values.shape), + bool(torch.isfinite(values).all().item()), + float(values.mean().item()), + float(values.std(unbiased=False).item()), + float(values.min().item()), + float(values.max().item()), + float(values.sum().item()), + ",".join(f"{value:.8f}" for value in sample), + ) + + class Gemma4DecoderLayer(BaseOP): """Gemma 4 decoder block: attention sandwich + feed-forward sandwich, scaled by a per-layer ``layer_scalar``. The feed-forward is the dual (shared MLP || routed MoE) @@ -125,7 +158,10 @@ def encode_images( ``image_position_ids``: ``[num_images, num_patches, 2]`` with ``(-1, -1)`` padding. """ features = self.vision_tower.forward(pixel_values, image_position_ids) - return self.embed_vision.forward(features) + _log_vision_embedding_summary("tower", features) + projected = self.embed_vision.forward(features) + _log_vision_embedding_summary("projected", projected) + return projected def forward(self) -> torch.Tensor: output = self.model.forward(get_global_ctx().batch.input_ids) diff --git a/python/freetoken/models/gemma4/vision.py b/python/freetoken/models/gemma4/vision.py index 7c260c48d7..530e67974d 100644 --- a/python/freetoken/models/gemma4/vision.py +++ b/python/freetoken/models/gemma4/vision.py @@ -1,15 +1,46 @@ from __future__ import annotations +import os from typing import TYPE_CHECKING, Tuple import torch import torch.nn.functional as F from freetoken.layers import BaseOP, GemmaRMSNorm, LinearReplicated, OPList +from freetoken.utils import init_logger if TYPE_CHECKING: from freetoken.models.gemma4.config import VisionConfig +logger = init_logger(__name__) + + +def _log_vision_stage(label: str, tensor: torch.Tensor) -> None: + """Record a compact, opt-in fingerprint at one vision-model boundary. + + llama.cpp's mtmd debugger can expose intermediate vision graph values. This + matching summary lets an AMD FreeToken investigation identify the first + divergent stage without dumping a full image embedding, which would both + distort a timing run and create enormous artifacts. + """ + if os.environ.get("FREETOKEN_GEMMA4_VISION_DEBUG") != "1": + return + values = tensor.detach().float() + logger.info_rank0( + "Gemma4 vision stage %s: shape=%s finite=%s mean=%.8f std=%.8f " + "min=%.8f max=%.8f sum=%.8f first16=%s", + label, + tuple(values.shape), + bool(torch.isfinite(values).all().item()), + float(values.mean().item()), + float(values.std(unbiased=False).item()), + float(values.min().item()), + float(values.max().item()), + float(values.sum().item()), + ",".join(f"{value:.8f}" for value in values.reshape(-1)[:16].cpu().tolist()), + ) + + def _rotate_half(x: torch.Tensor) -> torch.Tensor: half = x.shape[-1] // 2 return torch.cat((-x[..., half:], x[..., :half]), dim=-1) @@ -204,17 +235,26 @@ def forward(self, pixel_values: torch.Tensor, position_ids: torch.Tensor) -> tor padding = (position_ids == -1).all(dim=-1) # [B, P] True = padding patch h = self.patch_embedder.forward(pixel_values, position_ids, padding) + _log_vision_stage("patch_embed", h) + # Reference encoders operate on the unpadded patch grid. Record the + # equivalent compact view as well, so padding-query values cannot hide + # the first numerical mismatch during a parity investigation. + _log_vision_stage("patch_embed_valid", h[~padding]) cos, sin = self.encoder._rotary.cos_sin(position_ids, h.dtype) attn_mask = (~padding)[:, None, None, :] # [B, 1, 1, P] True = attend for layer in self.encoder.layers.op_list: h = layer.forward(h, cos, sin, attn_mask) + _log_vision_stage("encoder", h) + _log_vision_stage("encoder_valid", h[~padding]) h = h.masked_fill(padding.unsqueeze(-1), 0.0) pooled, mask = _avg_pool_by_positions(h, position_ids, output_length) pooled = pooled.float() * self._root_hidden + _log_vision_stage("pooled_scaled", pooled) pooled = pooled[mask] # [num_valid, hidden] fp32 if self._standardize: pooled = (pooled - self.std_bias.float()) * self.std_scale.float() + _log_vision_stage("standardized", pooled) return pooled.to(h.dtype) @@ -230,7 +270,11 @@ def __init__(self, vc: VisionConfig): ) def forward(self, x: torch.Tensor) -> torch.Tensor: - return self.embedding_projection.forward(self.embedding_pre_projection_norm.forward(x)) + normalized = self.embedding_pre_projection_norm.forward(x) + _log_vision_stage("projector_norm", normalized) + projected = self.embedding_projection.forward(normalized) + _log_vision_stage("projector_output", projected) + return projected __all__ = ["Gemma4VisionModel", "Gemma4MultimodalEmbedder"] diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py index 63b1a18b97..f419371ad7 100644 --- a/python/freetoken/models/gguf/config.py +++ b/python/freetoken/models/gguf/config.py @@ -18,6 +18,9 @@ # reuses the model classes but a GGUF parse_config / iter_weights). GGUF_ARCH_TO_REGISTRY: dict[str, str] = { "gemma4": "Gemma4GGUFForCausalLM", + # Dense Qwen3.8 shares the hybrid Qwen3.5 model class with Qwen3.6 MoE. + "qwen35": "Qwen3_5MoeGGUFForCausalLM", + "qwen35moe": "Qwen3_5MoeGGUFForCausalLM", } diff --git a/python/freetoken/models/gguf/dequant.py b/python/freetoken/models/gguf/dequant.py index 77c3ea0102..227f737d6b 100644 --- a/python/freetoken/models/gguf/dequant.py +++ b/python/freetoken/models/gguf/dequant.py @@ -23,6 +23,8 @@ GGML_F16 = 1 GGML_Q4_0 = 2 GGML_Q8_0 = 8 +GGML_Q4_K = 12 +GGML_Q5_K = 13 GGML_Q6_K = 14 GGML_BF16 = 30 @@ -33,6 +35,12 @@ GGML_BF16: (1, 2), GGML_Q4_0: (32, 18), GGML_Q8_0: (32, 34), + # Q4_K is the common ``Q4_K_M`` tensor encoding. The ``M`` label describes + # a model-wide mixed quantization recipe, while individual GGUF tensors carry + # the base GGML type Q4_K. Each super-block holds two fp16 scales, twelve packed + # six-bit sub-scales, and 128 packed four-bit values. + GGML_Q4_K: (256, 144), + GGML_Q5_K: (256, 176), GGML_Q6_K: (256, 210), } @@ -42,6 +50,8 @@ GGML_BF16: "BF16", GGML_Q4_0: "Q4_0", GGML_Q8_0: "Q8_0", + GGML_Q4_K: "Q4_K", + GGML_Q5_K: "Q5_K", GGML_Q6_K: "Q6_K", } @@ -79,6 +89,14 @@ def dequant_q4_0(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: return ((q - 8.0) * d).reshape(-1).to(out_dtype) +def dequant_q8_0(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: + """Q8_0: per 32-element block = fp16 scale followed by 32 signed int8 values.""" + raw = raw.reshape(-1, 34) + d = _f16_scales(raw, 0, 2) + q = raw[:, 2:34].view(torch.int8).to(torch.float32) + return (q * d).reshape(-1).to(out_dtype) + + def dequant_q6_k(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: """Q6_K: 256-elem super-block = 128B low nibbles + 64B high 2-bits + 16 int8 sub-scales + fp16 ``d``. Direct vectorization of ggml's two-half loop.""" @@ -115,8 +133,62 @@ def dequant_q6_k(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: return y.reshape(-1).to(out_dtype) +def dequant_q4_k(raw: torch.Tensor, out_dtype: torch.dtype) -> torch.Tensor: + """Q4_K reference decoder matching llama.cpp's ``dequantize_row_q4_K``. + + A Q4_K super-block covers 256 values as eight 32-value groups. ``scales`` + packs the eight positive scales followed by the eight minimum coefficients as + six-bit little-endian integers. For each group the decoded value is + ``d * scale * q - dmin * minimum``. This runs only in tests and non-hot + load-time conversions; GPU execution stays packed in the GGUF kernels. + """ + raw = raw.reshape(-1, 144) + block_count = raw.shape[0] + scale_bytes = raw[:, 4:16].to(torch.int32) + # Direct vector form of ggml's get_scale_min_k4. Entries 0..3 store a + # six-bit scale and minimum directly. Entries 4..7 split each high two + # bits across the first four bytes and the upper nibble of bytes 8..11. + scales = torch.empty((block_count, 8), dtype=torch.float32, device=raw.device) + minimums = torch.empty_like(scales) + scales[:, :4] = (scale_bytes[:, :4] & 0x3F).to(torch.float32) + minimums[:, :4] = (scale_bytes[:, 4:8] & 0x3F).to(torch.float32) + scales[:, 4:] = ( + (scale_bytes[:, 8:12] & 0x0F) | ((scale_bytes[:, :4] >> 6) << 4) + ).to(torch.float32) + minimums[:, 4:] = ( + (scale_bytes[:, 8:12] >> 4) | ((scale_bytes[:, 4:8] >> 6) << 4) + ).to(torch.float32) + d = _f16_scales(raw, 0, 2) + dmin = _f16_scales(raw, 2, 4) + quantized = raw[:, 16:144] + values = torch.empty((block_count, 256), dtype=torch.float32, device=raw.device) + # GGML stores each pair of 32-value groups in the same 32-byte region: + # the low nibbles are group ``2 * pair`` and the high nibbles are group + # ``2 * pair + 1``. They are therefore not eight consecutive 16-byte + # groups. Keeping this order identical to ``dequantize_block_q4_K`` is + # essential because this decoder is the independent correctness oracle for + # the packed HIP kernels. + for pair in range(4): + pair_bytes = quantized[:, pair * 32:(pair + 1) * 32] + low = (pair_bytes & 0x0F).to(torch.float32) + high = (pair_bytes >> 4).to(torch.float32) + low_group = 2 * pair + high_group = low_group + 1 + values[:, low_group * 32:(low_group + 1) * 32] = ( + d * scales[:, low_group:low_group + 1] * low + - dmin * minimums[:, low_group:low_group + 1] + ) + values[:, high_group * 32:(high_group + 1) * 32] = ( + d * scales[:, high_group:high_group + 1] * high + - dmin * minimums[:, high_group:high_group + 1] + ) + return values.reshape(-1).to(out_dtype) + + _DEQUANT = { GGML_Q4_0: dequant_q4_0, + GGML_Q8_0: dequant_q8_0, + GGML_Q4_K: dequant_q4_k, GGML_Q6_K: dequant_q6_k, } @@ -142,12 +214,16 @@ def dequantize(raw: torch.Tensor, ggml_type: int, out_dtype: torch.dtype) -> tor "GGML_F16", "GGML_BF16", "GGML_Q4_0", + "GGML_Q4_K", + "GGML_Q5_K", "GGML_Q8_0", "GGML_Q6_K", "GGML_NAME", "BLOCK_SHAPE", "row_bytes", "dequant_q4_0", + "dequant_q8_0", + "dequant_q4_k", "dequant_q6_k", "dequantize", ] diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.py index 6d5481c177..18bbc4794e 100644 --- a/python/freetoken/models/gguf/tokenizer.py +++ b/python/freetoken/models/gguf/tokenizer.py @@ -13,7 +13,36 @@ from .reader import gguf_architecture, load_gguf_metadata # GGUF architecture -> transformers GGUF tokenizer-converter key. -_TOKENIZER_ARCH = {"gemma4": "gemma4_text"} +_TOKENIZER_ARCH = {"gemma4": "gemma4_text", "qwen35": "qwen3", "qwen35moe": "qwen3_moe"} + + +def _register_embedded_special_tokens( + tokenizer: Any, tokens: list[Any], token_types: Any +) -> None: + """Restore GGUF CONTROL and USER_DEFINED token matching on a fast tokenizer. + + The GGUF converter supplies the vocabulary ids, but some transformer releases + do not install USER_DEFINED entries as fast-tokenizer special tokens. This + helper deliberately registers only GGML token classes 3 (CONTROL) and 4 + (USER_DEFINED), excluding the four roles already configured on the tokenizer. + ``add_special_tokens`` retains a pre-existing vocabulary id when the spelling + is already present, so model weights and prompt ids remain aligned. + """ + if not isinstance(token_types, (list, tuple)) or len(token_types) != len(tokens): + return + configured_specials = { + tokenizer.bos_token, + tokenizer.eos_token, + tokenizer.unk_token, + tokenizer.pad_token, + } + embedded_specials = [ + str(token) + for token, token_type in zip(tokens, token_types) + if token_type in (3, 4) and str(token) not in configured_specials + ] + if embedded_specials: + tokenizer.add_special_tokens({"additional_special_tokens": embedded_specials}) def load_gguf_tokenizer(model_path: str): @@ -46,6 +75,17 @@ def tok_for(id_key: str, default: str) -> str: unk_token=tok_for("unknown_token_id", ""), pad_token=tok_for("padding_token_id", ""), ) + + # ``convert_gguf_tokenizer`` preserves every vocabulary entry but does not + # consistently restore GGUF's USER_DEFINED token class as an atomic special + # token. Qwen3.6 declares ```` and ```` in that class. If + # they are not registered here, a caller-rendered ```` prompt is + # split into three ordinary pieces (````), so FreeToken + # runs a different token sequence from llama.cpp despite receiving exactly + # the same UTF-8 request body. GGUF token types 3 and 4 are CONTROL and + # USER_DEFINED respectively. Registering both groups retains their + # existing vocabulary ids while making their matching semantics explicit. + _register_embedded_special_tokens(tokenizer, tokens, tok_dict.get("token_type")) chat_template = meta.get("tokenizer.chat_template") if chat_template: tokenizer.chat_template = chat_template diff --git a/python/freetoken/models/glm5_next/__init__.py b/python/freetoken/models/glm5_next/__init__.py new file mode 100644 index 0000000000..c1c05140a3 --- /dev/null +++ b/python/freetoken/models/glm5_next/__init__.py @@ -0,0 +1,10 @@ +from .config import parse_config +from .model import Glm5NextForCausalLM +from .weight import iter_weights, load_nvfp4_expert_sources + +__all__ = [ + "Glm5NextForCausalLM", + "parse_config", + "iter_weights", + "load_nvfp4_expert_sources", +] diff --git a/python/freetoken/models/glm5_next/args.py b/python/freetoken/models/glm5_next/args.py new file mode 100644 index 0000000000..c25fef4856 --- /dev/null +++ b/python/freetoken/models/glm5_next/args.py @@ -0,0 +1,225 @@ +"""GLM-5.3-Flash (``glm5_next``) hyperparameters. + +GLM-5.3-Flash is the first hybrid GLM: 34 of 45 decoder layers run KDA linear +attention (Kimi-Delta-Attention: per-channel gated delta rule with separate q/k/v +short convolutions) and the remaining 11 run DeepSeek-sparse attention -- MLA +latent KV with a Lightning indexer whose K cache is *pool-compressed* +(``index_kpool`` tokens fold into one stored entry). The main attention is NoPE +(``qk_rope_head_dim == 0``, ``mla_use_nope``): no rotary embedding on Q/K at all; +positional information enters only through the indexer's pool-compression APE. +The residual stream is widened by mHC (Manifold-Constrained Hyper-Connections, +``hc_mult`` streams mixed by a Sinkhorn-projected matrix around every sublayer). + +This payload carries everything the model module needs beyond the generic +``ModelConfig`` fields; it is stashed on ``ModelConfig.glm5_args`` (opaque to the +engine). Field spellings follow the checkpoint's ``config.json`` (transformers +5.16 ``Glm5NextTextConfig``); ``load_args`` folds the checkpoint aliases +(``hc_mult``, ``hc_sinkhorn_iters``, ``mla_use_nope``, the nested +``linear_attn_config`` dict) the same way vLLM's config class does, so a future +flattened checkpoint keeps loading. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Tuple + +# Layer-type strings used by the checkpoint's ``layer_types`` field. +KDA_LAYER = "linear_attention" +DSA_LAYER = "deepseek_sparse_attention" + + +@dataclass(frozen=True) +class Glm5NextArgs: + hidden_size: int + num_heads: int + # ---- MLA (the 11 "deepseek_sparse_attention" layers) ---- + q_lora_rank: int + kv_lora_rank: int + qk_nope_head_dim: int + qk_rope_head_dim: int # 0: NoPE -- main attention carries no rotary dims + v_head_dim: int + mla_nope: bool + norm_eps: float + max_position: int + # ---- DSA indexer ---- + index_n_heads: int + index_head_dim: int + index_topk: int + indexer_types: Tuple[str, ...] + indexer_rope_interleave: bool + # kpool compression: every ``index_kpool`` indexer-K entries pool into one + # stored entry (softmax(gate + APE)-weighted sum); top-k selects pools + # (select_k = index_topk // index_kpool) and ``always_select_tail`` + # force-includes the in-progress tail pool. + index_kpool: int + index_kpool_compress: bool + index_kpool_always_select_tail: bool + # ---- KDA linear attention (the 34 "linear_attention" layers) ---- + linear_num_heads: int + linear_head_dim: int + linear_conv_kernel_dim: int + linear_lower_bound: float + # ---- per-layer layout ---- + layer_types: Tuple[str, ...] + mlp_layer_types: Tuple[str, ...] + # ---- mHC (Manifold-Constrained Hyper-Connections) ---- + mhc: bool + mhc_num_residual_streams: int + hc_eps: float + mhc_sinkhorn_iterations: int + mhc_tau: float + mhc_post_mult_value: float + mhc_no_norm_weight: bool + # ---- misc ---- + swiglu_limit: float | None + rope_theta: float # indexer-side only; the main attention is NoPE + + @property + def qk_head_dim(self) -> int: + return self.qk_nope_head_dim + self.qk_rope_head_dim + + @property + def latent_dim(self) -> int: + """Width of one MLA latent row in the paged pool: ckv | kpe.""" + return self.kv_lora_rank + self.qk_rope_head_dim + + @property + def kda_layer_ids(self) -> Tuple[int, ...]: + return tuple(i for i, t in enumerate(self.layer_types) if t == KDA_LAYER) + + @property + def dsa_layer_ids(self) -> Tuple[int, ...]: + return tuple(i for i, t in enumerate(self.layer_types) if t == DSA_LAYER) + + def is_kda_layer(self, layer_id: int) -> bool: + return self.layer_types[layer_id] == KDA_LAYER + + +def _require_mhc(mhc: bool) -> bool: + """The decoder layer wires the hyper-connection tensors unconditionally + (model.py touches hc_attn_fn on every forward), so an mhc=False checkpoint + would die with an AttributeError mid-forward -- refuse it at parse until a + real one exists to implement against.""" + if not mhc: + raise NotImplementedError( + "glm5_next requires mhc=True (manifold-constrained hyper-connections); " + "an mhc=False checkpoint has no implementation yet." + ) + return mhc + + +def _get(cfg: Any, name: str, default: Any = None) -> Any: + return getattr(cfg, name, default) + + +def load_args(hf_config: Any) -> Glm5NextArgs: + """Build ``Glm5NextArgs`` from the checkpoint config (nested ``text_config`` or a + flat text-only config). Raw checkpoint spellings win; the vLLM-normalized names + are accepted as fallbacks.""" + text = _get(hf_config, "text_config", hf_config) + + num_layers = int(text.num_hidden_layers) + layer_types = tuple(_get(text, "layer_types", ()) or ()) + if not layer_types: + raise ValueError("glm5_next config is missing layer_types") + if len(layer_types) != num_layers: + raise ValueError( + f"layer_types has {len(layer_types)} entries for {num_layers} layers" + ) + unknown = sorted(set(layer_types) - {KDA_LAYER, DSA_LAYER}) + if unknown: + raise ValueError(f"unsupported layer_types entries: {unknown}") + + mlp_layer_types = tuple(_get(text, "mlp_layer_types", ()) or ()) + if not mlp_layer_types: + # Older-schema fallback (mirrors vLLM): derive from first_k_dense_replace. + first_dense = int(_get(text, "first_k_dense_replace", 0) or 0) + mlp_layer_types = ("dense",) * first_dense + ("sparse",) * ( + num_layers - first_dense + ) + + # Checkpoint alias folding (transformers 5.16 spellings first). + mla_nope = _get(text, "mla_use_nope", _get(text, "mla_nope", False)) + hc_mult = _get(text, "hc_mult", _get(text, "mhc_num_residual_streams", 4)) + hc_sinkhorn = _get( + text, "hc_sinkhorn_iters", _get(text, "mhc_sinkhorn_iterations", 20) + ) + + # KDA head geometry ships as the nested ``linear_attn_config`` dict; a future + # flattened schema would carry vLLM-style ``linear_*`` top-level fields. + linear_cfg = _get(text, "linear_attn_config", None) + if linear_cfg is not None and not isinstance(linear_cfg, dict): + linear_cfg = { + k: getattr(linear_cfg, k) + for k in ( + "num_heads", + "head_dim", + "short_conv_kernel_size", + "gate_lower_bound", + ) + if hasattr(linear_cfg, k) + } + linear_cfg = linear_cfg or {} + linear_num_heads = int( + linear_cfg.get("num_heads", _get(text, "linear_num_heads", 0)) + ) + linear_head_dim = int(linear_cfg.get("head_dim", _get(text, "linear_head_dim", 0))) + linear_conv = int( + linear_cfg.get( + "short_conv_kernel_size", _get(text, "linear_conv_kernel_dim", 4) + ) + ) + linear_lower_bound = float( + linear_cfg.get("gate_lower_bound", _get(text, "linear_lower_bound", -5.0)) + ) + if KDA_LAYER in layer_types and (linear_num_heads <= 0 or linear_head_dim <= 0): + raise ValueError("glm5_next config is missing the KDA linear_attn_config dims") + + # The main attention is NoPE; rope exists only on the indexer side. The + # checkpoint ships no rope_theta -- fall back to the transformers default. + rope = _get(text, "rope_parameters", None) or {} + rope_theta = float(rope.get("rope_theta", _get(text, "rope_theta", 10000.0))) + + swiglu_limit = _get(text, "swiglu_limit", None) + + return Glm5NextArgs( + hidden_size=int(text.hidden_size), + num_heads=int(text.num_attention_heads), + q_lora_rank=int(text.q_lora_rank), + kv_lora_rank=int(text.kv_lora_rank), + qk_nope_head_dim=int(text.qk_nope_head_dim), + qk_rope_head_dim=int(text.qk_rope_head_dim), + v_head_dim=int(text.v_head_dim), + mla_nope=bool(mla_nope), + norm_eps=float(text.rms_norm_eps), + max_position=int(text.max_position_embeddings), + index_n_heads=int(_get(text, "index_n_heads", 0) or 0), + index_head_dim=int(_get(text, "index_head_dim", 0) or 0), + index_topk=int(_get(text, "index_topk", 0) or 0), + indexer_types=tuple(_get(text, "indexer_types", ()) or ()), + indexer_rope_interleave=bool(_get(text, "indexer_rope_interleave", False)), + index_kpool=int(_get(text, "index_kpool", 1) or 1), + index_kpool_compress=bool(_get(text, "index_kpool_compress", False)), + index_kpool_always_select_tail=bool( + _get(text, "index_kpool_always_select_tail", False) + ), + linear_num_heads=linear_num_heads, + linear_head_dim=linear_head_dim, + linear_conv_kernel_dim=linear_conv, + linear_lower_bound=linear_lower_bound, + layer_types=layer_types, + mlp_layer_types=mlp_layer_types, + mhc=_require_mhc(bool(_get(text, "mhc", False))), + mhc_num_residual_streams=int(hc_mult), + hc_eps=float(_get(text, "hc_eps", 1e-6)), + mhc_sinkhorn_iterations=int(hc_sinkhorn), + mhc_tau=float(_get(text, "mhc_tau", 0.05)), + mhc_post_mult_value=float(_get(text, "mhc_post_mult_value", 2.0)), + mhc_no_norm_weight=bool(_get(text, "mhc_no_norm_weight", False)), + swiglu_limit=(None if swiglu_limit is None else float(swiglu_limit)), + rope_theta=rope_theta, + ) + + +__all__ = ["Glm5NextArgs", "load_args", "KDA_LAYER", "DSA_LAYER"] diff --git a/python/freetoken/models/glm5_next/attention.py b/python/freetoken/models/glm5_next/attention.py new file mode 100644 index 0000000000..1ab6c34047 --- /dev/null +++ b/python/freetoken/models/glm5_next/attention.py @@ -0,0 +1,184 @@ +"""GLM-5.3-Flash NoPE Multi-head Latent Attention with a kpool DSA indexer. + +MLA weight-absorption as in glm_moe_dsa (kv_b absorbed into Q and onto the +output; the paged pool stores one latent row per token) with two GLM-5.3 +differences: + +* **NoPE** (``mla_use_nope``, ``qk_rope_head_dim == 0``): no rotary embedding + anywhere in the main attention -- Q is all-nope [T, H, 256], the latent row is + bare ckv (512, no kpe half). All rope plumbing degenerates to zero-width + tensors, which the DSA backend's cat/scatter handle natively. Positional + information enters ONLY through the indexer's pool-compression APE. +* **kpool indexer**: every DSA layer owns its indexer (no IndexShare). The + indexer K cache stores one entry per ``index_kpool`` (4) tokens: a per-channel + ``softmax(gate + ape)``-weighted sum of the raw keys. Scoring runs at pool + granularity (select_k = index_topk / kpool) and the selected pools expand back + to token rows, with the in-progress tail pool force-included + (``index_kpool_always_select_tail``). This module owns the PROJECTIONS (wq_b / + wk+k_norm / weights_proj / compress gate + APE); pooling, scoring, selection + and the tail buffer live in the backend (attention/dsa_indexer_kpool.py). + +Faithfulness note: the reference stack stores pooled entries as +Hadamard-rotated fp8 (a quantization device; the rotation cancels in the dot +product). FreeToken stores pooled entries in bf16 -- mathematically the same +score with strictly less quantization error -- matching the GLM-5.2 precedent +of bf16 indexer keys. +TODO: fp8 index slab (+ Hadamard rotation, upstream parity) to halve the slab bytes. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch +from freetoken.core import get_global_ctx +from freetoken.layers import BaseOP, LinearReplicated, RMSNorm +# Shared with GLM-5.2 (weight.py imports privately from the same package). +from freetoken.models.glm_moe_dsa.attention import _IdxLayerNorm, _make_proj +from freetoken.utils import nvtx_annotate + +if TYPE_CHECKING: + from freetoken.models.config import ModelConfig + + +class Glm5NextIndexer(BaseOP): + """kpool DSA indexer projections (every DSA layer owns one; no rope -- the + checkpoint's NoPE geometry leaves ``qk_rope_head_dim == 0`` so position enters + only via the compression APE). + + Kept bf16 in every quant mode: small (~17 MB/layer) and the top-k boundary is + precision-sensitive (same reasoning as glm_moe_dsa's indexer). + """ + + def __init__(self, config: ModelConfig, layer_id: int): + args = config.glm5_args + self.n_heads = args.index_n_heads + self.head_dim = args.index_head_dim + self.kpool = args.index_kpool + self.wq_b = LinearReplicated( + args.q_lora_rank, self.n_heads * self.head_dim, has_bias=False + ) + self.wk = LinearReplicated(args.hidden_size, self.head_dim, has_bias=False) + self.k_norm = _IdxLayerNorm(self.head_dim, eps=1e-6) + self.weights_proj = LinearReplicated( + args.hidden_size, self.n_heads, has_bias=False + ) + # Pool-compression parameters (checkpoint names, no ".weight" suffix on + # the gate: it is stored as a bare [head_dim, hidden] tensor). + self.index_kpool_compress_gate = torch.empty( + self.head_dim, args.hidden_size + ) + # Per-pool-slot position bias, fp32 (models/weight.py exempts it from the + # model-dtype downcast alongside A_log/dt_bias). + self.index_kpool_compress_ape = torch.empty( + self.kpool, self.head_dim, dtype=torch.float32 + ) + + def compute(self, x: torch.Tensor, q_resid: torch.Tensor) -> "DSAIndexerInputs": + """Per-token indexer projections as a typed inputs object: q [T, Hi, Di], + k [T, Di], weights [T, Hi] fp32, plus the kpool gate scores [T, Di] and + the [kpool, Di] APE parameter (passed per call; the backend keeps no copy).""" + from freetoken.attention.dsa import DSAIndexerInputs + + t = x.shape[0] + q = self.wq_b.forward(q_resid).view(t, self.n_heads, self.head_dim) + k = self.k_norm.forward(self.wk.forward(x)) + w = self.weights_proj.forward(x).float() * (self.n_heads**-0.5) + gate = torch.nn.functional.linear(x, self.index_kpool_compress_gate) + return DSAIndexerInputs( + q=q, k=k, w=w, gate=gate, ape=self.index_kpool_compress_ape + ) + + +class Glm5NextAttention(BaseOP): + def __init__(self, config: ModelConfig, layer_id: int): + args = config.glm5_args + self.layer_id = layer_id + self.indexer = Glm5NextIndexer(config, layer_id) + self.num_heads = args.num_heads + self.qk_nope_head_dim = args.qk_nope_head_dim + self.qk_rope_head_dim = args.qk_rope_head_dim # 0 (NoPE) + self.qk_head_dim = args.qk_head_dim + self.v_head_dim = args.v_head_dim + self.kv_lora_rank = args.kv_lora_rank + assert args.qk_rope_head_dim == 0 and args.mla_nope, ( + "glm5_next attention implements the NoPE geometry; a roped variant " + "would need the glm_moe_dsa rope plumbing back" + ) + + quant = config.attn_quant + self.q_a_proj = _make_proj(quant, args.hidden_size, args.q_lora_rank) + self.q_a_layernorm = RMSNorm(args.q_lora_rank, eps=args.norm_eps) + self.q_b_proj = _make_proj( + quant, args.q_lora_rank, self.num_heads * self.qk_head_dim + ) + # NoPE: kv_a projects to bare ckv (no +qk_rope_head_dim rows). + self.kv_a_proj_with_mqa = _make_proj( + quant, args.hidden_size, self.kv_lora_rank + ) + self.kv_a_layernorm = RMSNorm(self.kv_lora_rank, eps=args.norm_eps) + # kv_b stays bf16 in every mode (bmm absorption operand, not a Linear). + self.kv_b_proj = LinearReplicated( + self.kv_lora_rank, + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), + has_bias=False, + ) + self.o_proj = _make_proj( + quant, self.num_heads * self.v_head_dim, args.hidden_size + ) + self._w_uk: torch.Tensor | None = None + self._w_uv: torch.Tensor | None = None + + def _kv_b(self) -> tuple[torch.Tensor, torch.Tensor]: + """Per-head kv_b split in bmm-ready bf16 layout (same contract and + prepare_for_runtime budgeting as glm_moe_dsa; see that module).""" + if self._w_uk is None: + w = self.kv_b_proj.weight.view( + self.num_heads, + self.qk_nope_head_dim + self.v_head_dim, + self.kv_lora_rank, + ) + self._w_uk = w[:, : self.qk_nope_head_dim, :].contiguous() + self._w_uv = w[:, self.qk_nope_head_dim :, :].transpose(1, 2).contiguous() + return self._w_uk, self._w_uv + + def prepare_for_runtime(self) -> None: + self._kv_b() + self.kv_b_proj.weight = None # checkpoint layout freed; repacked forms serve + + @nvtx_annotate("MLA") + def forward(self, x: torch.Tensor) -> torch.Tensor: + ctx = get_global_ctx() + t = x.shape[0] + w_uk, w_uv = self._kv_b() + + q_a_resid = self.q_a_layernorm.forward(self.q_a_proj.forward(x)) + q = self.q_b_proj.forward(q_a_resid) + # NoPE: the whole head is the nope part; no rope split, no rope kernel. + q_nope = q.view(t, self.num_heads, self.qk_head_dim) + + c_kv = self.kv_a_layernorm.forward(self.kv_a_proj_with_mqa.forward(x)) + + # Absorb kv_b's k-part into the query: q_nope[H,T,nope] @ W_uk[H,nope,lora]. + q_absorbed = torch.bmm(q_nope.transpose(0, 1).contiguous(), w_uk).transpose(0, 1) + + indexer_inputs = ( + self.indexer.compute(x, q_a_resid) + if getattr(ctx.attn_backend, "dsa_enabled", False) + else None + ) + + # Zero-width rope halves: cat/scatter no-ops in the backend. + q_pe = q.new_empty(t, self.num_heads, 0) + k_rope = q.new_empty(t, 0) + o_latent = ctx.attn_backend.mla_forward( + q_absorbed.contiguous(), q_pe, c_kv.contiguous(), k_rope, + self.layer_id, ctx.batch, indexer_inputs=indexer_inputs, + ) # [T, H, kv_lora_rank] + + # Absorb kv_b's v-part onto the output: o_latent[H,T,lora] @ W_uv_t[H,lora,v]. + o = torch.bmm(o_latent.transpose(0, 1).contiguous(), w_uv).transpose(0, 1) + return self.o_proj.forward(o.reshape(t, self.num_heads * self.v_head_dim)) + + +__all__ = ["Glm5NextAttention", "Glm5NextIndexer"] diff --git a/python/freetoken/models/glm5_next/config.py b/python/freetoken/models/glm5_next/config.py new file mode 100644 index 0000000000..294ef579ac --- /dev/null +++ b/python/freetoken/models/glm5_next/config.py @@ -0,0 +1,198 @@ +"""Engine-facing config for GLM-5.3-Flash (``glm5_next``). + +Two attention groups, ordered by first layer id: + +* ``linear`` -- 34 KDA layers (``LinearGatedDeltaGroupConfig`` with + ``variant="kda"``). KDA's state geometry coincides with GDN's (conv width + ``2*K*dk + V*dv`` == q|k|v at ``H*d`` each; recurrent ``[H, d, d]``), so the + existing ``LinearStatePool`` shapes serve it unchanged. +* ``full`` -- 11 DSA layers (``FullAttentionGroupConfig`` with ``mla=True`` and + the indexer dims). The MLA is NoPE (``qk_rope_head_dim == 0``): the latent row + is bare ckv (512), and the indexer K slab is kpool-compressed + (``index_kpool=4`` -> tokens/4 stored entries; the spec's ``index_ratio`` + drives both the pool factory and the KV cost model). + +MoE routing mirrors glm_moe_dsa (sigmoid ``noaux_tc``, shared expert, routed +scaling 2.5) with 288 routed experts and a clamped SwiGLU +(``swiglu_limit=10``). Everything model-specific beyond ``ModelConfig`` rides in +``glm5_args`` (``Glm5NextArgs``). + +Resident-weight quantization (attn/dense/lm_head fp8-at-load, the GLM-5.2 +bandwidth trick) is OFF by default: the NVFP4 exports deliberately quantize only +the routed experts (attention / shared expert / lm_head ship bf16 behind the +quantization_config ignore list), and a serving engine must not override the +checkpoint author's precision decision silently -- the checkpoint is served +as-is, like vLLM/sglang. FREETOKEN_GLM5_ATTN_FP8=1 / FREETOKEN_GLM5_MLP_FP8=1 +opt into the W8A16 requantization (measured decode 36 -> 45.5 tok/s on the +hybrid reference setup; the win needs CUDA graphs -- launch-bound eager decode +gets slower). +""" + +from __future__ import annotations + +import os +from typing import Any + +from freetoken.models.config import ( + FullAttentionGroupConfig, + LinearGatedDeltaGroupConfig, + ModelConfig, + RotaryConfig, + detect_expert_quant, +) + +from .args import load_args + +# Load-time W8A16 fp8 for resident weights: default OFF, env opt-in (see module +# docstring for the rationale and measured numbers). attn covers the KDA +# in_proj_qkv/o_proj + DSA projections (the precision-sensitive b|f_a|g_a gate +# slice stays bf16, see kda.py); mlp covers dense MLPs, shared expert, lm_head. +_ATTN_FP8 = os.getenv("FREETOKEN_GLM5_ATTN_FP8", "0") != "0" +_MLP_FP8 = os.getenv("FREETOKEN_GLM5_MLP_FP8", "0") != "0" + + +def _dsa_on(args, dsa_layer_ids) -> bool: + """DSA serving switch, resolved ONCE into the attention-group spec (the pool + factory, KV cost model, and backend read the spec, never the env).""" + return ( + len(dsa_layer_ids) > 0 + and args.index_topk > 0 + and args.index_head_dim > 0 + and os.getenv("FREETOKEN_GLM5_DSA", "1") != "0" + ) + + +def parse_config(hf_config: Any) -> ModelConfig: + args = load_args(hf_config) + text = getattr(hf_config, "text_config", hf_config) + + num_layers = len(args.layer_types) + # Dev/testing only: cap the layer count so the forward path / KV / offload cache + # can be exercised without the full ~175 GB of experts. Unset in normal use. + _cap = os.environ.get("FREETOKEN_GLM5_MAX_LAYERS") + if _cap: + num_layers = min(num_layers, int(_cap)) + + kda_ids = tuple(i for i in args.kda_layer_ids if i < num_layers) + dsa_ids = tuple(i for i in args.dsa_layer_ids if i < num_layers) + + # NoPE: the main attention carries no rotary dims (rotary_dim == 0); rope + # survives only in the indexer geometry (args.rope_theta / interleave). + rotary_config = RotaryConfig( + head_dim=args.qk_head_dim, + rotary_dim=args.qk_rope_head_dim, + max_position=args.max_position, + base=args.rope_theta, + scaling=None, + ) + latent_dim = args.latent_dim # 512: bare ckv, no kpe rows + + dsa_on = _dsa_on(args, dsa_ids) + # Each DSA layer owns its indexer ("full" in indexer_types); count only the + # layers that actually exist under a dev layer cap. + num_index_layers = ( + sum( + 1 + for i in dsa_ids + if i < len(args.indexer_types) and args.indexer_types[i] == "full" + ) + if dsa_on + else 0 + ) + + linear_group = LinearGatedDeltaGroupConfig( + name="linear", + layer_ids=kda_ids, + # KDA: one qk-sized and one v-sized head set (H=64, d=128 each). Mapping onto + # the GDN field names keeps LinearStatePool's conv-width formula exact: + # 2*K*dk + V*dv = (q|k) + v = 3 * H * d. + num_key_heads=args.linear_num_heads, + num_value_heads=args.linear_num_heads, + key_head_dim=args.linear_head_dim, + value_head_dim=args.linear_head_dim, + conv_kernel_dim=args.linear_conv_kernel_dim, + output_gate="sigmoid", # KDA o_norm gates with sigmoid (kda.py) + variant="kda", + ) + full_group = FullAttentionGroupConfig( + name="full", + layer_ids=dsa_ids, + num_kv_heads=1, # single shared MLA latent + head_dim=latent_dim, + rotary_config=rotary_config, + mla=True, + index_head_dim=args.index_head_dim if dsa_on else 0, + num_index_layers=num_index_layers, + index_ratio=(args.index_kpool if dsa_on and args.index_kpool_compress else 1), + ) + groups = tuple( + sorted( + (linear_group, full_group), + key=lambda g: g.layer_ids[0] if g.layer_ids else 1 << 30, + ) + ) + + # The MLP layout is a dense prefix + sparse tail; ModelConfig models exactly that + # via first_k_dense_replace, so assert the checkpoint matches before collapsing. + mlp_types = args.mlp_layer_types[:num_layers] + first_dense = next( + (i for i, t in enumerate(mlp_types) if t == "sparse"), len(mlp_types) + ) + assert all(t == "dense" for t in mlp_types[:first_dense]) and all( + t == "sparse" for t in mlp_types[first_dense:] + ), f"mlp_layer_types is not a dense-prefix layout: {mlp_types}" + + return ModelConfig( + num_layers=num_layers, + num_qo_heads=args.num_heads, + num_kv_heads=1, + head_dim=latent_dim, + hidden_size=args.hidden_size, + vocab_size=text.vocab_size, + intermediate_size=text.intermediate_size, + # hidden_act stands proxy for the EXPERT activation everywhere the engine + # gates on it (NVFP4 backend selection, CPU-executor capability): GLM-5.3 + # experts and dense MLPs both run clamped SwiGLU when swiglu_limit is set, + # even though the HF config still says "silu". Passing "silu" through would + # let auto-selection repack experts into the marlin/b12x silu-only epilogue. + hidden_act=( + "swiglu_clamp" if args.swiglu_limit is not None else text.hidden_act + ), + rms_norm_eps=args.norm_eps, + tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)), + rotary_config=rotary_config, + attention_groups=groups, + num_experts=( + getattr(text, "n_routed_experts", None) or getattr(text, "num_experts", 0) + ), + num_experts_per_tok=( + getattr(text, "num_experts_per_tok", None) + or getattr(text, "num_experts_per_token", 0) + ), + moe_intermediate_size=getattr(text, "moe_intermediate_size", 0) + or text.intermediate_size, + norm_topk_prob=bool(getattr(text, "norm_topk_prob", True)), + model_type=getattr(hf_config, "model_type", "glm5_next"), + architectures=getattr( + hf_config, "architectures", ["Glm5NextForConditionalGeneration"] + ), + moe_enabled=True, + expert_quant=detect_expert_quant(hf_config), + first_k_dense_replace=first_dense, + n_shared_experts=int(getattr(text, "n_shared_experts", 0) or 0), + routed_scaling_factor=float(getattr(text, "routed_scaling_factor", 1.0)), + n_group=int(getattr(text, "n_group", 1) or 1), + topk_group=int(getattr(text, "topk_group", 1) or 1), + attn_sm_scale=args.qk_head_dim**-0.5, + has_attn_bias=bool(getattr(text, "attention_bias", False)), + swiglu_limit=args.swiglu_limit, + attn_quant="fp8_pertensor" if _ATTN_FP8 else "none", + dense_quant="fp8_pertensor" if _MLP_FP8 else "none", + lm_head_quant="fp8_pertensor" if _MLP_FP8 else "none", + vision_config=None, # text-only milestone; model.visual.* weights are dropped + image_token_id=getattr(hf_config, "image_token_id", None), + glm5_args=args, + ) + + +__all__ = ["parse_config"] diff --git a/python/freetoken/models/glm5_next/kda.py b/python/freetoken/models/glm5_next/kda.py new file mode 100644 index 0000000000..1d44e3a44b --- /dev/null +++ b/python/freetoken/models/glm5_next/kda.py @@ -0,0 +1,252 @@ +"""GLM-5.3-Flash KDA (Kimi Delta Attention) op. + +Per-channel gated delta rule over H=64 heads of D=128, with SEPARATE q/k/v short +convolutions (vs GDN's one fused conv), a low-rank forget gate (``f_a`` -> +``f_b`` -> raw per-channel logits; the bounded safe gate ``lower_bound * +sigmoid(exp(A_log) * (g + dt_bias))`` is computed inside the kernels), a +per-head sigmoid beta (``b_proj``), and a sigmoid-gated output RMSNorm +(``o_norm`` gated by ``g_b(g_a(x))``). + +State lives in ``ctx.linear_state_pool`` exactly like GDN: the conv state is the +merged q|k|v stream (width 3*H*D == the pool's ``2*K*dk + V*dv``) and the +recurrent state is one [D, D] matrix per head, stored in the KERNEL's [V, K] +layout (coincides with the pool's [K, V] declaration because D_k == D_v; same +convention as GDN, see qwen3_5_moe/gdn.py). + +Kernels: ``fused_recurrent_kda`` decodes with in-kernel gate/beta/l2norm and +per-slot state read/write (slot 0 is its NULL sentinel == the pool's padding +slot); ``chunk_kda_with_fused_gate`` prefills from an explicitly gathered +initial state and returns the final state, which this op scatters back (the +kernel CLOBBERS its v buffer -- v here is an ephemeral conv output, so that is +free). Hybrid-radix track snapshots ride the per-chunk h (``return_h``). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch +from freetoken.core import get_global_ctx +from freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen +from freetoken.layers import BaseOP, LinearColParallelMerged, LinearReplicated +from freetoken.utils import nvtx_annotate + +if TYPE_CHECKING: + from freetoken.models.config import ModelConfig + + +class _DepthwiseConv1d(BaseOP): + """Merged q|k|v depthwise conv weight ``[3*H*D, 1, K]`` (key ``conv1d.weight``; + the loader concatenates the checkpoint's q/k/v_conv1d along channels).""" + + def __init__(self, conv_dim: int, kernel: int): + self.weight = torch.empty(conv_dim, 1, kernel) + + +class _GatedRMSNormSigmoid(BaseOP): + """RMSNorm(x) * sigmoid(z), fused (KDA's o_norm; GDN's variant gates with silu).""" + + def __init__(self, dim: int, eps: float): + self.weight = torch.empty(dim) + self.eps = eps + + def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor: + from freetoken.kernel.fla import rms_norm_gated + + return rms_norm_gated( + x=x, weight=self.weight, bias=None, z=z, eps=self.eps, + is_rms_norm=True, norm_before_gate=True, activation="sigmoid", + ) + + +class Glm5NextKDA(BaseOP): + """KDA op; state is held in ``ctx.linear_state_pool`` keyed by the request's + linear slot (``FLAMetadata.cache_indices``). Parameter names follow the + checkpoint modulo two load-time fusions (see weight.py): ``in_proj`` is + q|k|v|b|f_a|g_a concatenated, ``conv1d`` is q|k|v conv concatenated.""" + + def __init__(self, config: ModelConfig, layer_id: int): + args = config.glm5_args + self.layer_id = layer_id + self.num_heads = args.linear_num_heads + self.head_dim = args.linear_head_dim + self.proj_size = self.num_heads * self.head_dim # H * D + self.conv_dim = 3 * self.proj_size # merged q|k|v stream + self.conv_kernel_size = args.linear_conv_kernel_dim + self.lower_bound = args.linear_lower_bound + self.scale = self.head_dim**-0.5 + + p, h, d = self.proj_size, self.num_heads, self.head_dim + # q|k|v dominate the resident read (3 * 8192 x 4096 = 201 MB/layer bf16 -- + # the single biggest dense-weight stream in the model); under the fp8 + # resident mode they split into their own W8A16 GEMM while the small, + # precision-sensitive gate projections (b|f_a|g_a) stay bf16 (the GDN + # qkvz/ba split precedent). BF16 mode keeps the single fused GEMM. + self._fp8 = config.attn_quant == "fp8_pertensor" + self._bfg_split = [h, d, d] + if self._fp8: + from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged + + self.in_proj_qkv = Fp8PerTensorColMerged( + args.hidden_size, [p, p, p], has_bias=False + ) + self.in_proj_bfg = LinearColParallelMerged( + args.hidden_size, self._bfg_split, has_bias=False + ) + else: + self._in_proj_split = [p, p, p, h, d, d] + self.in_proj = LinearColParallelMerged( + args.hidden_size, self._in_proj_split, has_bias=False + ) + # Low-rank gate up-projections (128 -> 8192): forget gate and output gate. + self.f_b_proj = LinearReplicated(d, p, has_bias=False) + self.g_b_proj = LinearReplicated(d, p, has_bias=False) + self.conv1d = _DepthwiseConv1d(self.conv_dim, self.conv_kernel_size) + # Gate params stay fp32 (exp/sigmoid precision; the kernels read fp32). + # models/weight.py exempts *.A_log / *.dt_bias from the model-dtype downcast. + self.A_log = torch.empty(h, dtype=torch.float32) + self.dt_bias = torch.empty(p, dtype=torch.float32) + self.o_norm = _GatedRMSNormSigmoid(d, eps=args.norm_eps) + # o_proj follows the resident quant mode (rationale: the split comment + # at _fp8 above); f_b/g_b stay bf16 alongside the gate slice. + from .attention import _make_proj + + self.o_proj = _make_proj(config.attn_quant, p, args.hidden_size) + + def _conv_weight(self) -> torch.Tensor: + return self.conv1d.weight.squeeze(1) # [conv_dim, kernel] + + def _write_track_snapshot(self, pool, li, conv_in, h, fla) -> None: + """Hybrid-radix: snapshot recurrent + conv state at the chunk-aligned track + boundary into a donatable pool slot (same contract as GDN, see + qwen3_5_moe/gdn.py). h rows are the kernel's per-chunk [V, K] states -- + a direct copy into the pool's [K, V] slots (D_k == D_v).""" + rec = pool.recurrent_states[li] + rec.index_copy_(0, fla.track_dst, h[0, fla.track_h_row].to(rec.dtype)) + cv = pool.conv_states[li] + conv_win = conv_in[fla.track_conv_src].transpose(-1, -2).contiguous() + cv.index_copy_(0, fla.track_dst, conv_win.to(cv.dtype)) + + @nvtx_annotate("KDA") + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + ctx = get_global_ctx() + batch = ctx.batch + pool = ctx.linear_state_pool + total = hidden_states.shape[0] + dtype = hidden_states.dtype + h, d, p = self.num_heads, self.head_dim, self.proj_size + + fla = batch.fla_metadata + if fla is None: + from freetoken.attention.linear import build_fla_metadata + + fla = build_fla_metadata(batch, hidden_states.device) + batch.fla_metadata = fla + + if self._fp8: + conv_in = self.in_proj_qkv.forward(hidden_states) + b, f_a, g_a = torch.split( + self.in_proj_bfg.forward(hidden_states), self._bfg_split, dim=-1 + ) + else: + proj = self.in_proj.forward(hidden_states) + conv_in, b, f_a, g_a = torch.split( + proj, [self.conv_dim, h, d, d], dim=-1 + ) + g1 = self.f_b_proj.forward(f_a) # raw forget-gate logits [T, H*D] + g2 = self.g_b_proj.forward(g_a) # output-gate logits [T, H*D] + li = pool.local_index(self.layer_id) + + if batch.is_decode: + mixed = causal_conv1d_decode( + conv_in, pool.conv_states[li], self._conv_weight(), fla.cache_indices + ) + bsz = mixed.shape[0] + q, k, v = ( + t.reshape(1, bsz, h, d).to(dtype) + for t in torch.split(mixed, [p, p, p], dim=-1) + ) + core_out, _ = _fused_recurrent( + q, k, v, + g=g1.view(1, bsz, h, d), + beta=b.view(1, bsz, h), + state_pool=pool.recurrent_states[li], + indices=fla.cache_indices, + cu_seqlens=fla.cu_seqlens, + a_log=self.A_log, + dt_bias=self.dt_bias, + lower_bound=self.lower_bound, + scale=self.scale, + ) + else: + x = conv_in.transpose(0, 1).contiguous() # [conv_dim, total] + mixed = causal_conv1d_varlen( + x, self._conv_weight(), pool.conv_states[li], + fla.cu_seqlens, fla.cache_indices, fla.has_initial_state, + ).transpose(0, 1) + q, k, v = ( + t.reshape(1, total, h, d).to(dtype) + for t in torch.split(mixed, [p, p, p], dim=-1) + ) + # Fresh sequences start from a zeroed slot; then gather every request's + # initial state (the chunk kernel takes it dense, [N, H, D, D]). + rec = pool.recurrent_states[li] + if fla.fresh_state_indices is not None: + rec.index_fill_(0, fla.fresh_state_indices, 0.0) + slot_ids = fla.cache_indices.long() + initial = rec.index_select(0, slot_ids) + + from freetoken.kernel.fla import chunk_kda_with_fused_gate + + track = fla.track_dst is not None + result = chunk_kda_with_fused_gate( + q=q, k=k, v=v, # NOTE: v (ephemeral conv output) is clobbered + raw_g=g1.view(1, total, h, d), + beta=b.float().sigmoid().view(1, total, h), + A_log=self.A_log, + g_bias=self.dt_bias, + scale=self.scale, + initial_state=initial, + output_final_state=True, + use_qk_l2norm_in_kernel=True, + cu_seqlens=fla.cu_seqlens, + safe_gate=True, + lower_bound=self.lower_bound, + return_h=track, + ) + if track: + core_out, final_state, chunk_h = result + self._write_track_snapshot(pool, li, conv_in, chunk_h, fla) + else: + core_out, final_state = result + rec.index_copy_(0, slot_ids, final_state.to(rec.dtype)) + + core_out = core_out.reshape(-1, d) + out = self.o_norm.forward(core_out, g2.reshape(-1, d)).reshape(total, -1) + return self.o_proj.forward(out.to(dtype)) + + +def _fused_recurrent( + q, k, v, g, beta, state_pool, indices, cu_seqlens, + a_log, dt_bias, lower_bound, scale, +): + """Decode via the vendored recurrent kernel: gate + beta-sigmoid + q/k l2norm + in-kernel, state read/written in place at ``indices`` (int32, 1 token/req).""" + from freetoken.kernel.fla import fused_recurrent_kda + + return fused_recurrent_kda( + q=q, k=k, v=v, g=g, beta=beta, + scale=scale, + initial_state=state_pool, + use_qk_l2norm_in_kernel=True, + cu_seqlens=cu_seqlens, + ssm_state_indices=indices, + sigmoid_beta=True, + a_log=a_log, + g_bias=dt_bias, + compute_gate=True, + lower_bound=lower_bound, + ) + + +__all__ = ["Glm5NextKDA"] diff --git a/python/freetoken/models/glm5_next/mlp.py b/python/freetoken/models/glm5_next/mlp.py new file mode 100644 index 0000000000..1ee508bf13 --- /dev/null +++ b/python/freetoken/models/glm5_next/mlp.py @@ -0,0 +1,47 @@ +"""Clamped-SwiGLU MLP for GLM-5.3-Flash's leading dense layers and shared experts. + +Same shape as glm_moe_dsa's GlmDsaGatedMLP (bf16 in the NVFP4 checkpoint; +optional W8A16 fp8-at-load via ``ModelConfig.dense_quant``), but the activation +is the GLM-5.3 clamped SwiGLU (``swiglu_limit``): +``clamp(gate, max=L) * sigmoid(gate_clamped) * clamp(up, +-L)``. +""" + +from __future__ import annotations + + +import torch +from freetoken.layers import BaseOP, swiglu_clamp_and_mul +from freetoken.utils import nvtx_annotate + +from .attention import _make_proj + + +class Glm5NextGatedMLP(BaseOP): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + quant: str = "none", + swiglu_limit: float | None = None, + ): + self.gate_proj = _make_proj(quant, hidden_size, intermediate_size) + self.up_proj = _make_proj(quant, hidden_size, intermediate_size) + self.down_proj = _make_proj(quant, intermediate_size, hidden_size) + self.swiglu_limit = swiglu_limit + + @nvtx_annotate("MLP") + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate = self.gate_proj.forward(x) + up = self.up_proj.forward(x) + del x + if self.swiglu_limit is None: + import torch.nn.functional as F + + return self.down_proj.forward(F.silu(gate) * up) + gated = swiglu_clamp_and_mul( + torch.cat([gate, up], dim=-1), alpha=1.0, limit=self.swiglu_limit + ) + return self.down_proj.forward(gated) + + +__all__ = ["Glm5NextGatedMLP"] diff --git a/python/freetoken/models/glm5_next/model.py b/python/freetoken/models/glm5_next/model.py new file mode 100644 index 0000000000..4c011dd6c1 --- /dev/null +++ b/python/freetoken/models/glm5_next/model.py @@ -0,0 +1,207 @@ +"""GLM-5.3-Flash (glm5_next) model: hybrid KDA/DSA decoder with mHC residual streams. + +Layer layout comes from the checkpoint's ``layer_types`` (34 KDA linear-attention +layers, 11 NoPE-MLA/DSA layers at 3:1) and ``mlp_layer_types`` (3 dense + 42 MoE). +The residual stream is mHC-widened to ``hc_mult`` (4) parallel streams: + + layer 0: residual = hc_expand(x); (post, comb, x) = mhc_pre(residual, hc_attn_*) + each sublayer boundary fuses the previous hc_post with the next hc_pre + (mhc_fused_post_pre), and the sublayer input is RMS-normed AFTER the mix + (the reference fuses the norm into its hc kernels; decomposed here, same math). + last layer: x = mhc_post(...); x = hc_contract(x) -> final norm -> lm_head. + +The deferred (post, comb) pair threads through the layer loop exactly like +glm_moe_dsa's (x, residual) pair. lm_head quant mirrors glm_moe_dsa (optional +W8A16 fp8 at load; the ~1.2 GiB bf16 head is read every decode step). +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Tuple + +import torch +from freetoken.core import get_global_ctx +from freetoken.layers import ( + BaseOP, + OPList, + ParallelLMHead, + RMSNorm, + VocabParallelEmbedding, +) +from freetoken.layers.mhc import hc_contract, hc_expand, mhc_fused_post_pre, mhc_post, mhc_pre +from freetoken.models.blocks import BaseLLMModel +from freetoken.utils import nvtx_annotate + +from .attention import Glm5NextAttention +from .kda import Glm5NextKDA +from .mlp import Glm5NextGatedMLP +from .moe import Glm5NextSparseBlock + +if TYPE_CHECKING: + from freetoken.models.config import ModelConfig + + +class Glm5Fp8LMHead(ParallelLMHead): + """W8A16 lm_head (fp8-e4m3 weight + per-row scale, quantized at load); the + full-vocab decode GEMV reads the whole head every step -- fp8 halves it. + Same contract as glm_moe_dsa's GlmFp8LMHead.""" + + def __init__(self, num_embeddings: int, embedding_dim: int): + super().__init__(num_embeddings, embedding_dim, tie_word_embeddings=False) + self.weight = torch.empty(num_embeddings, embedding_dim, dtype=torch.float8_e4m3fn) + self.weight_scale = torch.empty(num_embeddings, dtype=torch.float32) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear + + batch = get_global_ctx().batch + if batch.is_prefill: + indices = batch.attn_metadata.get_last_indices(batch.size) + x = x[indices].contiguous() + return fp8_pertensor_linear(x, self.weight, self.weight_scale) + + +class Glm5NextDecoderLayer(BaseOP): + def __init__(self, config: ModelConfig, layer_id: int): + args = config.glm5_args + self._layer_id = layer_id + self._is_last = layer_id == config.num_layers - 1 + self.mhc = args.mhc + self._n = args.mhc_num_residual_streams + self._hc_eps = args.hc_eps + self._rms_eps = args.norm_eps + self._post_mult = args.mhc_post_mult_value + self._sinkhorn = args.mhc_sinkhorn_iterations + + if args.is_kda_layer(layer_id): + self.self_attn: BaseOP = Glm5NextKDA(config, layer_id) + else: + self.self_attn = Glm5NextAttention(config, layer_id) + if layer_id >= config.first_k_dense_replace: + self.mlp: BaseOP = Glm5NextSparseBlock(config, layer_id) + else: + self.mlp = Glm5NextGatedMLP( + config.hidden_size, config.intermediate_size, + quant=config.dense_quant, swiglu_limit=config.swiglu_limit, + ) + self.input_layernorm = RMSNorm(size=config.hidden_size, eps=args.norm_eps) + self.post_attention_layernorm = RMSNorm(size=config.hidden_size, eps=args.norm_eps) + + if self.mhc: + n, hidden = self._n, config.hidden_size + mix = 2 * n + n * n + # fp32 mHC weights (models/weight.py exempts hc_* from the dtype downcast). + self.hc_attn_fn = torch.empty(mix, n * hidden, dtype=torch.float32) + self.hc_attn_base = torch.empty(mix, dtype=torch.float32) + self.hc_attn_scale = torch.empty(3, dtype=torch.float32) + self.hc_ffn_fn = torch.empty(mix, n * hidden, dtype=torch.float32) + self.hc_ffn_base = torch.empty(mix, dtype=torch.float32) + self.hc_ffn_scale = torch.empty(3, dtype=torch.float32) + + def _pre(self, residual, fn, scale, base): + # Layer 0's standalone pre rides the fused kernel too (HAS_POST=False + # path; x/post/comb are the no-post sentinels) -- same dispatch, same + # numerics, and the kernel wins at every batch size (see layers/mhc.py). + if residual.is_cuda: + _, post, comb, x = mhc_fused_post_pre( + residual.new_empty(residual.shape[0], residual.shape[-1]), + residual, None, None, fn, scale, base, + self._rms_eps, self._hc_eps, self._post_mult, self._sinkhorn, + ) + return post, comb, x + return mhc_pre( + residual, fn, scale, base, + self._rms_eps, self._hc_eps, self._post_mult, self._sinkhorn, + ) + + def _fused(self, x, residual, post, comb, fn, scale, base): + return mhc_fused_post_pre( + x, residual, post, comb, fn, scale, base, + self._rms_eps, self._hc_eps, self._post_mult, self._sinkhorn, + ) + + @nvtx_annotate("Layer_{}", layer_id_field="_layer_id") + def forward( + self, + x: torch.Tensor, + residual: torch.Tensor | None, + post: torch.Tensor | None, + comb: torch.Tensor | None, + ) -> Tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]: + if post is None: + if residual is None: + residual = hc_expand(x, self._n) + post, comb, x = self._pre( + residual, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base + ) + else: + residual, post, comb, x = self._fused( + x, residual, post, comb, + self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base, + ) + x = self.input_layernorm.forward(x) + x = self.self_attn.forward(x) + + residual, post, comb, x = self._fused( + x, residual, post, comb, + self.hc_ffn_fn, self.hc_ffn_scale, self.hc_ffn_base, + ) + x = self.post_attention_layernorm.forward(x) + x = self.mlp.forward(x) + + if self._is_last: + x = mhc_post(x, residual, post, comb) + return hc_contract(x), None, None, None + return x, residual, post, comb + + +class Glm5NextModel(BaseOP): + def __init__(self, config: ModelConfig): + self.embed_tokens = VocabParallelEmbedding( + num_embeddings=config.vocab_size, + embedding_dim=config.hidden_size, + ) + self.layers = OPList( + [Glm5NextDecoderLayer(config, i) for i in range(config.num_layers)] + ) + self.norm = RMSNorm(size=config.hidden_size, eps=config.rms_norm_eps) + + def forward(self, input_ids: torch.Tensor) -> torch.Tensor: + x = self.embed_tokens.forward(input_ids) + residual = post = comb = None + for layer in self.layers.op_list: + x, residual, post, comb = layer.forward(x, residual, post, comb) + return self.norm.forward(x) + + +class Glm5NextForCausalLM(BaseLLMModel): + def __init__(self, config: ModelConfig): + self._config = config + self.model = Glm5NextModel(config) + if config.lm_head_quant == "fp8_pertensor" and not config.tie_word_embeddings: + self.lm_head: BaseOP = Glm5Fp8LMHead( + num_embeddings=config.vocab_size, embedding_dim=config.hidden_size + ) + else: + self.lm_head = ParallelLMHead( + num_embeddings=config.vocab_size, + embedding_dim=config.hidden_size, + tie_word_embeddings=config.tie_word_embeddings, + tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None, + ) + + def prepare_for_runtime(self) -> None: + """Post-load, pre-KV-sizing hook: materialize the DSA layers' bmm-ready + kv_b splits and free the checkpoint-layout originals (glm_moe_dsa + precedent).""" + for layer in self.model.layers.op_list: + if isinstance(layer.self_attn, Glm5NextAttention): + layer.self_attn.prepare_for_runtime() + torch.cuda.empty_cache() + + def forward(self) -> torch.Tensor: + output = self.model.forward(get_global_ctx().batch.input_ids) + return self.lm_head.forward(output) + + +__all__ = ["Glm5NextForCausalLM"] diff --git a/python/freetoken/models/glm5_next/moe.py b/python/freetoken/models/glm5_next/moe.py new file mode 100644 index 0000000000..038ffc0dab --- /dev/null +++ b/python/freetoken/models/glm5_next/moe.py @@ -0,0 +1,92 @@ +"""GLM-5.3-Flash sparse MoE block. + +Routing is identical to glm_moe_dsa (sigmoid scores + selection-only +``e_score_correction_bias``, optional group-limited top-k, gather unbiased +scores, renormalize, scale by ``routed_scaling_factor``); the deltas are the +expert count (288, top-8) and the clamped-SwiGLU activation (``swiglu_limit`` = +10), which rides ``make_moe_layer``'s ``extra_attrs`` into the offload kernels +(triton NVFP4 in-GPU, generic-epilogue CPU GEMV). The marlin/b12x borrowed +kernels hard-code silu; the backend selector already falls back to triton for +non-silu experts. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Tuple + +import torch +import torch.nn.functional as F +from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer + +from .mlp import Glm5NextGatedMLP + +if TYPE_CHECKING: + from freetoken.models.config import ModelConfig + +TopK = Tuple[torch.Tensor, torch.Tensor] + + +class Glm5NextSparseBlock(BaseOP): + def __init__(self, config: ModelConfig, layer_id: int): + self.top_k = config.num_experts_per_tok + self.num_experts = config.num_experts + self.norm_topk_prob = config.norm_topk_prob + self.routed_scaling_factor = config.routed_scaling_factor + self.n_group = config.n_group + self.topk_group = config.topk_group + + self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False) + self.e_score_correction_bias = torch.empty(config.num_experts, dtype=torch.float32) + + # The offload cache indexes experts by MoE layer (global minus dense prefix). + self.experts = make_moe_layer( + config, + layer_id=layer_id - config.first_k_dense_replace, + activation="swiglu_clamp" if config.swiglu_limit is not None else "silu", + renormalize=config.norm_topk_prob, + extra_attrs={ + "swiglu_limit": config.swiglu_limit, + "hidden_act_alpha": 1.0, # plain sigmoid inside the clamped swiglu + }, + ) + self.shared_experts = Glm5NextGatedMLP( + config.hidden_size, + config.moe_intermediate_size * max(1, config.n_shared_experts), + quant=config.dense_quant, + swiglu_limit=config.swiglu_limit, + ) + + def _group_limited(self, scores_for_choice: torch.Tensor) -> torch.Tensor: + m = scores_for_choice.shape[0] + e, g = self.num_experts, self.n_group + group_scores = scores_for_choice.view(m, g, e // g).topk(2, dim=-1)[0].sum(dim=-1) + group_idx = torch.topk(group_scores, self.topk_group, dim=-1, sorted=False)[1] + group_mask = torch.zeros_like(group_scores) + group_mask.scatter_(1, group_idx, 1.0) + score_mask = group_mask.unsqueeze(-1).expand(m, g, e // g).reshape(m, e) + return scores_for_choice.masked_fill(~score_mask.bool(), float("-inf")) + + def _route(self, hidden_states: torch.Tensor) -> TopK: + # HF computes router logits in fp32 (moe_router_dtype: float32); match exactly. + logits = F.linear(hidden_states.float(), self.gate.weight.float()) + scores = logits.sigmoid() + scores_for_choice = scores + self.e_score_correction_bias.float() + if self.n_group > 1: + scores_for_choice = self._group_limited(scores_for_choice) + _, topk_ids = torch.topk(scores_for_choice, self.top_k, dim=-1) + topk_weights = scores.gather(-1, topk_ids) + if self.norm_topk_prob: + topk_weights = topk_weights / (topk_weights.sum(dim=-1, keepdim=True) + 1e-20) + topk_weights = topk_weights * self.routed_scaling_factor + return topk_weights.to(torch.float32).contiguous(), topk_ids.to(torch.int32).contiguous() + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_tokens, hidden_dim = hidden_states.shape + hidden_states = hidden_states.view(-1, hidden_dim) + topk_weights, topk_ids = self._route(hidden_states) + out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids) + out = out + self.shared_experts.forward(hidden_states) + return out.view(num_tokens, hidden_dim) + + +__all__ = ["Glm5NextSparseBlock"] diff --git a/python/freetoken/models/glm5_next/weight.py b/python/freetoken/models/glm5_next/weight.py new file mode 100644 index 0000000000..fda852256b --- /dev/null +++ b/python/freetoken/models/glm5_next/weight.py @@ -0,0 +1,275 @@ +"""Weight loading for GLM-5.3-Flash (``glm5_next``). + +Supported checkpoints: NVFP4 exports of GLM-5.3-Flash in the multimodal-wrapper +layout (``model.language_model.*``) -- ModelOpt tensor kinds (LibertAIDAI) or +compressed-tensors kinds (RedHatAI), selected by ``quantization_config``. Not +supported: bf16-expert originals (zai-org), text-only key layouts, TP > 1. + +Routed experts go to the offload cache via ``load_nvfp4_expert_sources``; +everything else loads bf16 with keys renamed ``model.language_model.X`` -> +``model.X``. ``model.visual.*`` and the trailing MTP layer are never read. + +Load-time fusions (must mirror the module split orders): + +* KDA ``in_proj`` = q|k|v|b|f_a|g_a projections concatenated on the output axis +* KDA ``conv1d`` = q|k|v depthwise conv weights concatenated on the channel axis + +fp32-kept tensors: ``A_log`` / ``dt_bias``, the mHC ``hc_*`` tensors, the indexer +APE, and the router ``e_score_correction_bias``. Optional W8A16 fp8-at-load +follows ``ModelConfig.attn_quant`` / ``dense_quant`` / ``lm_head_quant`` +(defaults and env opt-ins: see config.py). +""" + +from __future__ import annotations + +import json +import os +import re +from typing import Iterator + +import torch +from freetoken.distributed import get_tp_info +from freetoken.models.glm_moe_dsa.weight import _ShardReader, _quant_fp8_per_row +from freetoken.models.loader import drop_page_cache +from freetoken.models.nvfp4_banks import ( + Nvfp4ExpertSourceSpec, + load_nvfp4_expert_source_banks, +) +from freetoken.utils import cached_load_hf_config, download_hf_weight +from tqdm import tqdm + +from .args import Glm5NextArgs +from .config import parse_config + +# Checkpoint prefix (multimodal wrapper) -> model prefix. +_CKPT = "model.language_model" +_MODEL = "model" + +# MTP-layer experts (layer == num_layers under the full checkpoint) map to None +# alongside the dense prefix; the bank loader skips them. +def _layer_to_bank(layer, config): + return ( + None + if layer < config.first_k_dense_replace or layer >= config.num_layers + else layer - config.first_k_dense_replace + ) + + +# ModelOpt export (LibertAIDAI/GLM-5.3-Flash-NVFP4): weight | weight_scale | +# weight_scale_2 (dequant-side global). +_NVFP4_SOURCE_SPEC = Nvfp4ExpertSourceSpec( + key_pattern=re.compile( + r"^model\.language_model\.layers\.(?P\d+)\.mlp\.experts\.(?P\d+)\." + r"(?Pgate_proj|up_proj|down_proj)\.(?Pweight|weight_scale|weight_scale_2)$" + ), + proj_to_role={"gate_proj": "gate", "up_proj": "up", "down_proj": "down"}, + layer_to_bank=_layer_to_bank, + desc="GLM-5.3 NVFP4 experts", +) + +# llm-compressor export (RedHatAI/GLM-5.3-Flash-NVFP4): weight_packed | +# weight_scale | weight_global_scale (quant-side global -> reciprocal at ingest). +# ``input_global_scale`` (the calibrated W4A4 activation scale) deliberately does +# not match: our routed-expert paths are W4A16 and never quantize activations. +_NVFP4_CT_SOURCE_SPEC = Nvfp4ExpertSourceSpec( + key_pattern=re.compile( + r"^model\.language_model\.layers\.(?P\d+)\.mlp\.experts\.(?P\d+)\." + r"(?Pgate_proj|up_proj|down_proj)\." + r"(?Pweight_packed|weight_global_scale|weight_scale)$" + ), + proj_to_role={"gate_proj": "gate", "up_proj": "up", "down_proj": "down"}, + layer_to_bank=_layer_to_bank, + desc="GLM-5.3 NVFP4 experts (compressed-tensors)", + kind_map={"weight_packed": "weight", "weight_global_scale": "weight_scale_2"}, + global_reciprocal=True, +) + + +def _select_expert_source_spec(model_path: str) -> Nvfp4ExpertSourceSpec: + quant = getattr(cached_load_hf_config(model_path), "quantization_config", None) or {} + get = quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d)) + method = str(get("quant_method") or "").lower() + return _NVFP4_CT_SOURCE_SPEC if method == "compressed-tensors" else _NVFP4_SOURCE_SPEC + +# KDA in_proj fusion order; MUST match Glm5NextKDA._in_proj_split. +_KDA_IN_PROJ = ("q_proj", "k_proj", "v_proj", "b_proj", "f_a_proj", "g_a_proj") + + +def load_nvfp4_expert_sources(model_path: str, config, layer_sink=None): + return load_nvfp4_expert_source_banks( + model_path, + config, + _select_expert_source_spec(model_path), + drop_page_cache=drop_page_cache, + primary=get_tp_info().is_primary(), + layer_sink=layer_sink, + ) + + +def _maybe_fp8(key: str, w: torch.Tensor, fp8: bool): + if fp8: + q, scale = _quant_fp8_per_row(w) + yield f"{key}.weight", q + yield f"{key}.weight_scale", scale + else: + yield f"{key}.weight", w.to(torch.bfloat16) + + +def _iter_kda_layer(reader, layer: int, attn_fp8: bool) -> Iterator[tuple[str, torch.Tensor]]: + src = f"{_CKPT}.layers.{layer}.self_attn" + dst = f"{_MODEL}.layers.{layer}.self_attn" + if attn_fp8: + # fp8 resident: q|k|v (the 201 MB/layer read) as one W8A16 GEMM with + # per-row scales; the small gate projections b|f_a|g_a stay bf16. + qkv = torch.cat( + [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in ("q_proj", "k_proj", "v_proj")], + dim=0, + ) + q, scale = _quant_fp8_per_row(qkv) + yield f"{dst}.in_proj_qkv.weight", q + yield f"{dst}.in_proj_qkv.weight_scale", scale + bfg = torch.cat( + [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in ("b_proj", "f_a_proj", "g_a_proj")], + dim=0, + ) + yield f"{dst}.in_proj_bfg.weight", bfg + else: + # One fused input GEMM: q|k|v|b|f_a|g_a (output-axis concat). + fused = torch.cat( + [reader.get(f"{src}.{p}.weight").to(torch.bfloat16) for p in _KDA_IN_PROJ], dim=0 + ) + yield f"{dst}.in_proj.weight", fused + # One merged depthwise conv over the q|k|v stream (channel-axis concat). + conv = torch.cat( + [reader.get(f"{src}.{p}_conv1d.weight").to(torch.bfloat16) for p in ("q", "k", "v")], + dim=0, + ) + yield f"{dst}.conv1d.weight", conv + for p in ("f_b_proj", "g_b_proj"): + yield f"{dst}.{p}.weight", reader.get(f"{src}.{p}.weight").to(torch.bfloat16) + yield from _maybe_fp8(f"{dst}.o_proj", reader.get(f"{src}.o_proj.weight"), attn_fp8) + # Gate params stay fp32 (the recurrent kernels read them as fp32). + yield f"{dst}.A_log", reader.get(f"{src}.A_log").to(torch.float32) + yield f"{dst}.dt_bias", reader.get(f"{src}.dt_bias").to(torch.float32) + yield f"{dst}.o_norm.weight", reader.get(f"{src}.o_norm.weight").to(torch.bfloat16) + + +def _iter_dsa_layer(reader, layer: int, attn_fp8: bool) -> Iterator[tuple[str, torch.Tensor]]: + src = f"{_CKPT}.layers.{layer}.self_attn" + dst = f"{_MODEL}.layers.{layer}.self_attn" + fp8_projs = ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "o_proj") if attn_fp8 else () + for proj in ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"): + w = reader.get(f"{src}.{proj}.weight") + yield from _maybe_fp8(f"{dst}.{proj}", w, proj in fp8_projs) + for norm in ("q_a_layernorm", "kv_a_layernorm"): + yield f"{dst}.{norm}.weight", reader.get(f"{src}.{norm}.weight").to(torch.bfloat16) + # kpool indexer (every DSA layer owns one). Kept bf16; the APE is fp32. + for proj in ("wq_b", "wk", "weights_proj"): + yield f"{dst}.indexer.{proj}.weight", reader.get( + f"{src}.indexer.{proj}.weight" + ).to(torch.bfloat16) + for part, dtype in ( + ("k_norm.weight", torch.bfloat16), + ("k_norm.bias", torch.bfloat16), + ("index_kpool_compress_gate", torch.bfloat16), + ("index_kpool_compress_ape", torch.float32), + ): + yield f"{dst}.indexer.{part}", reader.get(f"{src}.indexer.{part}").to(dtype) + + +def iter_weights( + model_path: str, + device: torch.device, + *, + include_moe_experts: bool, + include_non_moe: bool, +) -> Iterator[tuple[str, torch.Tensor]]: + assert not include_moe_experts, ( + "GLM-5.3 stores routed experts as NVFP4 and only supports the offload backend; " + "experts are loaded into the offload cache via load_nvfp4_expert_sources()." + ) + assert include_non_moe + if get_tp_info().size > 1: + # The loader emits full fused KDA/DSA tensors; TP sharding (per-head q|k|v|b + # splits, replicated f_a|g_a, row-parallel o_proj) is not implemented yet -- + # same status as every other linear-hybrid / offload-family model in tree. + raise NotImplementedError("glm5_next weight loading currently supports TP=1 only") + config = parse_config(cached_load_hf_config(model_path)) + args: Glm5NextArgs = config.glm5_args + folder = download_hf_weight(model_path) + with open(os.path.join(folder, "model.safetensors.index.json")) as f: + weight_map = json.load(f)["weight_map"] + reader = _ShardReader(folder, weight_map, device) + primary = get_tp_info().is_primary() + attn_fp8 = config.attn_quant == "fp8_pertensor" + mlp_fp8 = config.dense_quant == "fp8_pertensor" + head_fp8 = config.lm_head_quant == "fp8_pertensor" + if primary: + from freetoken.utils import init_logger + + init_logger(__name__).info( + f"GLM-5.3 resident quant: attn={config.attn_quant} dense={config.dense_quant} " + f"lm_head={config.lm_head_quant} (FREETOKEN_GLM5_ATTN_FP8/FREETOKEN_GLM5_MLP_FP8; " + "an FTW conversion records these choices implicitly -- serve with the same flags)" + ) + try: + for layer in tqdm( + range(config.num_layers), + desc="Loading GLM-5.3 dense weights", + disable=not primary, + ): + src = f"{_CKPT}.layers.{layer}" + dst = f"{_MODEL}.layers.{layer}" + if args.is_kda_layer(layer): + yield from _iter_kda_layer(reader, layer, attn_fp8) + else: + yield from _iter_dsa_layer(reader, layer, attn_fp8) + + # mHC mixing tensors, fp32 on every layer. + for hc in ("hc_attn_fn", "hc_attn_base", "hc_attn_scale", + "hc_ffn_fn", "hc_ffn_base", "hc_ffn_scale"): + yield f"{dst}.{hc}", reader.get(f"{src}.{hc}").to(torch.float32) + + for norm in ("input_layernorm", "post_attention_layernorm"): + yield f"{dst}.{norm}.weight", reader.get(f"{src}.{norm}.weight").to( + torch.bfloat16 + ) + + if layer < config.first_k_dense_replace: + for proj in ("gate_proj", "up_proj", "down_proj"): + yield from _maybe_fp8( + f"{dst}.mlp.{proj}", reader.get(f"{src}.mlp.{proj}.weight"), mlp_fp8 + ) + else: + yield f"{dst}.mlp.gate.weight", reader.get(f"{src}.mlp.gate.weight").to( + torch.bfloat16 + ) + yield ( + f"{dst}.mlp.e_score_correction_bias", + # fp32 like HF's router math (the module declares fp32; a bf16 + # cast would perturb top-8 selection on fp32-bias checkpoints). + reader.get(f"{src}.mlp.gate.e_score_correction_bias").to(torch.float32), + ) + for proj in ("gate_proj", "up_proj", "down_proj"): + yield from _maybe_fp8( + f"{dst}.mlp.shared_experts.{proj}", + reader.get(f"{src}.mlp.shared_experts.{proj}.weight"), + mlp_fp8, + ) + + yield f"{_MODEL}.embed_tokens.weight", reader.get( + f"{_CKPT}.embed_tokens.weight" + ).to(torch.bfloat16) + yield f"{_MODEL}.norm.weight", reader.get(f"{_CKPT}.norm.weight").to(torch.bfloat16) + head = reader.get("lm_head.weight") + if head_fp8 and not config.tie_word_embeddings: + q, scale = _quant_fp8_per_row(head) + yield "lm_head.weight", q + yield "lm_head.weight_scale", scale + else: + yield "lm_head.weight", head.to(torch.bfloat16) + finally: + reader.close() + + +__all__ = ["iter_weights", "load_nvfp4_expert_sources"] diff --git a/python/freetoken/models/glm_moe_dsa/attention.py b/python/freetoken/models/glm_moe_dsa/attention.py index 14b1c100d1..e25d961fbd 100644 --- a/python/freetoken/models/glm_moe_dsa/attention.py +++ b/python/freetoken/models/glm_moe_dsa/attention.py @@ -93,14 +93,16 @@ def __init__(self, config: ModelConfig, layer_id: int): def compute( self, x: torch.Tensor, q_resid: torch.Tensor, positions: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """Per-token indexer projections: (q [T, H, D], k [T, D], weights [T, H] fp32).""" + ) -> "DSAIndexerInputs": + """Per-token indexer projections: q [T, H, D], k [T, D], weights [T, H] fp32.""" + from freetoken.attention.dsa import DSAIndexerInputs + t = x.shape[0] q = self.wq_b.forward(q_resid).view(t, self.n_heads * self.head_dim) k = self.k_norm.forward(self.wk.forward(x)).view(t, self.head_dim) q, k = self._rope.forward(positions, q, k) w = self.weights_proj.forward(x).float() * (self.n_heads**-0.5) - return q.view(t, self.n_heads, self.head_dim), k, w + return DSAIndexerInputs(q=q.view(t, self.n_heads, self.head_dim), k=k, w=w) class GlmMoeDsaAttention(BaseOP): @@ -213,7 +215,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: # DSA: full layers hand the backend this token's indexer projections (the # backend caches the keys, scores the history, and selects top-k); shared # layers pass None and reuse their group leader's selection. - indexer_qkw = ( + indexer_inputs = ( self.indexer.compute(x, q_a_resid, ctx.batch.positions) if self.indexer is not None and getattr(ctx.attn_backend, "dsa_enabled", False) else None @@ -223,7 +225,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: # concatenated latent copy on the hot path. o_latent = ctx.attn_backend.mla_forward( q_absorbed.contiguous(), q_rope.contiguous(), c_kv.contiguous(), - k_rope.contiguous(), self.layer_id, ctx.batch, indexer_qkw=indexer_qkw, + k_rope.contiguous(), self.layer_id, ctx.batch, indexer_inputs=indexer_inputs, ) # [T, H, kv_lora_rank] # Absorb kv_b's v-part onto the output: o_latent[H,T,lora] @ W_uv_t[H,lora,v]. diff --git a/python/freetoken/models/nvfp4_banks.py b/python/freetoken/models/nvfp4_banks.py index 0e3ab6a51c..6b933ff1da 100644 --- a/python/freetoken/models/nvfp4_banks.py +++ b/python/freetoken/models/nvfp4_banks.py @@ -22,6 +22,22 @@ class Nvfp4ExpertSourceSpec: proj_to_role: dict[str, str] layer_to_bank: LayerToBank desc: str + # Maps checkpoint tensor-kind names onto the canonical (modelopt) kinds, e.g. + # compressed-tensors' weight_packed -> weight, weight_global_scale -> weight_scale_2. + kind_map: dict[str, str] | None = None + # The checkpoint stores the QUANT-side global scale (local fp8 scales were + # multiplied by it before the cast); the banks keep its reciprocal. + global_reciprocal: bool = False + + +def _canon_kind(spec: "Nvfp4ExpertSourceSpec", kind: str) -> str: + return spec.kind_map.get(kind, kind) if spec.kind_map else kind + + +def _ingest_global(spec: "Nvfp4ExpertSourceSpec", tensor: torch.Tensor) -> torch.Tensor: + if spec.global_reciprocal: + tensor = 1.0 / tensor.float() + return tensor.to(torch.float16) def _num_moe_layers(config) -> int: @@ -112,7 +128,7 @@ def load_nvfp4_expert_source_banks( proj = match.group("proj") if proj not in spec.proj_to_role: raise ValueError(f"{spec.desc}: unknown NVFP4 expert projection {proj!r}") - kind = match.group("kind") + kind = _canon_kind(spec, match.group("kind")) if kind == "weight_scale_2": global_shards[shard].append((name, match, bank_layer)) elif kind in {"weight", "weight_scale"}: @@ -130,7 +146,7 @@ def load_nvfp4_expert_source_banks( int(match.group("expert")), match.group("proj"), ) - globals_map[key] = f.get_tensor(name).to(torch.float16) + globals_map[key] = _ingest_global(spec, f.get_tensor(name)) drop_page_cache(path) _hb = _alloc_nvfp4_host_banks(num_layers, E, H, I) # unpinned; pinned after fill @@ -154,7 +170,7 @@ def _load(sink) -> int: expert = int(match.group("expert")) proj = match.group("proj") role = spec.proj_to_role[proj] - kind = match.group("kind") + kind = _canon_kind(spec, match.group("kind")) tensor = f.get_tensor(name) if kind == "weight": if role == "gate": @@ -236,7 +252,7 @@ def load_nvfp4_expert_source_banks_parallel( bank_layer = _bank_layer(spec, int(match.group("layer")), config) if bank_layer is None: continue - kind = match.group("kind") + kind = _canon_kind(spec, match.group("kind")) if kind == "weight_scale_2": global_names_by_shard[shard].append(name) elif kind in {"weight", "weight_scale"}: @@ -253,7 +269,7 @@ def load_nvfp4_expert_source_banks_parallel( for name in global_names_by_shard[shard]: m = spec.key_pattern.match(name) globals_map[(int(m.group("layer")), int(m.group("expert")), m.group("proj"))] = ( - f.get_tensor(name).to(torch.float16) + _ingest_global(spec, f.get_tensor(name)) ) drop_page_cache(path) @@ -279,7 +295,7 @@ def _load(sink) -> int: expert = int(match.group("expert")) proj = match.group("proj") role = spec.proj_to_role[proj] - kind = match.group("kind") + kind = _canon_kind(spec, match.group("kind")) if kind == "weight": if role == "gate": gate_up_packed[bank_layer_id][expert, :I] = tensor diff --git a/python/freetoken/models/qwen3_5_moe/__init__.py b/python/freetoken/models/qwen3_5_moe/__init__.py index 98936e9f2e..a7fb568199 100644 --- a/python/freetoken/models/qwen3_5_moe/__init__.py +++ b/python/freetoken/models/qwen3_5_moe/__init__.py @@ -1,4 +1,11 @@ -from .config import parse_config +from .config import parse_config, parse_gguf_config +from .gguf import ( + convert_qwen3_5_to_gguf, + dummy_q4_k_q5_k_expert_sources, + is_gguf_model, + iter_gguf_weights, + load_q4_k_q5_k_expert_sources, +) from .model import Qwen3_5MoEForCausalLM from .weight import ( iter_weights, @@ -11,6 +18,12 @@ __all__ = [ "Qwen3_5MoEForCausalLM", "parse_config", + "parse_gguf_config", + "iter_gguf_weights", + "is_gguf_model", + "convert_qwen3_5_to_gguf", + "load_q4_k_q5_k_expert_sources", + "dummy_q4_k_q5_k_expert_sources", "iter_weights", "iter_weights_parallel", "load_nvfp4_expert_sources", diff --git a/python/freetoken/models/qwen3_5_moe/attention.py b/python/freetoken/models/qwen3_5_moe/attention.py index 2421264e91..7e0d950258 100644 --- a/python/freetoken/models/qwen3_5_moe/attention.py +++ b/python/freetoken/models/qwen3_5_moe/attention.py @@ -39,10 +39,19 @@ def __init__(self, config: ModelConfig, layer_id: int): # Fused q/k/v projection (one GEMM instead of three); q half is 2x for the # output gate. Split sizes: [num_q*head_dim*2, num_kv*head_dim, num_kv*head_dim]. self._qkv_split = [self.num_q * head_dim * 2, self.kv_attn_dim, self.kv_attn_dim] + self._gguf_mixed = config.attn_quant == "gguf_mixed" # Block-fp8 (Fp8BlockColMerged) when the checkpoint is quantized, else bf16 # LinearColParallelMerged. q/k/v out dims are all /128, so the merged fp8 weight + # weight_scale_inv concatenate cleanly along the output dim. - self.qkv_proj = make_col_merged(config, config.hidden_size, self._qkv_split, has_bias=False) + if self._gguf_mixed: + from freetoken.layers.gguf import GGUFLinear + types = dict(config.gguf_tensor_types) + self.qg_proj = GGUFLinear(config.hidden_size, self._qkv_split[0], types[f"blk.{layer_id}.attn_q.weight"]) + self.k_proj = GGUFLinear(config.hidden_size, self._qkv_split[1], types[f"blk.{layer_id}.attn_k.weight"]) + v_type = types[f"blk.{layer_id}.attn_v.weight"] + self.v_proj = GGUFLinear(config.hidden_size, self._qkv_split[2], v_type) + else: + self.qkv_proj = make_col_merged(config, config.hidden_size, self._qkv_split, has_bias=False) # Qwen3.5 uses Gemma-style (1+weight) RMSNorm; the weight loader bakes the +1 # into the stored weight (GemmaRMSNorm scales by the raw weight). self.q_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps) @@ -58,14 +67,23 @@ def __init__(self, config: ModelConfig, layer_id: int): else None ), ) - self.o_proj = make_replicated(config, self.qo_attn_dim, config.hidden_size, has_bias=False) + if self._gguf_mixed: + from freetoken.layers.gguf import GGUFLinear + self.o_proj = GGUFLinear(self.qo_attn_dim, config.hidden_size, types[f"blk.{layer_id}.attn_output.weight"]) + else: + self.o_proj = make_replicated(config, self.qo_attn_dim, config.hidden_size, has_bias=False) def _project(self, x: torch.Tensor): """Returns (q, k, v, gate): q [N, num_q, head_dim] post qk-norm+rope, k [N, num_kv*head_dim] post norm+rope, v [N, num_kv*head_dim], gate [N, num_q*head_dim].""" positions = get_global_ctx().batch.positions - qkv = self.qkv_proj.forward(x) - qg, k, v = torch.split(qkv, self._qkv_split, dim=-1) + if self._gguf_mixed: + qg = self.qg_proj.forward(x) + k = self.k_proj.forward(x) + v = self.v_proj.forward(x) + else: + qkv = self.qkv_proj.forward(x) + qg, k, v = torch.split(qkv, self._qkv_split, dim=-1) qg = qg.view(-1, self.num_q, self.head_dim * 2) q = qg[..., : self.head_dim].contiguous() # [N, num_q, head_dim] gate = qg[..., self.head_dim :].reshape(-1, self.qo_attn_dim) diff --git a/python/freetoken/models/qwen3_5_moe/config.py b/python/freetoken/models/qwen3_5_moe/config.py index dc16cfff8c..a20a3f44fc 100644 --- a/python/freetoken/models/qwen3_5_moe/config.py +++ b/python/freetoken/models/qwen3_5_moe/config.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any from freetoken.models.config import ( FullAttentionGroupConfig, @@ -10,6 +10,9 @@ detect_compressed_tensors_nvfp4, ) +if TYPE_CHECKING: + from freetoken.models.gguf.config import GgufConfigShim + def _quant_accessor(hf_config: Any): """A ``get(key, default=None)`` accessor over the HF ``quantization_config`` (dict or @@ -79,9 +82,10 @@ def _lm_head_quant(hf_config: Any) -> str: if not isinstance(layers, dict): return "none" for name, spec in layers.items(): - if name == "lm_head" or name.endswith(".lm_head"): - if "fp4" in str((spec or {}).get("quant_algo", "")).lower(): - return "nvfp4" + if (name == "lm_head" or name.endswith(".lm_head")) and "fp4" in str( + (spec or {}).get("quant_algo", "") + ).lower(): + return "nvfp4" return "none" @@ -98,9 +102,10 @@ def _dense_mlp_quant(hf_config: Any) -> str: if not isinstance(layers, dict): return "none" for name, spec in layers.items(): - if name.endswith((".mlp.gate_proj", ".mlp.up_proj", ".mlp.down_proj")): - if "fp4" in str((spec or {}).get("quant_algo", "")).lower(): - return "nvfp4" + if name.endswith((".mlp.gate_proj", ".mlp.up_proj", ".mlp.down_proj")) and "fp4" in str( + (spec or {}).get("quant_algo", "") + ).lower(): + return "nvfp4" return "none" @@ -152,7 +157,7 @@ def parse_config(hf_config: Any) -> ModelConfig: or getattr(text, "partial_rotary_factor", None) or 1.0 ) - rotary_dim = round(head_dim * partial) + rotary_dim = int(head_dim * partial) # For text-only with the default rope type, partial NeoX rope needs no scaling dict # (the mRoPE params reduce to standard partial rope for text). Avoid carrying the @@ -218,7 +223,7 @@ def parse_config(hf_config: Any) -> ModelConfig: key_head_dim=text.linear_key_head_dim, value_head_dim=text.linear_value_head_dim, conv_kernel_dim=text.linear_conv_kernel_dim, - output_gate=True, + output_gate="silu", ) # Order groups by their first layer id for deterministic iteration. groups = tuple( @@ -244,7 +249,7 @@ def parse_config(hf_config: Any) -> ModelConfig: num_experts_per_tok=getattr(text, "num_experts_per_tok", 0), moe_intermediate_size=getattr(text, "moe_intermediate_size", 0), shared_expert_intermediate_size=getattr(text, "shared_expert_intermediate_size", 0), - norm_topk_prob=bool(getattr(text, "norm_topk_prob", False)), + norm_topk_prob=True, moe_enabled=moe_enabled, use_qk_norm=True, model_type=getattr(hf_config, "model_type", "qwen3_5_moe"), @@ -260,4 +265,158 @@ def parse_config(hf_config: Any) -> ModelConfig: ) -__all__ = ["parse_config"] +def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig: + """Build a Qwen3.5 hybrid runtime configuration from GGUF metadata. + + llama.cpp records the same hybrid decoder geometry as the official Hugging Face + configuration, but expresses the Gated DeltaNet fields with its SSM vocabulary. + This parser keeps that translation in one audited location. It intentionally + describes the model only: native Q4_K_M tensor loading and kernel dispatch are + separate implementation milestones, so callers cannot mistake metadata parsing + for a runnable GGUF path. + """ + metadata = shim.metadata + # Qwen3.8-27B uses the dense ``qwen35`` GGUF architecture, while the + # qualified Qwen3.6-35B-A3B control uses ``qwen35moe``. Both share the + # hybrid attention and Gated DeltaNet geometry, but only the latter has + # routed-expert fields. + prefix = "qwen35moe" if shim.model_type == "qwen35moe" else "qwen35" + is_moe = prefix == "qwen35moe" + + def value(key: str): + """Read one required architecture-scoped GGUF value with a clear error.""" + full_key = f"{prefix}.{key}" + if full_key not in metadata: + raise KeyError(f"missing GGUF metadata key {full_key}") + return metadata[full_key] + + hidden_size = int(value("embedding_length")) + head_dim = int(value("attention.key_length")) + num_qo_heads = int(value("attention.head_count")) + num_kv_heads = int(value("attention.head_count_kv")) + linear_key_head_dim = int(value("ssm.state_size")) + linear_value_head_dim = int(value("ssm.state_size")) + linear_num_key_heads = int(value("ssm.group_count")) + linear_inner_size = int(value("ssm.inner_size")) + if linear_inner_size % linear_value_head_dim: + raise ValueError( + "qwen35moe.ssm.inner_size must divide exactly into value-head groups: " + f"{linear_inner_size} / {linear_value_head_dim}" + ) + linear_num_value_heads = linear_inner_size // linear_value_head_dim + + # What: read every serialized transformer block; why: GGUF block_count includes optional trailing predictor blocks. + total_layers = int(value("block_count")) + # What: read the optional next-token predictor count; why: ordinary decoder execution must exclude speculative MTP blocks. + nextn_predict_layers = int(metadata.get(f"{prefix}.nextn_predict_layers", 0)) + # What: reject impossible predictor geometry; why: malformed counts must fail before weight allocation or service mutation. + if nextn_predict_layers < 0 or nextn_predict_layers >= total_layers: + # What: raise a bounded compatibility error; why: callers need an actionable failure rather than an invalid model. + raise ValueError( + # What: report the non-sensitive total count; why: maintainers need the artifact geometry that failed validation. + f"invalid {prefix} predictor geometry: block_count={total_layers}, " + # What: report the predictor count; why: the excluded quantity explains the exact rejected relationship. + f"nextn_predict_layers={nextn_predict_layers}" + # What: close the grouped exception construction; why: Python requires the call boundary before execution continues. + ) + # What: derive executable decoder depth; why: FreeToken text generation does not run the trailing MTP head. + num_layers = total_layers - nextn_predict_layers + # What: read the attention cadence; why: layer groups below require the validated main-decoder depth and interval. + full_interval = int(value("full_attention_interval")) + if full_interval <= 0: + raise ValueError(f"invalid qwen35moe.full_attention_interval {full_interval}") + layer_types = tuple( + "full_attention" if (layer_index + 1) % full_interval == 0 else "linear_attention" + for layer_index in range(num_layers) + ) + full_ids = tuple(index for index, kind in enumerate(layer_types) if kind == "full_attention") + linear_ids = tuple(index for index, kind in enumerate(layer_types) if kind == "linear_attention") + + rotary = RotaryConfig( + head_dim=head_dim, + rotary_dim=int(value("rope.dimension_count")), + max_position=int(value("context_length")), + base=float(value("rope.freq_base")), + scaling=None, + ) + full_group = FullAttentionGroupConfig( + name="full", + layer_ids=full_ids, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + rotary_config=rotary, + ) + linear_group = LinearGatedDeltaGroupConfig( + name="linear", + layer_ids=linear_ids, + num_key_heads=linear_num_key_heads, + num_value_heads=linear_num_value_heads, + key_head_dim=linear_key_head_dim, + value_head_dim=linear_value_head_dim, + conv_kernel_dim=int(value("ssm.conv_kernel")), + output_gate="silu", + ) + + # Q4_K_M is a recipe, not one homogeneous tensor type. The exact Qwen + # control has Q6_K down experts in a small set of late layers. Read the + # tensor table when available, while allowing metadata-only converter tests + # to exercise the architecture parser without a 22 GiB model file. + q6_down_layers: tuple[int, ...] = () + tensor_types: tuple[tuple[str, int], ...] = () + try: + from freetoken.models.gguf.dequant import GGML_Q6_K + from freetoken.models.gguf.reader import iter_gguf_tensors + + tensor_types = tuple((t.name, int(t.ggml_type)) for t in iter_gguf_tensors(shim.model_path)) + + q6_down_layers = tuple( + int(t.name.split(".")[1]) + for t in iter_gguf_tensors(shim.model_path) + if t.name.startswith("blk.") + and t.name.endswith("ffn_down_exps.weight") + and t.ggml_type == GGML_Q6_K + ) + except FileNotFoundError: + pass + + # Dense qwen35 stores one feed-forward width. The MoE GGUF stores only + # routed and shared expert widths, so its generic dense width remains zero. + intermediate_size = int(value("feed_forward_length")) if not is_moe else 0 + return ModelConfig( + num_layers=num_layers, + num_qo_heads=num_qo_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + hidden_size=hidden_size, + vocab_size=int(shim.vocab_size), + intermediate_size=intermediate_size, + hidden_act="silu", + rms_norm_eps=float(value("attention.layer_norm_rms_epsilon")), + tie_word_embeddings=bool(shim.tie_word_embeddings), + rotary_config=rotary, + num_experts=int(value("expert_count")) if is_moe else 0, + num_experts_per_tok=int(value("expert_used_count")) if is_moe else 0, + moe_intermediate_size=int(value("expert_feed_forward_length")) if is_moe else 0, + shared_expert_intermediate_size=( + int(value("expert_shared_feed_forward_length")) if is_moe else 0 + ), + norm_topk_prob=True, + moe_enabled=is_moe, + use_qk_norm=True, + model_type="qwen3_5_moe", + architectures=["Qwen3_5MoeForConditionalGeneration"], + vision_config=None, + attention_groups=(linear_group, full_group), + # The Qwen3.6-35B-A3B Q4_K_M GGUF stores routed gate/up in Q4_K and + # routed down in Q5_K. The explicit tag selects the mixed bank provider. + expert_quant="q4_k_q5_k" if is_moe else "none", + moe_weight_format="q4_k_q5_k" if is_moe else "qwen35_dense", + gguf_q6_down_layer_ids=q6_down_layers, + gguf_tensor_types=tensor_types, + # Dense Qwen3.6-27B-Q4_K_M has Q6_K qkv and a Q4_K GDN gate. Those + # packed layouts have different row widths, while b|a remains F32. + attn_quant="gguf_mixed" if not is_moe else "gguf_q8", + ) + + +__all__ = ["parse_config", "parse_gguf_config"] diff --git a/python/freetoken/models/qwen3_5_moe/gdn.py b/python/freetoken/models/qwen3_5_moe/gdn.py index 2e7320051c..8c4e30b633 100644 --- a/python/freetoken/models/qwen3_5_moe/gdn.py +++ b/python/freetoken/models/qwen3_5_moe/gdn.py @@ -54,6 +54,7 @@ def __init__( self, hidden_size, num_k_heads, num_v_heads, head_k_dim, head_v_dim, conv_kernel_size, rms_norm_eps, layer_id, expert_quant: str = "none", attn_quant: str = "none", + config=None, ): self.layer_id = layer_id # The fla chunk/decode kernels read+write the recurrent state and the per-chunk h as @@ -77,13 +78,35 @@ def __init__( self._block_fp8 = expert_quant == "fp8_block" self._pertensor_fp8 = attn_quant == "fp8_pertensor" self._fp8 = self._block_fp8 or self._pertensor_fp8 + # Older Qwen GGUF exports store qkv|z as Q8_0. Qwen3.6-27B-Q4_K_M uses + # Q6_K for qkv and Q4_K for z, so those projections cannot share a packed + # qweight tensor. + self._gguf_q8 = attn_quant == "gguf_q8" + self._gguf_mixed = attn_quant == "gguf_mixed" self._in_proj_split = [self.conv_dim, self.value_dim, num_v_heads, num_v_heads] - if self._fp8: - ColMerged = Fp8BlockColMerged if self._block_fp8 else Fp8PerTensorColMerged - self.in_proj_qkvz = ColMerged( - hidden_size, [self.conv_dim, self.value_dim], has_bias=False - ) + if self._fp8 or self._gguf_q8 or self._gguf_mixed: + if self._gguf_mixed: + from freetoken.layers.gguf import GGUFLinear + + types = dict(config.gguf_tensor_types) + qkv_type = types[f"blk.{layer_id}.attn_qkv.weight"] + self.in_proj_qkv = GGUFLinear(hidden_size, self.conv_dim, qkv_type, has_bias=False) + self.in_proj_z = GGUFLinear( + hidden_size, self.value_dim, types[f"blk.{layer_id}.attn_gate.weight"], has_bias=False + ) + elif self._gguf_q8: + from freetoken.layers.gguf import GGUFLinear + from freetoken.models.gguf.dequant import GGML_Q8_0 + + self.in_proj_qkvz = GGUFLinear( + hidden_size, self.conv_dim + self.value_dim, GGML_Q8_0, has_bias=False + ) + else: + ColMerged = Fp8BlockColMerged if self._block_fp8 else Fp8PerTensorColMerged + self.in_proj_qkvz = ColMerged( + hidden_size, [self.conv_dim, self.value_dim], has_bias=False + ) self.in_proj_ba = LinearColParallelMerged( hidden_size, [num_v_heads, num_v_heads], has_bias=False ) @@ -110,6 +133,20 @@ def _gate_params(self, a: torch.Tensor, b: torch.Tensor): g = -self.A_log.exp() * F.softplus(a.float() + self.dt_bias) return g, beta + def _gguf_group_value_heads_for_out_proj(self, x: torch.Tensor) -> torch.Tensor: + """Return HF-ordered GDN values to the original GGUF grouped head order. + + Q4_K blocks in ``ssm_out`` span two 128-wide value heads. Reordering the + packed weights would invalidate their block metadata, so keep the weights + byte-exact and invert the GGUF-to-HF value-head permutation on activations. + """ + if self.num_v_heads % self.num_k_heads: + raise ValueError( + f"GDN value heads {self.num_v_heads} are not divisible by key heads {self.num_k_heads}" + ) + ratio = self.num_v_heads // self.num_k_heads + return x.reshape(-1, self.num_k_heads, ratio, self.head_v_dim).transpose(1, 2).reshape_as(x) + def _conv_weight(self) -> torch.Tensor: return self.conv1d.weight.squeeze(1) # [conv_dim, kernel] for the fused kernel @@ -161,7 +198,12 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: fla = build_fla_metadata(batch, hidden_states.device) batch.fla_metadata = fla - if self._fp8: + if self._gguf_mixed: + conv_in = self.in_proj_qkv.forward(hidden_states) + z = self.in_proj_z.forward(hidden_states) + ba = self.in_proj_ba.forward(hidden_states) + b, a = torch.split(ba, [self.num_v_heads, self.num_v_heads], dim=-1) + elif self._fp8 or self._gguf_q8: qkvz = self.in_proj_qkvz.forward(hidden_states) conv_in, z = torch.split(qkvz, [self.conv_dim, self.value_dim], dim=-1) ba = self.in_proj_ba.forward(hidden_states) @@ -218,6 +260,8 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: core_out = core_out.reshape(-1, self.head_v_dim) z = z.reshape(-1, self.head_v_dim) out = self.norm.forward(core_out, z).reshape(total, -1) + if self._gguf_mixed: + out = self._gguf_group_value_heads_for_out_proj(out) return self.out_proj.forward(out) diff --git a/python/freetoken/models/qwen3_5_moe/gguf.py b/python/freetoken/models/qwen3_5_moe/gguf.py new file mode 100644 index 0000000000..6082d48235 --- /dev/null +++ b/python/freetoken/models/qwen3_5_moe/gguf.py @@ -0,0 +1,651 @@ +"""Native Qwen3.6 GGUF loading for the Q4_K_M control checkpoint. + +The checkpoint remains block-quantized end to end. Q8_0 and Q6_K dense +projections are retained as packed tensors and execute through FreeToken's +native ggml HIP kernels. Routed expert gate/up rows stay Q4_K and down rows +stay Q5_K in the AMD offload cache. Only scalar parameters such as norms, +router weights, and the Gated DeltaNet recurrence parameters are materialized +as bf16 or fp32, because they are stored as F32 in the GGUF. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass + +import torch + +from freetoken.layers import BaseOP +from freetoken.models.config import ModelConfig +from freetoken.models.gguf.dequant import ( + GGML_Q4_K, + GGML_Q5_K, + GGML_Q6_K, + GGML_Q8_0, + dequantize, + row_bytes, +) + +# F32 GGUF tensors whose runtime parameter has a direct one-to-one mapping. +# The Gated DeltaNet alpha/beta naming describes the recurrence semantics: +# alpha maps to the softplus ``a`` input and beta maps to the sigmoid ``b`` input. +_SCALAR_MAP = { + "attn_norm.weight": "input_layernorm.weight", + "attn_q_norm.weight": "self_attn.q_norm.weight", + "attn_k_norm.weight": "self_attn.k_norm.weight", + "post_attention_norm.weight": "post_attention_layernorm.weight", + "ssm_conv1d.weight": "linear_attn.conv1d.weight", + "ssm_dt.bias": "linear_attn.dt_bias", + "ssm_norm.weight": "linear_attn.norm.weight", + "ffn_gate_inp.weight": "mlp.gate.weight", + "ffn_gate_inp_shexp.weight": "mlp.shared_expert_gate.weight", +} +_EXPERT_SUFFIXES = ("ffn_gate_exps.weight", "ffn_up_exps.weight", "ffn_down_exps.weight") +_GDN_BA_SUFFIXES = {"ssm_alpha.weight": "a", "ssm_beta.weight": "b"} + + +def _ssm_a_to_a_log(ssm_a: torch.Tensor) -> torch.Tensor: + """Recover HF ``A_log`` from llama.cpp's precomputed negative decay. + + The GGUF Qwen3.5 exporter writes ``ssm_a = -exp(A_log)`` because llama.cpp + multiplies that value directly by the softplus alpha gate. FreeToken's Gated + DeltaNet instead owns the equivalent ``-A_log.exp()`` expression. Loading the + GGUF value as ``A_log`` would exponentiate it a second time and destabilize every + linear-attention layer, so invert the exporter transformation exactly here. + """ + value = ssm_a.to(torch.float32) + if not torch.isfinite(value).all() or not torch.all(value < 0): + raise ValueError("Qwen GGUF ssm_a must contain finite negative -exp(A_log) values") + return torch.log(-value) + + +def _restore_gdn_value_head_order(value: torch.Tensor, num_key_heads: int) -> torch.Tensor: + """Convert llama.cpp's grouped GDN value-head order to Qwen's interleaved order. + + Qwen3.5 uses more value than key heads. GGUF places all value heads belonging + to the first position of each key-head group before the second position, while + the Hugging Face checkpoint and FreeToken's Gated DeltaNet use consecutive + per-key-head values. This applies both to one scalar per value head (``ssm_a`` + and ``ssm_dt``) and to matrices whose output axis is the value-head axis + (``ssm_alpha`` and ``ssm_beta``). + """ + if value.ndim < 1: + raise ValueError("Qwen GDN value-head tensor must have at least one dimension") + num_value_heads = value.shape[0] + if num_key_heads <= 0 or num_value_heads % num_key_heads: + raise ValueError( + "Qwen GDN value-head tensor is incompatible with the GGUF key-head count: " + f"{tuple(value.shape)} vs {num_key_heads}" + ) + head_ratio = num_value_heads // num_key_heads + if head_ratio == 1: + return value + # [ratio, key_head, ...] in GGUF becomes [key_head, ratio, ...], then a + # contiguous leading output axis matching the HF/FreeToken projection order. + return value.reshape(head_ratio, num_key_heads, *value.shape[1:]).transpose(0, 1).reshape_as(value) + + +def _restore_gdn_value_head_rows( + value: torch.Tensor, + num_key_heads: int, + head_dim: int, +) -> torch.Tensor: + """Restore grouped GGUF rows whose leading axis contains complete value heads. + + A GDN projection can contain a prefix of key-head rows followed by its value + rows, such as the Q|K|V and depthwise-convolution projections. Only the + value suffix needs the llama.cpp-to-HF permutation. ``head_dim`` is the + number of consecutive rows occupied by one value head, so this routine also + handles projections such as ``z`` where each head spans 128 output rows. + """ + if value.ndim < 1 or head_dim <= 0 or value.shape[0] % head_dim: + raise ValueError( + "Qwen GDN value-head rows require a positive whole-head leading axis: " + f"{tuple(value.shape)} with head_dim={head_dim}" + ) + num_value_heads = value.shape[0] // head_dim + ordered = _restore_gdn_value_head_order( + value.reshape(num_value_heads, head_dim, *value.shape[1:]), num_key_heads + ) + return ordered.reshape_as(value) + + +def _restore_gdn_value_head_input_blocks( + packed: torch.Tensor, + num_key_heads: int, + head_dim: int, + quant_type: int = GGML_Q8_0, +) -> torch.Tensor: + """Restore GDN value-head order along a Q8_0 packed projection input axis. + + ``ssm_out`` consumes all value heads as its input. Q8_0 stores independent + 32-element blocks along each output row, and Qwen's 128-element value heads + therefore occupy four complete byte blocks. Reordering those blocks is exact: + it neither dequantizes weights nor changes their Q8 scales or integers. + """ + if packed.ndim != 2: + raise ValueError(f"Qwen GDN packed output projection must be rank 2, got {tuple(packed.shape)}") + bytes_per_head = row_bytes(head_dim, quant_type) + if head_dim <= 0 or packed.shape[1] % bytes_per_head: + raise ValueError( + "Qwen GDN packed output projection does not contain complete value-head blocks: " + f"{tuple(packed.shape)} with head_dim={head_dim}" + ) + num_value_heads = packed.shape[1] // bytes_per_head + grouped = packed.reshape(packed.shape[0], num_value_heads, bytes_per_head) + # The generic helper operates on the leading head axis. Transpose the packed + # view so the same explicit permutation is applied to every output row. + return _restore_gdn_value_head_order(grouped.transpose(0, 1), num_key_heads).transpose(0, 1).reshape_as(packed) + + +def _to_bf16(t) -> torch.Tensor: + """Dequantize one GGUF scalar tensor to its logical torch shape.""" + return dequantize(t.packed().reshape(-1), t.ggml_type, torch.bfloat16).reshape(t.shape) + + +def _require_weight_tp1() -> None: + """Reject TP before loading unsharded GGUF packed rows.""" + from freetoken.distributed import get_tp_info + + if get_tp_info().size > 1: + raise NotImplementedError("Qwen3.5 GGUF weight loading currently supports TP=1 only") + + +def iter_gguf_weights( + model_path: str, + device, + *, + include_moe_experts: bool, + include_non_moe: bool, +) -> Iterator[tuple[str, torch.Tensor]]: + """Yield every non-routed-expert Qwen GGUF parameter in runtime key order. + + Full-attention Q/K/V and Gated DeltaNet qkv/z/b/a each arrive as individual + GGUF tensors. FreeToken executes them as fused projections, so their packed + rows are concatenated only on the output axis. This is byte preserving because + every fused member has the same input width and quantization type (Q8_0). + """ + from freetoken.models.gguf.reader import iter_gguf_tensors, load_gguf_metadata + + _require_weight_tp1() + + metadata = load_gguf_metadata(model_path) + arch = metadata.get("general.architecture") + prefix = "qwen35moe" if arch == "qwen35moe" else "qwen35" + dense_model = prefix == "qwen35" + if not include_non_moe: + if dense_model: + return + raise AssertionError("Qwen GGUF routed experts are supplied by the offload cache") + # The generic engine invokes this iterator for both weight phases. Dense + # qwen35 checkpoints have no routed experts, so their expert phase is an + # intentional no-op. Keep rejecting that phase for qwen35moe, whose + # routed experts are supplied by the offload cache instead. + if include_moe_experts and not dense_model: + raise AssertionError("Qwen GGUF routed experts are supplied by the offload cache") + gdn_num_key_heads = int(metadata[f"{prefix}.ssm.group_count"]) + gdn_num_value_heads = int(metadata[f"{prefix}.ssm.time_step_rank"]) + gdn_inner_size = int(metadata[f"{prefix}.ssm.inner_size"]) + if gdn_num_value_heads <= 0 or gdn_inner_size % gdn_num_value_heads: + raise ValueError( + "Qwen GGUF GDN metadata has an invalid value-head geometry: " + f"inner_size={gdn_inner_size}, time_step_rank={gdn_num_value_heads}" + ) + gdn_value_head_dim = gdn_inner_size // gdn_num_value_heads + # What: read the serialized block count; why: GGUF includes optional trailing MTP blocks in this total. + total_layers = int(metadata[f"{prefix}.block_count"]) + # What: read the optional predictor count; why: older non-MTP artifacts omit it and must retain prior behavior. + nextn_predict_layers = int(metadata.get(f"{prefix}.nextn_predict_layers", 0)) + # What: derive the decoder boundary; why: non-expert loading must not emit unsupported MTP tensors. + main_layers = total_layers - nextn_predict_layers + + qkv_buf: dict[int, dict[str, torch.Tensor]] = {} + gdn_buf: dict[int, dict[str, torch.Tensor]] = {} + shared_buf: dict[int, dict[str, torch.Tensor]] = {} + dense_buf: dict[int, dict[str, torch.Tensor]] = {} + + for t in iter_gguf_tensors(model_path): + name = t.name + if name == "token_embd.weight": + if t.ggml_type not in (GGML_Q8_0, GGML_Q4_K): + raise ValueError(f"{name} expected Q8_0 or Q4_K, got {t.ggml_type}") + yield "model.embed_tokens.qweight", t.packed() + continue + if name == "output.weight": + if t.ggml_type != GGML_Q6_K: + raise ValueError(f"{name} expected Q6_K, got {t.ggml_type}") + yield "lm_head.qweight", t.packed() + continue + if name == "output_norm.weight": + # Unlike Gemma GGUF checkpoints, Qwen stores the final RMSNorm scale + # directly. Adding one here would apply the Gemma delta convention + # to an already complete Qwen weight and corrupt every output logit. + yield "model.norm.weight", _to_bf16(t) + continue + if not name.startswith("blk."): + continue + + parts = name.split(".") + layer = int(parts[1]) + # What: skip blocks outside executable decoder depth; why: the trailing MTP block is not part of text inference. + if layer >= main_layers: + # What: continue without yielding predictor tensors; why: runtime modules exist only for ordinary decoder layers. + continue + suffix = ".".join(parts[2:]) + base = f"model.layers.{layer}" + if suffix in _EXPERT_SUFFIXES: + continue + if dense_model and suffix in ("ffn_gate.weight", "ffn_up.weight"): + dense_buf.setdefault(layer, {})[suffix.removeprefix("ffn_").removesuffix(".weight")] = t.packed() + elif dense_model and suffix == "ffn_down.weight": + yield f"{base}.mlp.down_proj.qweight", t.packed() + elif suffix in _GDN_BA_SUFFIXES: + # The split GGUF path keeps qkv|z packed Q8_0, while recurrence b|a + # remains a conventional dense fused projection. The runtime order is + # explicitly b then a, matching Qwen3_5GatedDeltaNet._in_proj_split. + gdn_buf.setdefault(layer, {})[_GDN_BA_SUFFIXES[suffix]] = _restore_gdn_value_head_order( + _to_bf16(t), gdn_num_key_heads + ) + slots = gdn_buf[layer] + if all(key in slots for key in ("b", "a")): + yield f"{base}.linear_attn.in_proj_ba.weight", torch.cat( + [slots.pop("b"), slots.pop("a")], dim=0 + ) + if not slots: + del gdn_buf[layer] + continue + if suffix == "ssm_a": + # llama.cpp serializes the already-exponentiated negative coefficient; + # FreeToken stores A_log and evaluates -exp(A_log) at runtime. + yield f"{base}.linear_attn.A_log", _ssm_a_to_a_log( + _restore_gdn_value_head_order(_to_bf16(t), gdn_num_key_heads) + ) + continue + if suffix in _SCALAR_MAP: + tensor = _to_bf16(t) + rel = _SCALAR_MAP[suffix] + if suffix == "ssm_dt.bias": + tensor = _restore_gdn_value_head_order(tensor, gdn_num_key_heads) + if suffix == "ssm_conv1d.weight": + # GGUF stores depthwise filters as [channels, kernel]; FreeToken's + # causal-convolution holder uses the PyTorch depthwise layout + # [channels, 1, kernel]. + # Its Q|K prefix retains key-head order, while its V suffix uses + # llama.cpp's grouped value-head order and must be made consistent + # with the restored scalar recurrence terms. + gdn_key_dim = gdn_num_key_heads * gdn_value_head_dim + gdn_value_dim = gdn_num_value_heads * gdn_value_head_dim + qk_prefix = tensor[: 2 * gdn_key_dim] + value_rows = _restore_gdn_value_head_rows( + tensor[2 * gdn_key_dim : 2 * gdn_key_dim + gdn_value_dim], + gdn_num_key_heads, + gdn_value_head_dim, + ) + tensor = torch.cat((qk_prefix, value_rows), dim=0).unsqueeze(1) + elif suffix == "ffn_gate_inp_shexp.weight": + # The single shared-expert gate is stored as a vector in GGUF but + # executes as a one-row replicated linear projection. + tensor = tensor.unsqueeze(0) + # Qwen GGUF stores all of these RMSNorm vectors as direct scales. + # ``GemmaRMSNorm`` in this runtime applies the raw stored tensor; the + # safetensors loader performs a separate +1 bake only because HF Qwen + # checkpoints carry delta-from-unity weights. GGUF must not adjust it. + if rel.endswith(("linear_attn.A_log", "linear_attn.dt_bias")): + tensor = tensor.to(torch.float32) + yield f"{base}.{rel}", tensor + continue + + if suffix == "attn_q.weight": + if dense_model: + yield f"{base}.self_attn.qg_proj.qweight", t.packed() + else: + qkv_buf.setdefault(layer, {})["qg"] = t.packed() + elif suffix == "attn_k.weight": + if dense_model: + yield f"{base}.self_attn.k_proj.qweight", t.packed() + else: + qkv_buf.setdefault(layer, {})["k"] = t.packed() + elif suffix == "attn_v.weight": + if dense_model: + yield f"{base}.self_attn.v_proj.qweight", t.packed() + else: + qkv_buf.setdefault(layer, {})["v"] = t.packed() + elif suffix == "attn_output.weight": + yield f"{base}.self_attn.o_proj.qweight", t.packed() + elif suffix == "attn_qkv.weight": + # The Q|K prefix is keyed by the 16 GDN key heads. The V suffix is + # keyed by the 32 value heads and is grouped by llama.cpp in GGUF. + if t.ggml_type not in (GGML_Q4_K, GGML_Q6_K, GGML_Q8_0): + raise ValueError(f"{name} has unsupported packed type {t.ggml_type}") + packed = t.packed() + gdn_key_dim = gdn_num_key_heads * gdn_value_head_dim + qk_rows = packed[: 2 * gdn_key_dim] + value_rows = _restore_gdn_value_head_rows( + packed[2 * gdn_key_dim :], gdn_num_key_heads, gdn_value_head_dim + ) + gdn_buf.setdefault(layer, {})["qkv"] = torch.cat((qk_rows, value_rows), dim=0) + elif suffix == "attn_gate.weight": + if t.ggml_type not in (GGML_Q4_K, GGML_Q6_K, GGML_Q8_0): + raise ValueError(f"{name} has unsupported packed type {t.ggml_type}") + gdn_buf.setdefault(layer, {})["z"] = _restore_gdn_value_head_rows( + t.packed(), gdn_num_key_heads, gdn_value_head_dim + ) + elif suffix == "ssm_out.weight": + # Q4_K blocks span two 128-wide value heads. Preserve the packed rows + # byte-exact; GatedDeltaNet inversely groups its activation before this + # projection instead of reordering block-quantized weight bytes. + if dense_model: + yield f"{base}.linear_attn.out_proj.qweight", t.packed() + else: + yield f"{base}.linear_attn.out_proj.qweight", _restore_gdn_value_head_input_blocks( + t.packed(), gdn_num_key_heads, gdn_value_head_dim + ) + elif suffix == "ffn_gate_shexp.weight": + shared_buf.setdefault(layer, {})["gate"] = t.packed() + elif suffix == "ffn_up_shexp.weight": + shared_buf.setdefault(layer, {})["up"] = t.packed() + elif suffix == "ffn_down_shexp.weight": + yield f"{base}.mlp.shared_expert.down_proj.qweight", t.packed() + elif not (dense_model and suffix in ("ffn_gate.weight", "ffn_up.weight", "ffn_down.weight")): + raise ValueError(f"unmapped Qwen3.5 GGUF tensor: {name}") + + slots = qkv_buf.get(layer) + if slots is not None and all(key in slots for key in ("qg", "k", "v")): + yield f"{base}.self_attn.qkv_proj.qweight", torch.cat( + [slots["qg"], slots["k"], slots["v"]], dim=0 + ) + del qkv_buf[layer] + slots = gdn_buf.get(layer) + if slots is not None and all(key in slots for key in ("qkv", "z")): + # Qwen3.6-27B-Q4_K_M stores qkv as Q6_K and z as Q4_K. Execute + # them separately, then concatenate activations in GatedDeltaNet. + if dense_model: + yield f"{base}.linear_attn.in_proj_qkv.qweight", slots["qkv"] + yield f"{base}.linear_attn.in_proj_z.qweight", slots["z"] + else: + yield f"{base}.linear_attn.in_proj_qkvz.qweight", torch.cat( + [slots["qkv"], slots["z"]], dim=0 + ) + del slots["qkv"], slots["z"] + if not slots: + del gdn_buf[layer] + slots = shared_buf.get(layer) + if slots is not None and all(key in slots for key in ("gate", "up")): + yield f"{base}.mlp.shared_expert.gate_up_proj.qweight", torch.cat( + [slots["gate"], slots["up"]], dim=0 + ) + del shared_buf[layer] + slots = dense_buf.get(layer) + if slots is not None and all(key in slots for key in ("gate", "up")): + yield f"{base}.mlp.gate_up_proj.qweight", torch.cat( + [slots["gate"], slots["up"]], dim=0 + ) + del dense_buf[layer] + + assert not qkv_buf, f"incomplete Qwen attention QKV groups: {sorted(qkv_buf)}" + assert not gdn_buf, f"incomplete Qwen GDN qkv/z groups: {sorted(gdn_buf)}" + assert not shared_buf, f"incomplete Qwen shared gate/up groups: {sorted(shared_buf)}" + assert not dense_buf, f"incomplete Qwen dense gate/up groups: {sorted(dense_buf)}" + + +class GGUFLMHead(BaseOP): + """Untied Q6_K language head that preserves last-token prefill semantics.""" + + def __init__(self, num_embeddings: int, embedding_dim: int): + self.qweight = torch.empty( + num_embeddings, row_bytes(embedding_dim, GGML_Q6_K), dtype=torch.uint8 + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + from freetoken.core import get_global_ctx + from freetoken.layers.gguf import fused_mul_mat_gguf + + batch = get_global_ctx().batch + if batch.is_prefill: + x = x[batch.attn_metadata.get_last_indices(batch.size)].contiguous() + return fused_mul_mat_gguf(x, self.qweight, GGML_Q6_K) + + +def is_gguf_model(config: ModelConfig) -> bool: + """Return whether this model uses the Qwen packed-GGUF runtime path.""" + return getattr(config, "moe_weight_format", None) in {"q4_k_q5_k", "qwen35_dense"} + + +def convert_qwen3_5_to_gguf(model, config: ModelConfig) -> None: + """Replace Qwen dense projections with packed GGUF HIP operators in place.""" + from freetoken.layers.gguf import GGUFEmbedding, GGUFLinear + + dense_model = not config.moe_enabled + types = dict(config.gguf_tensor_types) + embed_quant = types.get("token_embd.weight", GGML_Q4_K) if dense_model else GGML_Q8_0 + full_output_quant = GGML_Q6_K if dense_model else GGML_Q8_0 + + def swap_linear(owner, attr: str, quant_type: int, in_features: int, out_features: int): + old = getattr(owner, attr) + setattr(owner, attr, GGUFLinear(in_features, out_features, quant_type, old.bias is not None)) + + inner = model.model + inner.embed_tokens = GGUFEmbedding( + config.vocab_size, config.hidden_size, embed_quant, embed_scale=None + ) + for layer in inner.layers.op_list: + if layer._is_linear: + g = config.linear_attention_group() + assert g is not None + # The GDN constructor provides separate native packed qkv and z + # projections, plus a dense b|a projection. + assert hasattr(layer.linear_attn, "in_proj_qkv" if dense_model else "in_proj_qkvz") + if dense_model: + assert hasattr(layer.linear_attn, "in_proj_z") + assert hasattr(layer.linear_attn, "in_proj_ba") + swap_linear( + layer.linear_attn, "out_proj", types[f"blk.{layer._layer_id}.ssm_out.weight"] if dense_model else GGML_Q8_0, + layer.linear_attn.value_dim, config.hidden_size, + ) + elif config.attn_quant != "gguf_mixed": + swap_linear( + layer.self_attn, "qkv_proj", GGML_Q8_0, + config.hidden_size, sum(layer.self_attn._qkv_split), + ) + swap_linear( + layer.self_attn, "o_proj", full_output_quant, + layer.self_attn.qo_attn_dim, config.hidden_size, + ) + if config.moe_enabled: + owner = layer.mlp.shared_expert + intermediate = config.shared_expert_intermediate_size + mlp_quant = GGML_Q8_0 + else: + owner = layer.mlp + intermediate = config.intermediate_size + mlp_quant = GGML_Q4_K + swap_linear(owner, "gate_up_proj", mlp_quant, config.hidden_size, 2 * intermediate) + down_type = types[f"blk.{layer._layer_id}.ffn_down.weight"] if dense_model else mlp_quant + swap_linear(owner, "down_proj", down_type, intermediate, config.hidden_size) + model.lm_head = GGUFLMHead(config.vocab_size, config.hidden_size) + + +def _require_tp1() -> None: + """Reject unsupported tensor parallelism before allocating unsharded GGUF banks.""" + from freetoken.distributed import get_tp_info + + if get_tp_info().size > 1: + raise NotImplementedError("Qwen3.5 GGUF expert banks currently support TP=1 only") + + +def _expert_specs(config) -> dict[str, tuple[tuple[int, ...], torch.dtype]]: + """Return host-bank shapes expressed in exact packed GGML row bytes.""" + experts = int(config.num_experts) + hidden = int(config.hidden_size) + intermediate = int(config.moe_intermediate_size) + return { + "gate_up": ((experts, 2 * intermediate, row_bytes(hidden, GGML_Q4_K)), torch.uint8), + "down": ((experts, hidden, row_bytes(intermediate, GGML_Q5_K)), torch.uint8), + } + + +def _q6_down_specs(config) -> dict[str, tuple[tuple[int, ...], torch.dtype]]: + """One Q6_K down bank for each exceptional Qwen GGUF layer.""" + experts = int(config.num_experts) + hidden = int(config.hidden_size) + intermediate = int(config.moe_intermediate_size) + return { + "down": ((experts, hidden, row_bytes(intermediate, GGML_Q6_K)), torch.uint8), + } + + +@dataclass(frozen=True) +class QwenGGUFExpertSources: + """Primary Q4_K/Q5_K banks plus exact Q6_K down-only exceptional banks. + + ``primary`` stays shape-uniform for the existing cache. Its Q5_K down rows + for ``q6_layer_ids`` are deliberately unused placeholders. ``q6_down`` has + only the actual Q6_K layers in the same order as ``q6_layer_ids`` and feeds a + small auxiliary cache, avoiding any conversion between the two GGML layouts. + """ + + primary: dict[str, list[torch.Tensor]] + q6_down: list[torch.Tensor] + q6_layer_ids: tuple[int, ...] + + +def load_q4_k_q5_k_expert_sources( + model_path: str, config, *, layer_sink=None +) -> QwenGGUFExpertSources: + """Load byte-exact Qwen GGUF experts into per-layer host banks. + + The loader fuses separately stored `ffn_gate_exps` and `ffn_up_exps` rows + along their output dimension, which is safe because both use the same Q4_K + input-row geometry. Most down rows remain Q5_K. The explicit Q6_K late + layers are held in a compact side list for an auxiliary cache instead of + being coerced into the primary Q5_K bank. + """ + from freetoken.models.gguf.reader import iter_gguf_tensors + from freetoken.moe.host_banks import ( + LayerCompletionTracker, + PinPipeline, + alloc_layer_banks, + ) + + _require_tp1() + layers = int(config.num_layers) + experts = int(config.num_experts) + hidden = int(config.hidden_size) + intermediate = int(config.moe_intermediate_size) + gate_row_bytes = row_bytes(hidden, GGML_Q4_K) + down_row_bytes = row_bytes(intermediate, GGML_Q5_K) + q6_down_row_bytes = row_bytes(intermediate, GGML_Q6_K) + q6_layer_ids = tuple(int(layer) for layer in getattr(config, "gguf_q6_down_layer_ids", ())) + q6_index = {layer: index for index, layer in enumerate(q6_layer_ids)} + if layer_sink is not None and q6_layer_ids: + raise NotImplementedError( + "Qwen GGUF FTW conversion does not yet serialize the auxiliary Q6_K down banks" + ) + host_banks = alloc_layer_banks(_expert_specs(config), layers) + banks = {name: [bank.tensor for bank in host_banks[name]] for name in host_banks} + q6_host_banks = alloc_layer_banks(_q6_down_specs(config), len(q6_layer_ids)) + q6_down = [bank.tensor for bank in q6_host_banks["down"]] + gate_seen: set[int] = set() + up_seen: set[int] = set() + down_seen: set[int] = set() + completed_gate_up: set[int] = set() + + def load(sink) -> None: + # A completed layer consists of a fused Q4_K gate/up bank and one Q5_K down bank. + tracker = LayerCompletionTracker(2, host_banks, sink) if sink is not None else None + for tensor in iter_gguf_tensors(model_path): + if not tensor.name.startswith("blk."): + continue + parts = tensor.name.split(".") + layer = int(parts[1]) + # What: skip blocks beyond allocated decoder banks; why: trailing MTP experts are unsupported predictor weights. + if layer >= layers: + # What: continue before bank indexing; why: exclusion prevents an out-of-range write and accidental MTP loading. + continue + suffix = ".".join(parts[2:]) + if suffix == "ffn_gate_exps.weight": + if tensor.ggml_type != GGML_Q4_K: + raise ValueError(f"{tensor.name} expected Q4_K, got {tensor.ggml_type}") + banks["gate_up"][layer][:, :intermediate].copy_( + tensor.packed().reshape(experts, intermediate, gate_row_bytes) + ) + gate_seen.add(layer) + elif suffix == "ffn_up_exps.weight": + if tensor.ggml_type != GGML_Q4_K: + raise ValueError(f"{tensor.name} expected Q4_K, got {tensor.ggml_type}") + banks["gate_up"][layer][:, intermediate:].copy_( + tensor.packed().reshape(experts, intermediate, gate_row_bytes) + ) + up_seen.add(layer) + elif suffix == "ffn_down_exps.weight": + if layer in q6_index: + if tensor.ggml_type != GGML_Q6_K: + raise ValueError(f"{tensor.name} expected Q6_K, got {tensor.ggml_type}") + q6_down[q6_index[layer]].copy_( + tensor.packed().reshape(experts, hidden, q6_down_row_bytes) + ) + # The primary cache must retain one uniform Q5_K bank shape. The + # Q6 layers never read this placeholder because their execution + # uses the auxiliary Q6_K cache. + banks["down"][layer].zero_() + else: + if tensor.ggml_type != GGML_Q5_K: + raise ValueError(f"{tensor.name} expected Q5_K, got {tensor.ggml_type}") + banks["down"][layer].copy_( + tensor.packed().reshape(experts, hidden, down_row_bytes) + ) + down_seen.add(layer) + if tracker is not None: + tracker.note(layer) + else: + continue + if layer in gate_seen and layer in up_seen and layer not in completed_gate_up: + completed_gate_up.add(layer) + if tracker is not None: + tracker.note(layer) + + if layer_sink is not None: + load(layer_sink) + elif torch.cuda.is_available(): + with PinPipeline() as pins: + load(pins) + for bank in q6_host_banks["down"]: + pins.submit(bank) + else: + load(None) + + wanted = set(range(layers)) + assert gate_seen == wanted and up_seen == wanted and down_seen == wanted, ( + "incomplete Qwen GGUF expert tensors: " + f"gate={sorted(wanted - gate_seen)}, up={sorted(wanted - up_seen)}, " + f"down={sorted(wanted - down_seen)}" + ) + return QwenGGUFExpertSources(banks, q6_down, q6_layer_ids) + + +def dummy_q4_k_q5_k_expert_sources(config) -> QwenGGUFExpertSources: + """Build correctly shaped random packed banks for loader and cache tests.""" + from freetoken.moe.host_banks import alloc_layer_banks, pin_banks + + host_banks = alloc_layer_banks(_expert_specs(config), int(config.num_layers)) + banks = {name: [bank.tensor for bank in host_banks[name]] for name in host_banks} + q6_layer_ids = tuple(int(layer) for layer in getattr(config, "gguf_q6_down_layer_ids", ())) + q6_host_banks = alloc_layer_banks(_q6_down_specs(config), len(q6_layer_ids)) + q6_down = [bank.tensor for bank in q6_host_banks["down"]] + for tensor in banks["gate_up"] + banks["down"]: + tensor.random_(0, 256) + for tensor in q6_down: + tensor.random_(0, 256) + if torch.cuda.is_available(): + pin_banks(host_banks) + pin_banks(q6_host_banks) + return QwenGGUFExpertSources(banks, q6_down, q6_layer_ids) + + +__all__ = [ + "convert_qwen3_5_to_gguf", + "dummy_q4_k_q5_k_expert_sources", + "is_gguf_model", + "iter_gguf_weights", + "load_q4_k_q5_k_expert_sources", +] diff --git a/python/freetoken/models/qwen3_5_moe/model.py b/python/freetoken/models/qwen3_5_moe/model.py index eba7fd24f1..c268a34aeb 100644 --- a/python/freetoken/models/qwen3_5_moe/model.py +++ b/python/freetoken/models/qwen3_5_moe/model.py @@ -44,6 +44,7 @@ def __init__(self, config: ModelConfig, layer_id: int): layer_id=layer_id, expert_quant=config.expert_quant, attn_quant=config.attn_quant, + config=config, ) else: self.self_attn = Qwen3_5Attention(config, layer_id) @@ -91,7 +92,13 @@ def forward(self, input_ids: torch.Tensor) -> torch.Tensor: class Qwen3_5MoEForCausalLM(BaseLLMModel): def __init__(self, config: ModelConfig): self.model = Qwen3_5Model(config) - if getattr(config, "lm_head_quant", "none") == "nvfp4": + from .gguf import convert_qwen3_5_to_gguf, is_gguf_model + + if is_gguf_model(config): + # GGUF has its own packed embedding, dense Q8_0 projections and untied + # Q6_K output projection. Install every replacement before loading state. + convert_qwen3_5_to_gguf(self, config) + elif getattr(config, "lm_head_quant", "none") == "nvfp4": # checkpoint stores the (untied) lm_head as NVFP4: keep it native (W4A16) -- the # bf16 dequant of this ~1 GB matrix was the single largest decode kernel. from freetoken.kernel.triton.nvfp4_linear import Nvfp4LMHead diff --git a/python/freetoken/models/qwen3_5_moe/moe.py b/python/freetoken/models/qwen3_5_moe/moe.py index fc0bb7c22c..b5ab1cf3d0 100644 --- a/python/freetoken/models/qwen3_5_moe/moe.py +++ b/python/freetoken/models/qwen3_5_moe/moe.py @@ -69,7 +69,10 @@ def __init__(self, config: ModelConfig, layer_id: int | None = None): "fp8_block" if getattr(config, "expert_quant", "none") == "fp8_block" else "bf16" ) self.experts = make_moe_layer( - config, layer_id=layer_id, renormalize=True, weight_format=weight_format + config, + layer_id=layer_id, + renormalize=config.norm_topk_prob, + weight_format=weight_format, ) self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False) self.shared_expert = _SharedExpert( diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.py index b341408903..d07f18cd7b 100644 --- a/python/freetoken/models/qwen3_5_moe/weight.py +++ b/python/freetoken/models/qwen3_5_moe/weight.py @@ -911,11 +911,17 @@ def _build_fp8_expert_banks( B = 128 L, E, H, I, dense = _moe_dims(config) + + # 16B-align the per-expert scale rows (Qwen3.8: down_scale is 20x5 bf16 = 200 B) so the + # fused multi-bank copy engages; the GEMMs read scales through explicit strides, so the + # padding is inert. Unconditional: one layout per format, shared with the byte formulas. + from freetoken.moe.offload_cache import fp8_block_scale_pad as _pad_cols + specs = { "gate_up": ((E, 2 * I, H), FP8), - "gate_up_scale": ((E, 2 * I // B, H // B), torch.bfloat16), + "gate_up_scale": ((E, 2 * I // B, _pad_cols(2 * I // B, H // B)), torch.bfloat16), "down": ((E, H, I), FP8), - "down_scale": ((E, H // B, I // B), torch.bfloat16), + "down_scale": ((E, H // B, _pad_cols(H // B, I // B)), torch.bfloat16), } hb = None if pin: @@ -950,8 +956,9 @@ def place(raw_name: str, t: torch.Tensor) -> int | None: (gate_up[li][e, :I] if proj == "gate" else gate_up[li][e, I:] if proj == "up" else down[li][e]).copy_(t) else: # weight_scale_inv - (gate_up_scale[li][e, : I // B] if proj == "gate" else - gate_up_scale[li][e, I // B:] if proj == "up" else down_scale[li][e]).copy_(t) + (gate_up_scale[li][e, : I // B, : H // B] if proj == "gate" else + gate_up_scale[li][e, I // B :, : H // B] if proj == "up" else + down_scale[li][e, :, : I // B]).copy_(t) return li if parallel is None: diff --git a/python/freetoken/models/qwen4_exp/__init__.py b/python/freetoken/models/qwen4_exp/__init__.py new file mode 100644 index 0000000000..04c29d7785 --- /dev/null +++ b/python/freetoken/models/qwen4_exp/__init__.py @@ -0,0 +1,35 @@ +"""Qwen3.8-Flash-Next (model_type qwen4_exp), served text-only. + +48 decoder layers on hc_count=4 hyper-connection residual streams R [T, 4*hidden]: +embed -> repeat(1, 4) -> [PLE at zero-based layer 1] -> per layer attn_hc.mix -> (GDN | QSA) -> attn_hc.combine -> mlp_hc.mix -> MoE -> mlp_hc.combine -> top-level mixer.mix -> lm_head. +Layer contract: forward(R [T, 4*hidden], batch) -> R' [T, 4*hidden]. + +Contracts shared across modules (do not rename): +- The PLE dilated-conv left context lives on the LinearStatePool slots as the declared slot state ``ple_conv`` (config.ple_slot_states -> ModelConfig.slot_states), read back with ``pool.slot_state("ple_conv", layer_id)``; same slot / COW / snapshot lifecycle as conv_states / recurrent_states. +- kvcache.qsa_pool.QSAKVCache(MHAKVCache): ``cmp_k_cache(slot) -> [rows, index_head_dim]`` (compressed index keys, row = kv slot // index_ratio), ``pending_ring(slot) -> [num_req_slots, ring_capacity, index_head_dim]`` (per-request pre-RoPE index-k tail indexed by table_idx, never cleared), ``cmp_scratch_base`` (int, first scratch row for non-closing decode writes). ``slot`` is the sparse layer's order in the attention backend. +""" + +from .config import parse_config +from .model import Qwen4ExpForCausalLM +from .weight import ( + iter_weights, + load_nvfp4_expert_sources, + load_nvfp4_expert_sources_parallel, + load_ple_table, +) + +# Official FP8 checkpoints share qwen3_5_moe's block-fp8 expert layout (same +# model.language_model.layers.* keys), so reuse its bank hook; for every other +# expert_quant it defers to the per-quant providers, which resolve this module's +# load_nvfp4_expert_sources via the model spec. +from freetoken.models.qwen3_5_moe.weight import setup_offload_expert_banks + +__all__ = [ + "Qwen4ExpForCausalLM", + "iter_weights", + "load_nvfp4_expert_sources", + "load_nvfp4_expert_sources_parallel", + "load_ple_table", + "parse_config", + "setup_offload_expert_banks", +] diff --git a/python/freetoken/models/qwen4_exp/attention.py b/python/freetoken/models/qwen4_exp/attention.py new file mode 100644 index 0000000000..d6ab2867af --- /dev/null +++ b/python/freetoken/models/qwen4_exp/attention.py @@ -0,0 +1,241 @@ +"""QSA full-attention layer for Qwen3.8-Flash-Next (12 of 48 layers). + +Gated GQA (24 q heads / 2 kv heads / head_dim 256, per-head zero-centered q/k norms, partial NeoX +rope over 64 dims, ``q_proj`` twice as wide for the output gate) plus the weights of the QSA +indexer (``index_qk_proj`` [640, 2560] = 4 index q heads x 128 then 1 index k head x 128, and the +two per-head index norms). + +Model/backend split, same shape as MiniMax-M3's ``bsa_forward``: the layer owns the weights and +hands the backend the RAW index projections; the backend owns everything stateful (compressed key +slab, pending ring, scoring, top-k, expansion, sparse attend). The index k norm runs AFTER the +fp32 mean over each group of ``index_ratio`` raw keys, so it cannot be applied here -- both index +norm weights travel with the call (:class:`QSAIndexerInputs`). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol + +import torch +from freetoken.core import get_global_ctx +from freetoken.layers import BaseOP, GemmaPlusOneRMSNorm, LinearColParallelMerged, LinearReplicated +from freetoken.layers.rotary import get_rope +from freetoken.utils import nvtx_annotate + +if TYPE_CHECKING: + from freetoken.core import Batch + from freetoken.models.config import ModelConfig + + +@dataclass(frozen=True) +class QSAIndexerInputs: + """Everything the QSA backend needs from the indexer for one layer's forward. + + Frozen contract. ``q``/``k`` are the raw ``index_qk_proj`` slices: no norm, no rope. The + backend applies, per HF ``Qwen4ExpTextQSAIndexer`` (modeling_qwen4_exp.py:611):: + + q_h = rope64(rmsnorm(q_h) * (1 + q_norm_weight), pos = query position) + kbar_b = rope64(rmsnorm(mean_fp32(k[4b:4b+4])) * (1 + k_norm_weight), pos = 4b) + s_b = sum_h relu() / sqrt(index_head_dim) + + and the pending ring stores ``k`` PRE-norm and PRE-rope, because a group's mean is only final + once all ``index_ratio`` members exist. rope64 is ``get_rope(index_head_dim, + config.rotary_config.rotary_dim, ...)`` -- the same frequencies as the main attention, a + different ``head_size``, so the backend builds its own (cached) instance. + """ + + q: torch.Tensor # [T, index_n_heads, index_head_dim] + k: torch.Tensor # [T, index_head_dim] + q_norm_weight: torch.Tensor # [index_head_dim], zero-centered: scale is (1 + w), fp32 + k_norm_weight: torch.Tensor # [index_head_dim], zero-centered + eps: float + + +class QSAAttentionBackend(Protocol): + """The hook ``Qwen4ExpAttention`` calls; ``attention/qsa_sparse.py`` implements it. + + ``q`` is [T, num_qo_heads, head_dim] and ``k``/``v`` are [T, num_kv_heads*head_dim], all post + norm+rope and in the model dtype; the return is [T, num_qo_heads, head_dim] (the layer applies + the output gate and ``o_proj``). ``layer_id`` is the decoder layer id; the backend maps it to + its own sparse-layer slot. Everything else -- KV store, per-request lengths, page rows -- comes + from ``batch`` exactly as for ``BaseAttnBackend.forward``. + """ + + def qsa_forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + index: QSAIndexerInputs, + layer_id: int, + batch: Batch, + ) -> torch.Tensor: ... + + +class Qwen4ExpIndexer(BaseOP): + """QSA indexer weights (checkpoint prefix ``self_attn.indexer``); the scoring lives in the backend.""" + + def __init__(self, config: ModelConfig, layer_id: int) -> None: + args = config.qwen4_args + self.layer_id = layer_id + self.num_heads = args.index_n_heads + self.num_kv_heads = args.index_kv_heads + self.head_dim = args.index_head_dim + self.eps = config.rms_norm_eps + self._split = [self.num_heads * self.head_dim, self.num_kv_heads * self.head_dim] + self.index_qk_proj = LinearReplicated(args.hidden_size, sum(self._split), has_bias=False) + self.q_layernorm = GemmaPlusOneRMSNorm(self.head_dim, eps=self.eps) + self.k_layernorm = GemmaPlusOneRMSNorm(self.head_dim, eps=self.eps) + + def forward(self, x: torch.Tensor) -> QSAIndexerInputs: + q, k = self.index_qk_proj.forward(x).split(self._split, dim=-1) + return QSAIndexerInputs( + q=q.reshape(-1, self.num_heads, self.head_dim).contiguous(), + k=k.reshape(-1, self.head_dim).contiguous(), + q_norm_weight=self.q_layernorm.weight, + k_norm_weight=self.k_layernorm.weight, + eps=self.eps, + ) + + +class Qwen4ExpAttention(BaseOP): + """Gated GQA with a QSA indexer:: + + q, gate = chunk(q_proj(x).view(-1, num_q, 2*head_dim), 2, -1) + q, k = rope(q_norm(q), k_norm(k_proj(x))) # first rotary_dim dims + o = backend.qsa_forward(q, k, v_proj(x), indexer(x), layer_id, batch) + out = o_proj(o * sigmoid(gate)) + + q/k/v are one merged GEMM (``qkv_proj``, split ``[num_q*head_dim*2, kv, kv]``); the checkpoint + ships ``q_proj``/``k_proj``/``v_proj`` separately, so the loader concatenates along dim 0. + Other keys keep the checkpoint names: ``o_proj.weight``, ``q_norm.weight``, ``k_norm.weight`` + (both zero-centered, loaded RAW), ``indexer.*``. + """ + + def __init__(self, config: ModelConfig, layer_id: int) -> None: + self.layer_id = layer_id + self.num_q = config.num_qo_heads + self.num_kv = config.num_kv_heads + self.head_dim = config.head_dim + self.qo_attn_dim = self.num_q * self.head_dim + self.kv_attn_dim = self.num_kv * self.head_dim + self._qkv_split = [self.qo_attn_dim * 2, self.kv_attn_dim, self.kv_attn_dim] + self.qkv_proj = LinearColParallelMerged( + config.hidden_size, self._qkv_split, has_bias=False + ) + self.o_proj = LinearReplicated(self.qo_attn_dim, config.hidden_size, has_bias=False) + self.q_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.k_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=config.rms_norm_eps) + rotary = config.rotary_config + self.rotary = get_rope( + head_dim=self.head_dim, + rotary_dim=rotary.rotary_dim, + max_position=rotary.max_position, + base=rotary.base, + rope_scaling=tuple(rotary.scaling.items()) if rotary.scaling else None, + ) + self.indexer = Qwen4ExpIndexer(config, layer_id) + + @nvtx_annotate("QSA") + def forward(self, x: torch.Tensor, batch: Batch) -> torch.Tensor: + qg, k, v = self.qkv_proj.forward(x).split(self._qkv_split, dim=-1) + qg = qg.view(-1, self.num_q, self.head_dim * 2) + q = qg[..., : self.head_dim].contiguous() + gate = qg[..., self.head_dim :].reshape(-1, self.qo_attn_dim) + k = k.contiguous().view(-1, self.num_kv, self.head_dim) + v = v.contiguous() + self.q_norm.forward_inplace(q) + self.k_norm.forward_inplace(k) + q, k = self.rotary.forward( + batch.positions, q.view(-1, self.qo_attn_dim), k.view(-1, self.kv_attn_dim) + ) + index = self.indexer.forward(x) + o = get_global_ctx().attn_backend.qsa_forward( + q.view(-1, self.num_q, self.head_dim), k, v, index, self.layer_id, batch + ) + gated = o.reshape(-1, self.qo_attn_dim) * torch.sigmoid(gate) + return self.o_proj.forward(gated) + + +class TorchDenseQSAReference: + """Dense oracle for :class:`QSAAttentionBackend` (fp32 math): attend to every visible token. + + QSA is exactly dense while a request sees at most ``index_budget + index_ratio - 1`` tokens + (every complete block is selected), so this doubles as the equivalence oracle for the sparse backend. It + keeps its own ``[slot, position]`` KV instead of a paged pool, so it needs no engine wiring; + it is a test/reference object and is never registered as an attention backend. + """ + + def __init__( + self, + config: ModelConfig, + num_slots: int, + max_len: int, + device: torch.device, + dtype: torch.dtype, + ) -> None: + self.num_kv = config.num_kv_heads + self.head_dim = config.head_dim + self.sm_scale = config.attn_sm_scale or self.head_dim**-0.5 + self._cache: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} + self._shape = (num_slots, max_len, self.num_kv, self.head_dim) + self._device = device + self._dtype = dtype + + def _layer_cache(self, layer_id: int) -> tuple[torch.Tensor, torch.Tensor]: + if layer_id not in self._cache: + self._cache[layer_id] = tuple( + torch.zeros(self._shape, device=self._device, dtype=self._dtype) for _ in range(2) + ) + return self._cache[layer_id] + + def qsa_forward( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + index: QSAIndexerInputs, + layer_id: int, + batch: Batch, + ) -> torch.Tensor: + del index + k_cache, v_cache = self._layer_cache(layer_id) + k = k.view(-1, self.num_kv, self.head_dim) + v = v.view(-1, self.num_kv, self.head_dim) + out = torch.empty_like(q) + offset = 0 + for r in batch.padded_reqs: + n, slot, prefix = r.extend_len, r.table_idx, r.cached_len + rows = slice(offset, offset + n) + k_cache[slot, prefix : prefix + n] = k[rows] + v_cache[slot, prefix : prefix + n] = v[rows] + out[rows] = self._attend( + q[rows], k_cache[slot, : prefix + n], v_cache[slot, : prefix + n], prefix + ) + offset += n + return out + + def _attend( + self, q: torch.Tensor, keys: torch.Tensor, values: torch.Tensor, prefix: int + ) -> torch.Tensor: + n, num_q, _ = q.shape + total = keys.shape[0] + rep = num_q // self.num_kv + keys = keys.repeat_interleave(rep, dim=1).float() + values = values.repeat_interleave(rep, dim=1).float() + scores = torch.einsum("qhd,khd->hqk", q.float(), keys) * self.sm_scale + visible = torch.arange(total, device=q.device) <= ( + prefix + torch.arange(n, device=q.device) + ).unsqueeze(-1) + scores = scores.masked_fill(~visible, float("-inf")) + return torch.einsum("hqk,khd->qhd", scores.softmax(-1), values).to(q.dtype) + + +__all__ = [ + "QSAAttentionBackend", + "QSAIndexerInputs", + "Qwen4ExpAttention", + "Qwen4ExpIndexer", + "TorchDenseQSAReference", +] diff --git a/python/freetoken/models/qwen4_exp/config.py b/python/freetoken/models/qwen4_exp/config.py new file mode 100644 index 0000000000..bb5d1dff52 --- /dev/null +++ b/python/freetoken/models/qwen4_exp/config.py @@ -0,0 +1,293 @@ +from __future__ import annotations + +from dataclasses import dataclass +from fnmatch import fnmatch +from typing import Any, Tuple + +import torch + +from freetoken.models.config import ( + FullAttentionGroupConfig, + LinearGatedDeltaGroupConfig, + ModelConfig, + RotaryConfig, + SlotStateSpec, +) + + +@dataclass(frozen=True) +class Qwen4ExpArgs: + """Qwen3.8-Flash-Next geometry beyond the generic ModelConfig fields (ModelConfig.qwen4_args).""" + + hidden_size: int + # Hyper-connections: every layer reads/writes hc_count residual streams [T, hc_count*hidden]. + hc_count: int + hc_lowrank: int + # PLE n-gram embedding; layer ids are zero-based decoder layers. + ple_layer_ids: Tuple[int, ...] + ple_embed_dim: int + ple_conv_kernel_size: int + ngram_size: int + heads_per_ngram: int + ngram_vocab_size_base: int + make_ngram_vocab_size_divisible_by: int + split_ngram_parts: int + # n-gram hash windows never cross this token (the eos id); they restart after it. + ngram_boundary_token_id: int + # QSA indexer scoring geometry (the slab/ratio geometry lives on the attention group). + index_n_heads: int + index_kv_heads: int + index_head_dim: int + index_budget: int + index_ratio: int + + @property + def index_topk_blocks(self) -> int: + return self.index_budget // self.index_ratio + + @property + def num_ngram_heads(self) -> int: + # one head group per n-gram order 2..ngram_size (Qwen3.8: 8 x 2-gram + 8 x 3-gram) + return (self.ngram_size - 1) * self.heads_per_ngram + + @property + def ngram_head_dim(self) -> int: + return self.ple_embed_dim // self.num_ngram_heads + + @property + def ple_conv_dilation(self) -> int: + # HF Qwen4ExpTextPLELayer sets the depthwise conv dilation to ngram_size + return self.ngram_size + + @property + def ple_conv_state_len(self) -> int: + return (self.ple_conv_kernel_size - 1) * self.ple_conv_dilation + + @property + def ple_state_width(self) -> int: + return self.hc_count * self.hidden_size + + +PLE_CONV_STATE = "ple_conv" +PLE_NGRAM_STATE = "ple_ngram_ctx" + + +def ple_slot_states(args: Qwen4ExpArgs) -> Tuple[SlotStateSpec, ...]: + """Per-request PLE state riding the linear-state slots (see LinearStatePool.slot_states).""" + if not args.ple_layer_ids: + return () + return ( + # dilated-conv left context; replicated (not TP-sharded), model dtype + SlotStateSpec( + name=PLE_CONV_STATE, + shape=(args.ple_state_width, args.ple_conv_state_len), + layer_ids=args.ple_layer_ids, + ), + # last ngram_size-1 token ids, shared by every PLE layer; eos = hash boundary + SlotStateSpec( + name=PLE_NGRAM_STATE, + shape=(args.ngram_size - 1,), + dtype=torch.int32, + fill_value=float(args.ngram_boundary_token_id), + ), + ) + + +def _quant_get(hf_config: Any): + quant = getattr(hf_config, "quantization_config", None) + if quant is None: + return None + return quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d)) + + +def _ignored(patterns, module_name: str) -> bool: + return any(fnmatch(module_name, pat) for pat in patterns) + + +def _layer_types(text: Any) -> list[str]: + layer_types = getattr(text, "layer_types", None) + if layer_types is not None: + # HF Qwen4ExpTextConfig rewrites full_attention to qwen_sparse_attention in __post_init__. + return [ + "full_attention" if t == "qwen_sparse_attention" else t for t in layer_types + ] + # Fall back to full_attention_interval: every Nth layer (1-indexed) is full. + interval = int(getattr(text, "full_attention_interval", 4)) + n = int(text.num_hidden_layers) + return [ + "full_attention" if (i + 1) % interval == 0 else "linear_attention" + for i in range(n) + ] + + +def parse_config(hf_config: Any) -> ModelConfig: + text = getattr(hf_config, "text_config", hf_config) + + head_dim = ( + getattr(text, "head_dim", None) + or text.hidden_size // text.num_attention_heads + ) + num_kv_heads = getattr(text, "num_key_value_heads", text.num_attention_heads) + + rope_params = getattr(text, "rope_parameters", None) or {} + rope_theta = rope_params.get("rope_theta", getattr(text, "rope_theta", None)) + partial = ( + rope_params.get("partial_rotary_factor") + or getattr(text, "partial_rotary_factor", None) + or 1.0 + ) + # int(), not round(): HF configuration_qwen4_exp truncates head_dim * partial. + rotary_dim = int(head_dim * partial) + + # Text-only serving with the default rope type: the mRoPE sections reduce to standard + # partial rope, and the unhashable ``mrope_section`` list must not reach get_rope's + # cache key. + rope_type = rope_params.get("rope_type", "default") + rope_scaling = ( + None + if rope_type in (None, "default") + else {k: v for k, v in rope_params.items() if not isinstance(v, (list, dict))} + ) + + get = _quant_get(hf_config) + if get is None: + expert_quant = attn_quant = dense_quant = lm_head_quant = "none" + else: + algo = str(get("quant_algo") or get("quant_method") or "").lower() + block = get("weight_block_size") + if algo == "fp8" and block: + # Official FP8 build (DeepSeek-V3-style block-fp8): only the routed experts + # are quantized (fp8-e4m3 weights + per-block weight_scale_inv); attention, + # GDN, the shared expert, HC, PLE and lm_head stay bf16. + bs = tuple(int(x) for x in block) + assert bs == (128, 128), f"only 128x128 block-fp8 is supported, got {bs}" + expert_quant = "fp8_block" + attn_quant = dense_quant = lm_head_quant = "none" + else: + is_fp4 = "fp4" in algo + ignore = list(get("ignore") or []) + + # The RadixArk NVFP4 build quantizes only the routed experts; attention/GDN, + # the shared expert, HC, PLE and lm_head all sit in the modelopt ignore list + # and stay bf16. Derive every flag from that list instead of assuming the split. + def _quant(probe: str) -> str: + return "nvfp4" if is_fp4 and not _ignored(ignore, probe) else "none" + + prefix = "model.language_model.layers.0" + expert_quant = _quant(f"{prefix}.mlp.experts.0.gate_proj") + dense_quant = _quant(f"{prefix}.mlp.shared_expert.gate_proj") + attn_quant = _quant(f"{prefix}.self_attn.q_proj") + lm_head_quant = _quant("lm_head") + + layer_types = _layer_types(text) + full_ids = tuple(i for i, t in enumerate(layer_types) if t == "full_attention") + linear_ids = tuple(i for i, t in enumerate(layer_types) if t == "linear_attention") + + # HF stores ple_layer_ids one-indexed (validated upstream as [1, num_layers]). + ple_layer_ids = tuple(int(i) - 1 for i in (getattr(text, "ple_layer_ids", None) or ())) + for lid in ple_layer_ids: + if layer_types[lid] != "linear_attention": + raise ValueError(f"PLE must sit on a linear_attention layer, got layer {lid}") + + full_rotary = RotaryConfig( + head_dim=head_dim, + rotary_dim=rotary_dim, + max_position=text.max_position_embeddings, + base=rope_theta, + scaling=rope_scaling, + ) + full_group = FullAttentionGroupConfig( + name="full", + layer_ids=full_ids, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + rotary_config=full_rotary, + index_head_dim=int(text.indexer_head_dim), + num_index_layers=len(full_ids), + index_ratio=int(text.indexer_compress_ratio), + ) + linear_group = LinearGatedDeltaGroupConfig( + name="linear", + layer_ids=linear_ids, + num_key_heads=text.linear_num_key_heads, + num_value_heads=text.linear_num_value_heads, + key_head_dim=text.linear_key_head_dim, + value_head_dim=text.linear_value_head_dim, + conv_kernel_dim=text.linear_conv_kernel_dim, + # HF resolves a null output_gate_type to hidden_act; mirror that instead of + # stringifying None. + output_gate=str(getattr(text, "output_gate_type", None) or text.hidden_act), + ) + # Order groups by their first layer id for deterministic iteration. + groups = tuple( + sorted( + (full_group, linear_group), + key=lambda g: g.layer_ids[0] if g.layer_ids else 1 << 30, + ) + ) + + num_experts = int(getattr(text, "num_experts", 0) or 0) + + # HF accepts int | list here and uses the first entry (modeling_qwen4_exp Qwen4ExpTextNGramEmbedding) + eos_token_id = text.eos_token_id + if isinstance(eos_token_id, (list, tuple)): + eos_token_id = eos_token_id[0] + + qwen4_args = Qwen4ExpArgs( + hidden_size=text.hidden_size, + hc_count=int(text.hc_count), + hc_lowrank=int(text.hc_lowrank), + ple_layer_ids=ple_layer_ids, + ple_embed_dim=int(text.ple_embed_dim), + ple_conv_kernel_size=int(text.ple_conv_kernel_size), + ngram_size=int(text.ngram_size), + heads_per_ngram=int(text.heads_per_ngram), + ngram_vocab_size_base=int(text.ngram_vocab_size_base), + make_ngram_vocab_size_divisible_by=int(text.make_ngram_vocab_size_divisible_by), + split_ngram_parts=int(text.split_ngram_parts), + ngram_boundary_token_id=int(eos_token_id), + index_n_heads=int(text.indexer_n_heads), + index_kv_heads=int(text.indexer_kv_heads), + index_head_dim=int(text.indexer_head_dim), + index_budget=int(text.indexer_budget), + index_ratio=int(text.indexer_compress_ratio), + ) + + return ModelConfig( + num_layers=text.num_hidden_layers, + num_qo_heads=text.num_attention_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + hidden_size=text.hidden_size, + vocab_size=text.vocab_size, + intermediate_size=getattr(text, "intermediate_size", 0) or 0, + hidden_act=text.hidden_act, + rms_norm_eps=text.rms_norm_eps, + tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)), + rotary_config=full_rotary, + num_experts=num_experts, + num_experts_per_tok=int(getattr(text, "num_experts_per_tok", 0) or 0), + moe_intermediate_size=int(getattr(text, "moe_intermediate_size", 0) or 0), + shared_expert_intermediate_size=int( + getattr(text, "shared_expert_intermediate_size", 0) or 0 + ), + # Absent from the shipped config; HF Qwen4ExpTextConfig defaults it True and the + # Qwen3_5MoE block renormalizes unconditionally -- keep the two in agreement. + norm_topk_prob=bool(getattr(text, "norm_topk_prob", True)), + moe_enabled=num_experts > 0, + use_qk_norm=True, + model_type=getattr(hf_config, "model_type", "qwen4_exp"), + architectures=getattr(hf_config, "architectures", ["Qwen4ExpForConditionalGeneration"]), + vision_config=None, # served text-only + image_token_id=getattr(hf_config, "image_token_id", None), + attention_groups=groups, + expert_quant=expert_quant, + attn_quant=attn_quant, + dense_quant=dense_quant, + lm_head_quant=lm_head_quant, + qwen4_args=qwen4_args, + slot_states=ple_slot_states(qwen4_args), + ) + + +__all__ = ["PLE_CONV_STATE", "PLE_NGRAM_STATE", "Qwen4ExpArgs", "parse_config", "ple_slot_states"] diff --git a/python/freetoken/models/qwen4_exp/gdn.py b/python/freetoken/models/qwen4_exp/gdn.py new file mode 100644 index 0000000000..69838153f5 --- /dev/null +++ b/python/freetoken/models/qwen4_exp/gdn.py @@ -0,0 +1,233 @@ +from __future__ import annotations + +import torch +import torch.nn.functional as F +from freetoken.core import get_global_ctx +from freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen +from freetoken.layers import BaseOP, LinearColParallelMerged + +from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged +from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged +from freetoken.models.qwen3_5_moe.gdn_kernels import gdn_decode_fla, gdn_prefill_chunk_fla +from freetoken.models.quant_linear import make_replicated_quant + + +_GATE_ACTIVATIONS = ("silu", "swish", "sigmoid") + + +class _DepthwiseConv1d(BaseOP): + """Holds the depthwise conv weight ``[conv_dim, 1, K]`` (key ``conv1d.weight``).""" + + def __init__(self, conv_dim: int, kernel: int): + self.weight = torch.empty(conv_dim, 1, kernel) + + +class _GatedRMSNorm(BaseOP): + """RMSNorm of x followed by an ``activation(z)`` gate (HF Qwen4ExpTextRMSNormGated). + + Uses the fused fla ``rms_norm_gated`` triton kernel (norm(x) * act(z) in one + kernel) instead of the unfused pow/mean/rsqrt/mul/act chain, matching sglang's + ``RMSNormGated`` -- collapses ~8 elementwise kernels per GDN layer into one. + Qwen3.8-Flash-Next gates with sigmoid where Qwen3.5 gates with silu.""" + + def __init__(self, dim: int, eps: float, activation: str): + # rms_norm_gated drops the gate entirely (no error) for a name it does not know. + assert activation in _GATE_ACTIVATIONS, f"unsupported GDN output gate {activation!r}" + self.weight = torch.empty(dim) + self.eps = eps + self.activation = activation + + def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor: + from freetoken.kernel.fla import rms_norm_gated + + return rms_norm_gated( + x=x, weight=self.weight, bias=None, z=z, eps=self.eps, + is_rms_norm=True, norm_before_gate=True, activation=self.activation, + ) + + +class Qwen4ExpGatedDeltaNet(BaseOP): + """GatedDeltaNet op using the vendored flash-linear-attention triton kernels + (``freetoken.kernel.fla``) for the recurrence and a per-request + recurrent + conv state held in ``ctx.linear_state_pool`` (keyed by ``Req.table_idx``). + + Parameter names match HF (``in_proj_qkv``/``in_proj_z``/``in_proj_b``/``in_proj_a``/ + ``conv1d``/``A_log``/``dt_bias``/``norm``/``out_proj``). Handles prefill (incl. chunked + continuation) and single-token decode; state is fresh when ``req.cached_len == 0``. + + ``output_gate`` is the gate activation name from ``LinearGatedDeltaGroupConfig`` + ("sigmoid" for Qwen3.8-Flash-Next). + """ + + def __init__( + self, hidden_size, num_k_heads, num_v_heads, head_k_dim, head_v_dim, + conv_kernel_size, rms_norm_eps, layer_id, output_gate: str = "sigmoid", + expert_quant: str = "none", attn_quant: str = "none", + ): + self.layer_id = layer_id + # The fla chunk/decode kernels read+write the recurrent state and the per-chunk h as + # [V, K] while the LinearStatePool declares it [K, V]; these coincide (and the + # hybrid-radix snapshot scatter h[h_row]->slot is a plain copy) only when the two head + # dims are equal. Qwen3.5/3.6/3.8 satisfy this (128/128); guard any future config. + assert head_k_dim == head_v_dim, ( + f"GatedDeltaNet requires head_k_dim == head_v_dim, got {head_k_dim} != {head_v_dim}" + ) + self.num_k_heads = num_k_heads + self.num_v_heads = num_v_heads + self.head_k_dim = head_k_dim + self.head_v_dim = head_v_dim + self.key_dim = num_k_heads * head_k_dim + self.value_dim = num_v_heads * head_v_dim + self.conv_dim = 2 * self.key_dim + self.value_dim + self.conv_kernel_size = conv_kernel_size + # qkv|z carry a weight scale (block-fp8 weight_scale_inv, or per-tensor FP8 + # weight_scale); b|a stay bf16. Both quant modes therefore split the four-way + # fusion into an fp8 qkvz GEMM + a bf16 ba GEMM (matches sglang/vLLM). + self._block_fp8 = expert_quant == "fp8_block" + self._pertensor_fp8 = attn_quant == "fp8_pertensor" + self._fp8 = self._block_fp8 or self._pertensor_fp8 + + self._in_proj_split = [self.conv_dim, self.value_dim, num_v_heads, num_v_heads] + if self._fp8: + ColMerged = Fp8BlockColMerged if self._block_fp8 else Fp8PerTensorColMerged + self.in_proj_qkvz = ColMerged( + hidden_size, [self.conv_dim, self.value_dim], has_bias=False + ) + self.in_proj_ba = LinearColParallelMerged( + hidden_size, [num_v_heads, num_v_heads], has_bias=False + ) + else: + # Fused input projection (one GEMM instead of four): qkv | z | b | a. + self.in_proj = LinearColParallelMerged(hidden_size, self._in_proj_split, has_bias=False) + self.conv1d = _DepthwiseConv1d(self.conv_dim, conv_kernel_size) + # Recurrence-gating params kept in fp32 (exp/softplus is precision-sensitive, + # and the fla kernel reads them as fp32) -- matches HF/sglang, and avoids a + # per-call .float() upcast in the decode wrapper. The weight loader exempts + # *.A_log / *.dt_bias from the model-dtype downcast. + self.dt_bias = torch.empty(num_v_heads, dtype=torch.float32) + self.A_log = torch.empty(num_v_heads, dtype=torch.float32) + self.norm = _GatedRMSNorm(head_v_dim, eps=rms_norm_eps, activation=output_gate) + # out_proj follows the checkpoint quant: block-fp8 / per-tensor-fp8 / compressed-tensors + # NVFP4 (W4A16) / bf16. in_proj_* stay bf16 in every mode (above), so a compressed-tensors + # NVFP4 checkpoint (attn_quant=="nvfp4") only makes out_proj native FP4. + self.out_proj = make_replicated_quant( + expert_quant, attn_quant, self.value_dim, hidden_size, has_bias=False + ) + + def _gate_params(self, a: torch.Tensor, b: torch.Tensor): + beta = b.sigmoid() + g = -self.A_log.exp() * F.softplus(a.float() + self.dt_bias) + return g, beta + + def _conv_weight(self) -> torch.Tensor: + return self.conv1d.weight.squeeze(1) # [conv_dim, kernel] for the fused kernel + + def _conv_prefill(self, conv_in, pool, cu_seqlens, cache_indices, has_initial_state) -> torch.Tensor: + """Varlen causal conv (fused sgl_kernel) with silu; reads/updates each request's + conv state in place by ``cache_indices`` slot. ``conv_in`` [total, conv_dim]. + ``cu_seqlens`` / ``cache_indices`` / ``has_initial_state`` come from FLAMetadata.""" + li = pool.local_index(self.layer_id) + x = conv_in.transpose(0, 1).contiguous() # [conv_dim, total] + out = causal_conv1d_varlen(x, self._conv_weight(), pool.conv_states[li], + cu_seqlens, cache_indices, has_initial_state) + return out.transpose(0, 1) # [total, conv_dim] + + def _conv_decode(self, conv_in: torch.Tensor, table_idx: torch.Tensor, pool) -> torch.Tensor: + """Single-token causal conv update (fused sgl_kernel) by ``table_idx`` slot; + updates conv state in place, no host loop -> CUDA-graph capturable. + ``conv_in`` [B, conv_dim] -> silu(conv) [B, conv_dim].""" + li = pool.local_index(self.layer_id) + return causal_conv1d_decode(conv_in, pool.conv_states[li], self._conv_weight(), table_idx) + + def _write_track_snapshot(self, pool, li: int, conv_in: torch.Tensor, + h: torch.Tensor, fla) -> None: + """Snapshot this layer's recurrent + conv state at the chunk-aligned track boundary + into a donatable pool slot, on the forward stream (hybrid-radix extra_buffer path). + SSM: ``recurrent_states[li, dst] = h[0, h_row]`` -- a DIRECT copy (h is [V,K], the + state pool is [K,V]; they coincide because GDN requires head_k_dim == head_v_dim). + Conv: the last (kernel-1) raw conv-input timesteps ending at the boundary.""" + rec = pool.recurrent_states[li] + rec.index_copy_(0, fla.track_dst, h[0, fla.track_h_row].to(rec.dtype)) + cv = pool.conv_states[li] + # conv_in [total, conv_dim]; gather the (kernel-1) window per tracked req. + conv_win = conv_in[fla.track_conv_src].transpose(-1, -2).contiguous() # [nt, conv_dim, K-1] + cv.index_copy_(0, fla.track_dst, conv_win.to(cv.dtype)) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + ctx = get_global_ctx() + batch = ctx.batch + pool = ctx.linear_state_pool + total = hidden_states.shape[0] + dtype = hidden_states.dtype + + # Per-forward GDN metadata (cu_seqlens / cache_indices / continuation flags), + # built once and shared by all GDN layers. The scheduler/graph set it; build it + # lazily here (cached on the batch) for direct-op callers (tests). + fla = batch.fla_metadata + if fla is None: + from freetoken.attention.linear import build_fla_metadata + + fla = build_fla_metadata(batch, hidden_states.device) + batch.fla_metadata = fla + + if self._fp8: + qkvz = self.in_proj_qkvz.forward(hidden_states) + conv_in, z = torch.split(qkvz, [self.conv_dim, self.value_dim], dim=-1) + ba = self.in_proj_ba.forward(hidden_states) + b, a = torch.split(ba, [self.num_v_heads, self.num_v_heads], dim=-1) + else: + proj = self.in_proj.forward(hidden_states) + conv_in, z, b, a = torch.split(proj, self._in_proj_split, dim=-1) + z = z.reshape(total, self.num_v_heads, self.head_v_dim) + li = pool.local_index(self.layer_id) + + if batch.is_decode: + # Fused fla decode kernel: gating + in-kernel l2norm + recurrent update + + # per-request state read/write-by-index, all in one kernel (no gather/scatter, + # no clone, no external l2norm). q/k stay at num_k_heads (kernel handles GQA). + mixed = self._conv_decode(conv_in, fla.cache_indices, pool) # [B, conv_dim] + B = mixed.shape[0] + qf, kf, vf = torch.split(mixed, [self.key_dim, self.key_dim, self.value_dim], dim=-1) + q = qf.reshape(1, B, self.num_k_heads, self.head_k_dim).to(dtype) + k = kf.reshape(1, B, self.num_k_heads, self.head_k_dim).to(dtype) + v = vf.reshape(1, B, self.num_v_heads, self.head_v_dim).to(dtype) + core_out = gdn_decode_fla( + q, k, v, a, b, A_log=self.A_log, dt_bias=self.dt_bias, + state_source=pool.recurrent_states[li], indices=fla.cache_indices, + cu_seqlens=fla.cu_seqlens, scale=self.head_k_dim ** -0.5, + ) + else: + mixed = self._conv_prefill( + conv_in, pool, fla.cu_seqlens, fla.cache_indices, fla.has_initial_state) + # fla chunk handles GQA in-kernel: q/k stay at num_k_heads, v at num_v_heads. + qf, kf, vf = torch.split(mixed, [self.key_dim, self.key_dim, self.value_dim], dim=-1) + q = qf.reshape(1, total, self.num_k_heads, self.head_k_dim).to(dtype) + k = kf.reshape(1, total, self.num_k_heads, self.head_k_dim).to(dtype) + v = vf.reshape(1, total, self.num_v_heads, self.head_v_dim).to(dtype) + g, beta = self._gate_params(a, b) + g = g.reshape(1, total, self.num_v_heads) + beta = beta.float().reshape(1, total, self.num_v_heads) + # The chunk kernel reads + writes back initial_state[cache_indices] in place; + # fresh sequences (cached_len==0) must start from a zeroed slot. + if fla.fresh_state_indices is not None: + pool.recurrent_states[li].index_fill_(0, fla.fresh_state_indices, 0.0) + track = fla.track_dst is not None + result = gdn_prefill_chunk_fla( + q, k, v, g, beta, + state_source=pool.recurrent_states[li], indices=fla.cache_indices, + cu_seqlens=fla.cu_seqlens, scale=self.head_k_dim ** -0.5, + return_h=track, + ) + if track: + core_out, h = result + self._write_track_snapshot(pool, li, conv_in, h, fla) + else: + core_out = result + + core_out = core_out.reshape(-1, self.head_v_dim) + z = z.reshape(-1, self.head_v_dim) + out = self.norm.forward(core_out, z).reshape(total, -1) + return self.out_proj.forward(out) + + +__all__ = ["Qwen4ExpGatedDeltaNet"] diff --git a/python/freetoken/models/qwen4_exp/gdn_reference.py b/python/freetoken/models/qwen4_exp/gdn_reference.py new file mode 100644 index 0000000000..fbd9647bcb --- /dev/null +++ b/python/freetoken/models/qwen4_exp/gdn_reference.py @@ -0,0 +1,279 @@ +"""Pure-torch Gated DeltaNet reference (text-only, no cache). + +Correctness oracle for the kernel-backed GDN op (``gdn.Qwen4ExpGatedDeltaNet``). +The two delta rules and the forward are transcribed from +``transformers.models.qwen4_exp.modeling_qwen4_exp`` (``torch_chunk_gated_delta_rule``, +``torch_recurrent_gated_delta_rule`` and ``Qwen4ExpTextGatedDeltaNet.forward`` no-cache path). +Qwen3.8-Flash-Next gates the output norm with ``config.output_gate_type`` (sigmoid) where +Qwen3.5 hardcodes silu; the conv keeps ``config.hidden_act`` (silu). +""" + +from __future__ import annotations + +import torch +import torch.nn as nn +import torch.nn.functional as F + +_GATE_ACTS = {"silu": F.silu, "swish": F.silu, "sigmoid": torch.sigmoid} + + +def _l2norm(x: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: + return x * torch.rsqrt(x.pow(2).sum(dim=-1, keepdim=True) + eps) + + +def recurrent_gated_delta_rule( + query: torch.Tensor, # [B, T, Hv, Dk] + key: torch.Tensor, # [B, T, Hv, Dk] + value: torch.Tensor, # [B, T, Hv, Dv] + g: torch.Tensor, # [B, T, Hv] (log-decay; per-step decay = exp(g)) + beta: torch.Tensor, # [B, T, Hv] + *, + initial_state: torch.Tensor | None = None, + use_qk_l2norm: bool = True, +) -> tuple[torch.Tensor, torch.Tensor]: + """Verbatim port of HF ``torch_recurrent_gated_delta_rule`` (output_final_state=True).""" + initial_dtype = query.dtype + if use_qk_l2norm: + query = _l2norm(query, eps=1e-6) + key = _l2norm(key, eps=1e-6) + query, key, value, beta, g = [ + t.transpose(1, 2).contiguous().to(torch.float32) + for t in (query, key, value, beta, g) + ] + b, h, t_len, dk = key.shape + dv = value.shape[-1] + scale = 1.0 / (dk ** 0.5) + query = query * scale + + out = torch.zeros(b, h, t_len, dv, dtype=value.dtype, device=value.device) + state = ( + torch.zeros(b, h, dk, dv, dtype=value.dtype, device=value.device) + if initial_state is None + else initial_state.to(value) + ) + for i in range(t_len): + q_t = query[:, :, i] + k_t = key[:, :, i] + v_t = value[:, :, i] + g_t = g[:, :, i].exp().unsqueeze(-1).unsqueeze(-1) + beta_t = beta[:, :, i].unsqueeze(-1) + state = state * g_t + kv_mem = (state * k_t.unsqueeze(-1)).sum(dim=-2) + delta = (v_t - kv_mem) * beta_t + state = state + k_t.unsqueeze(-1) * delta.unsqueeze(-2) + out[:, :, i] = (state * q_t.unsqueeze(-1)).sum(dim=-2) + + out = out.transpose(1, 2).contiguous().to(initial_dtype) # [B, T, Hv, Dv] + return out, state + + +def chunk_gated_delta_rule( + query: torch.Tensor, # [B, T, Hv, Dk] + key: torch.Tensor, # [B, T, Hv, Dk] + value: torch.Tensor, # [B, T, Hv, Dv] + g: torch.Tensor, # [B, T, Hv] + beta: torch.Tensor, # [B, T, Hv] + *, + chunk_size: int = 64, + initial_state: torch.Tensor | None = None, + use_qk_l2norm: bool = True, +) -> tuple[torch.Tensor, torch.Tensor]: + """Verbatim port of HF ``torch_chunk_gated_delta_rule`` (output_final_state=True). + + Same recurrence as ``recurrent_gated_delta_rule`` in exact arithmetic; it is the form the + fla chunk kernel implements, so it is the closer oracle for the prefill path.""" + initial_dtype = query.dtype + if use_qk_l2norm: + query = _l2norm(query, eps=1e-6) + key = _l2norm(key, eps=1e-6) + query, key, value, beta, g = [ + x.transpose(1, 2).contiguous().to(torch.float32) + for x in (query, key, value, beta, g) + ] + + batch_size, num_heads, sequence_length, k_head_dim = key.shape + v_head_dim = value.shape[-1] + pad_size = (chunk_size - sequence_length % chunk_size) % chunk_size + query = F.pad(query, (0, 0, 0, pad_size)) + key = F.pad(key, (0, 0, 0, pad_size)) + value = F.pad(value, (0, 0, 0, pad_size)) + beta = F.pad(beta, (0, pad_size)) + g = F.pad(g, (0, pad_size)) + total_sequence_length = sequence_length + pad_size + scale = 1 / (query.shape[-1] ** 0.5) + query = query * scale + + v_beta = value * beta.unsqueeze(-1) + k_beta = key * beta.unsqueeze(-1) + # reshape to chunks + query, key, value, k_beta, v_beta = [ + x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1]) + for x in (query, key, value, k_beta, v_beta) + ] + g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size) + mask = torch.triu( + torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0 + ) + + # chunk decay + g = g.cumsum(dim=-1) + decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril() + attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(mask, 0) + for i in range(1, chunk_size): + row = attn[..., i, :i].clone() + sub = attn[..., :i, :i].clone() + attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2) + attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device) + value = attn @ v_beta + k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1)) + last_recurrent_state = ( + torch.zeros( + batch_size, num_heads, k_head_dim, v_head_dim, + dtype=value.dtype, device=value.device, + ) + if initial_state is None + else initial_state.to(value) + ) + core_attn_out = torch.zeros_like(value) + + # for each chunk; decay_mask is already lower-triangular so no extra causal mask is needed + for i in range(0, total_sequence_length // chunk_size): + q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i] + attn = q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i] + v_prime = (k_cumdecay[:, :, i]) @ last_recurrent_state + v_new = v_i - v_prime + attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_recurrent_state + core_attn_out[:, :, i] = attn_inter + attn @ v_new + last_recurrent_state = ( + last_recurrent_state * g[:, :, i, -1, None, None].exp() + + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2) + @ v_new + ) + + core_attn_out = core_attn_out.reshape( + core_attn_out.shape[0], core_attn_out.shape[1], -1, core_attn_out.shape[-1] + ) + core_attn_out = core_attn_out[:, :, :sequence_length] + core_attn_out = core_attn_out.transpose(1, 2).contiguous().to(initial_dtype) + return core_attn_out, last_recurrent_state + + +class _RMSNormGated(nn.Module): + """RMSNorm of x followed by an ``activation(z)`` gate (norm_before_gate=True). + + Mirrors ``Qwen4ExpTextRMSNormGated`` over head_v_dim groups. + """ + + def __init__(self, dim: int, eps: float, activation: str = "sigmoid"): + super().__init__() + self.weight = nn.Parameter(torch.ones(dim)) + self.eps = eps + self.act = _GATE_ACTS[activation] + + def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor: + in_dtype = x.dtype + x = x.float() + x = x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps) + x = x * self.weight.float() + x = x * self.act(z.float()) + return x.to(in_dtype) + + +class Qwen4ExpGatedDeltaNetReference(nn.Module): + """Pure-torch Gated DeltaNet (text-only, no cache).""" + + def __init__( + self, + hidden_size: int, + num_k_heads: int, + num_v_heads: int, + head_k_dim: int, + head_v_dim: int, + conv_kernel_size: int, + rms_norm_eps: float, + hidden_act: str = "silu", + output_gate: str = "sigmoid", + ): + super().__init__() + if hidden_act != "silu": + raise ValueError(f"GDN reference only supports silu conv activation, got {hidden_act!r}") + if output_gate not in _GATE_ACTS: + raise ValueError(f"unsupported GDN output gate {output_gate!r}") + self.num_k_heads = num_k_heads + self.num_v_heads = num_v_heads + self.head_k_dim = head_k_dim + self.head_v_dim = head_v_dim + self.key_dim = num_k_heads * head_k_dim + self.value_dim = num_v_heads * head_v_dim + self.conv_dim = self.key_dim * 2 + self.value_dim + self.conv_kernel_size = conv_kernel_size + + self.in_proj_qkv = nn.Linear(hidden_size, self.conv_dim, bias=False) + self.in_proj_z = nn.Linear(hidden_size, self.value_dim, bias=False) + self.in_proj_b = nn.Linear(hidden_size, num_v_heads, bias=False) + self.in_proj_a = nn.Linear(hidden_size, num_v_heads, bias=False) + self.conv1d = nn.Conv1d( + self.conv_dim, self.conv_dim, kernel_size=conv_kernel_size, + groups=self.conv_dim, padding=conv_kernel_size - 1, bias=False, + ) + self.dt_bias = nn.Parameter(torch.zeros(num_v_heads)) + self.A_log = nn.Parameter(torch.zeros(num_v_heads)) + self.norm = _RMSNormGated(head_v_dim, eps=rms_norm_eps, activation=output_gate) + self.out_proj = nn.Linear(self.value_dim, hidden_size, bias=False) + + @torch.no_grad() + def load_from_hf(self, hf_gdn) -> None: + """Copy weights from a transformers ``Qwen4ExpTextGatedDeltaNet``.""" + self.in_proj_qkv.weight.copy_(hf_gdn.in_proj_qkv.weight) + self.in_proj_z.weight.copy_(hf_gdn.in_proj_z.weight) + self.in_proj_b.weight.copy_(hf_gdn.in_proj_b.weight) + self.in_proj_a.weight.copy_(hf_gdn.in_proj_a.weight) + # HF stores conv1d weight as [conv_dim, 1, K]; our depthwise Conv1d matches. + self.conv1d.weight.copy_(hf_gdn.conv1d.weight.view_as(self.conv1d.weight)) + if hf_gdn.conv1d.bias is not None and self.conv1d.bias is not None: + self.conv1d.bias.copy_(hf_gdn.conv1d.bias) + self.dt_bias.copy_(hf_gdn.dt_bias) + self.A_log.copy_(hf_gdn.A_log) + self.norm.weight.copy_(hf_gdn.norm.weight) + self.out_proj.weight.copy_(hf_gdn.out_proj.weight) + + def forward(self, hidden_states: torch.Tensor, *, use_chunk_rule: bool = False) -> torch.Tensor: + b, t_len, _ = hidden_states.shape + + mixed_qkv = self.in_proj_qkv(hidden_states).transpose(1, 2) # [B, conv_dim, T] + z = self.in_proj_z(hidden_states).reshape(b, t_len, -1, self.head_v_dim) + a = self.in_proj_a(hidden_states) + bb = self.in_proj_b(hidden_states) + + # causal depthwise conv + silu (drop the right padding back to T) + mixed_qkv = F.silu(self.conv1d(mixed_qkv)[..., :t_len]).transpose(1, 2) # [B, T, conv_dim] + query, key, value = torch.split( + mixed_qkv, [self.key_dim, self.key_dim, self.value_dim], dim=-1 + ) + query = query.reshape(b, t_len, -1, self.head_k_dim) + key = key.reshape(b, t_len, -1, self.head_k_dim) + value = value.reshape(b, t_len, -1, self.head_v_dim) + + beta = bb.sigmoid() + g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias) + + # GQA expand: replicate q/k heads up to num_v_heads + rep = self.num_v_heads // self.num_k_heads + if rep > 1: + query = query.repeat_interleave(rep, dim=2) + key = key.repeat_interleave(rep, dim=2) + + rule = chunk_gated_delta_rule if use_chunk_rule else recurrent_gated_delta_rule + core, _ = rule(query, key, value, g, beta, use_qk_l2norm=True) + + core = core.reshape(-1, self.head_v_dim) + z = z.reshape(-1, self.head_v_dim) + core = self.norm(core, z).reshape(b, t_len, -1) + return self.out_proj(core) + + +__all__ = [ + "Qwen4ExpGatedDeltaNetReference", + "chunk_gated_delta_rule", + "recurrent_gated_delta_rule", +] diff --git a/python/freetoken/models/qwen4_exp/hc.py b/python/freetoken/models/qwen4_exp/hc.py new file mode 100644 index 0000000000..6fed76c060 --- /dev/null +++ b/python/freetoken/models/qwen4_exp/hc.py @@ -0,0 +1,147 @@ +"""Hyper-connection (gated residual) blocks for Qwen3.8-Flash-Next. + +Every layer reads and writes ``hc_count`` residual streams packed as ``R [T, hc_count*hidden]`` +(stream outer, hidden inner -- the checkpoint layout). On CUDA the mix/combine bodies are the +vendored vLLM Triton kernels (``kernel/triton/hc.py``: grouped_gemma_rmsnorm / hc_silu / +hc_gate_mix / hc_combine) around two ``F.linear`` GEMMs; the pure-torch chain stays as the CPU +path and as the reference the kernels are diffed against. Both keep fp32 intermediates and cast +back at the store, so they agree to fp32 rounding. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Tuple + +import torch +import torch.nn.functional as F +from freetoken.kernel.triton.hc import ( + grouped_gemma_rmsnorm, + hc_combine, + hc_gate_mix, + hc_silu, +) +from freetoken.layers import BaseOP, LinearReplicated + +if TYPE_CHECKING: + from freetoken.models.config import ModelConfig + + +def grouped_plus_one_rms_norm( + x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int +) -> torch.Tensor: + """RMSNorm each of ``num_groups`` equal slices of the last dim on its own fp32 statistic, then scale by (1+w).""" + xf = x.float().unflatten(-1, (num_groups, -1)) + xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) + return (xf.flatten(-2) * (1.0 + weight.float())).to(x.dtype) + + +class GroupedPlusOneRMSNorm(BaseOP): + """Per-stream RMSNorm of an ``[..., num_groups*group]`` tensor with one weight element per feature. + + HF ``Qwen4ExpTextRMSNorm(dim, group_size)``. The checkpoint weight is zero-centered and is + loaded RAW: (1+w) is applied at runtime in fp32, never folded into the bf16 weight (the + vendored Triton kernel does the same). ``ple.py`` reuses this class for norm_key / + norm_query / norm_conv, so keep it exported. + """ + + def __init__(self, size: int, eps: float, num_groups: int) -> None: + self.weight = torch.empty(size) + self.eps = eps + self.num_groups = num_groups + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # the kernel is 2D-only, higher-rank callers keep the torch chain + if x.is_cuda and x.dim() == 2: + return grouped_gemma_rmsnorm(x, self.weight, self.eps, self.num_groups) + return grouped_plus_one_rms_norm(x, self.weight, self.eps, self.num_groups) + + +class GatedResidual(BaseOP): + """One hyper-connection block: ``mix`` reads the residual streams, ``combine`` writes a block output back. + + Frozen API (HF ``Qwen4ExpTextGatedResidual``, formulas at modeling_qwen4_exp.py:959-969):: + + x, s = hc.mix(R) # R [T, hc_count*hidden] -> x [T, hidden], s [T, hc_count] or None + y = block(x) # attention / GDN / MoE, plain [T, hidden] -> [T, hidden] + R = hc.combine(R, y, s) + + Rn = groupRMSNorm(R) * (1 + hc_norm.weight) # per hidden-size stream, fp32 stats + lora, s = input_mix_weight_down_block_inject(Rn) # merged GEMM: [lowrank | hc_count | pad] + gate = input_mix_weight_up(silu(lora / hc_count)) + x = mean_i(sigmoid(gate_i) * Rn_i) + R'_i = R_i + 2*sigmoid(s_i / hc_count) * y + + ``s`` is the RAW inject logit slice of the merged GEMM (pre 2*sigmoid), which is what the + vendored ``hc_combine`` kernel expects; ``combine`` applies the activation. The merged weight + is ``[lowrank + hc_count + pad, hc_count*hidden]`` (Qwen3.8: 320 + 4 + 12 = 336 rows), the pad + rows are zero and their GEMM output is dropped. ``use_combine=False`` is the top-level mixer: + it owns the unmerged ``input_mix_weight_down``, returns ``s = None`` and has no ``combine``. + + Weight keys (checkpoint names, prefix stripped): ``hc_norm.weight``, + ``input_mix_weight_down_block_inject.weight`` (loader: concat of + ``input_mix_weight_down`` [lowrank, hc*hidden], ``block_inject_weight`` [hc_count, hc*hidden] + and ``pad`` zero rows), ``input_mix_weight_up.weight``. + + Launch budget on CUDA: ``mix`` is 3 kernels around 2 GEMMs, ``combine`` is 1. + """ + + def __init__(self, config: ModelConfig, use_combine: bool = True) -> None: + args = config.qwen4_args + self.hc_count = args.hc_count + self.hidden_size = args.hidden_size + self.lowrank = args.hc_lowrank + self.use_combine = use_combine + width = args.ple_state_width + self.hc_norm = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count) + if use_combine: + # 16-row alignment for the merged skinny GEMM (vLLM hyperconnection.py:98) + self.pad_size = (-(self.lowrank + self.hc_count)) % 16 + self.input_mix_weight_down_block_inject = LinearReplicated( + width, self.lowrank + self.hc_count + self.pad_size, has_bias=False + ) + else: + self.pad_size = 0 + self.input_mix_weight_down = LinearReplicated(width, self.lowrank, has_bias=False) + self.input_mix_weight_up = LinearReplicated(self.lowrank, width, has_bias=False) + + def _down(self, rn: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]: + """Run the down GEMM and split off the raw inject logits; the pad columns are dropped.""" + if not self.use_combine: + return self.input_mix_weight_down.forward(rn), None + down = self.input_mix_weight_down_block_inject.forward(rn) + # both slices keep unit inner stride, so the kernels read them without a copy + return down[:, : self.lowrank], down[:, self.lowrank : self.lowrank + self.hc_count] + + def _mix_kernel(self, R: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]: + rn = grouped_gemma_rmsnorm(R, self.hc_norm.weight, self.hc_norm.eps, self.hc_count) + lora, s = self._down(rn) + gate = self.input_mix_weight_up.forward(hc_silu(lora, self.hc_count)) + return hc_gate_mix(rn, gate, self.hc_count), s + + def _mix_torch(self, R: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]: + rn = grouped_plus_one_rms_norm(R, self.hc_norm.weight, self.hc_norm.eps, self.hc_count) + lora, s = self._down(rn) + lora = F.silu(lora.float() / self.hc_count) + gate = self.input_mix_weight_up.forward(lora.to(R.dtype)) + mixed = torch.sigmoid(gate.float()).unflatten(-1, (self.hc_count, self.hidden_size)) + mixed = mixed * rn.float().unflatten(-1, (self.hc_count, self.hidden_size)) + return mixed.mean(-2).to(R.dtype), s + + def _combine_torch(self, R: torch.Tensor, y: torch.Tensor, s: torch.Tensor) -> torch.Tensor: + inject = 2.0 * torch.sigmoid(s.float() / self.hc_count) + out = R.float().unflatten(-1, (self.hc_count, self.hidden_size)) + out = out + y.float().unsqueeze(-2) * inject.unsqueeze(-1) + return out.flatten(-2).to(R.dtype) + + def mix(self, R: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]: + """Return the block input ``x [T, hidden]`` and the inject logits ``s [T, hc_count]`` (None if no combine).""" + return self._mix_kernel(R) if R.is_cuda else self._mix_torch(R) + + def combine(self, R: torch.Tensor, y: torch.Tensor, s: torch.Tensor) -> torch.Tensor: + """Inject the block output ``y [T, hidden]`` back into every stream of ``R``.""" + if R.is_cuda: + return hc_combine(R, y, s, self.hc_count) + return self._combine_torch(R, y, s) + + +__all__ = ["GatedResidual", "GroupedPlusOneRMSNorm", "grouped_plus_one_rms_norm"] diff --git a/python/freetoken/models/qwen4_exp/model.py b/python/freetoken/models/qwen4_exp/model.py new file mode 100644 index 0000000000..19c31e2fc5 --- /dev/null +++ b/python/freetoken/models/qwen4_exp/model.py @@ -0,0 +1,217 @@ +"""Qwen3.8-Flash-Next decoder stack (text-only). + +The residual state is ``R [T, hc_count*hidden]`` end to end: the embedding is repeated over the +``hc_count`` streams, every layer mixes them down to one ``[T, hidden]`` block input and injects +its output back, and the top-level mixer collapses them once before ``lm_head``. There is no +input/post layernorm and no final ``model.norm`` -- the hyper-connection norms are the only ones. + +Layer contract (frozen): ``forward(R [T, hc*hidden], batch) -> R' [T, hc*hidden]`` with an +immediate combine:: + + R = R + ple(R, batch) # zero-based layer 1 only + x, s = attn_hc.mix(R); y = (GDN | QSA)(x); R = attn_hc.combine(R, y, s) + x, s = mlp_hc.mix(R); y = MoE(x); R = mlp_hc.combine(R, y, s) +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, List + +import torch +from freetoken.core import get_global_ctx +from freetoken.layers import BaseOP, OPList, ParallelLMHead, VocabParallelEmbedding +from freetoken.models.blocks import BaseLLMModel +from freetoken.utils import nvtx_annotate + +from .attention import Qwen4ExpAttention +from .hc import GatedResidual +from .moe import Qwen4ExpMoE +from .ple import PLELayer + +if TYPE_CHECKING: + from freetoken.core import Batch + from freetoken.models.config import ModelConfig + + +def build_linear_mixer(config: ModelConfig, layer_id: int) -> BaseOP: + """GDN mixer of a linear_attention layer (Qwen3.5's GDN with a configurable output gate).""" + from .gdn import Qwen4ExpGatedDeltaNet + + g = config.linear_attention_group() + return Qwen4ExpGatedDeltaNet( + hidden_size=config.hidden_size, + num_k_heads=g.num_key_heads, + num_v_heads=g.num_value_heads, + head_k_dim=g.key_head_dim, + head_v_dim=g.value_head_dim, + conv_kernel_size=g.conv_kernel_dim, + rms_norm_eps=config.rms_norm_eps, + layer_id=layer_id, + output_gate=g.output_gate, + # Qwen3.8's block-fp8 checkpoint keeps the GDN projections bf16 (only the routed + # experts are quantized), so do not let expert_quant flip them to Fp8Block. + expert_quant="none" if config.expert_quant == "fp8_block" else config.expert_quant, + attn_quant=config.attn_quant, + ) + + +class Qwen4ExpDecoderLayer(BaseOP): + """One decoder layer over the hyper-connection streams (see the module docstring for the flow).""" + + def __init__(self, config: ModelConfig, layer_id: int) -> None: + self._layer_id = layer_id + self._is_linear = config.is_linear_layer(layer_id) + if self._is_linear: + self.linear_attn = build_linear_mixer(config, layer_id) + else: + self.self_attn = Qwen4ExpAttention(config, layer_id) + self.mlp = Qwen4ExpMoE(config, layer_id) + self.attn_hyper_connection = GatedResidual(config) + self.mlp_hyper_connection = GatedResidual(config) + self.ple = ( + PLELayer(config, layer_id) if layer_id in config.qwen4_args.ple_layer_ids else None + ) + + @nvtx_annotate("Layer_{}", layer_id_field="_layer_id") + def forward(self, hidden: torch.Tensor, batch: Batch) -> torch.Tensor: + if self.ple is not None: + hidden = hidden + self.ple.forward(hidden, batch) + block_input, inject = self.attn_hyper_connection.mix(hidden) + if self._is_linear: + block_output = self.linear_attn.forward(block_input) + else: + block_output = self.self_attn.forward(block_input, batch) + hidden = self.attn_hyper_connection.combine(hidden, block_output, inject) + block_input, inject = self.mlp_hyper_connection.mix(hidden) + return self.mlp_hyper_connection.combine(hidden, self.mlp.forward(block_input), inject) + + +class Qwen4ExpModel(BaseOP): + def __init__(self, config: ModelConfig) -> None: + self.hc_count = config.qwen4_args.hc_count + self.embed_tokens = VocabParallelEmbedding( + num_embeddings=config.vocab_size, + embedding_dim=config.hidden_size, + ) + self.layers = OPList( + [Qwen4ExpDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)] + ) + self.hyper_connection_mixer = GatedResidual(config, use_combine=False) + # plain tuple (not an OP child), so it never shows up in the state dict + self._ple = tuple(layer.ple for layer in self.layers.op_list if layer.ple is not None) + + @property + def ple_layers(self) -> List[PLELayer]: + """The PLE layers in decoder order -- the seam the loader attaches table backends to.""" + return list(self._ple) + + def forward(self, input_ids: torch.Tensor, batch: Batch) -> torch.Tensor: + hidden = self.embed_tokens.forward(input_ids).repeat(1, self.hc_count) + meta = None + if self._ple: + from .ple import build_ple_metadata, commit_ngram_context + + meta = build_ple_metadata(batch, self._ple[0].args, input_ids.device) + for ple in self._ple: # gather the pinned-host PLE rows while the early layers run + ple.start_prefetch(batch, meta) + for layer in self.layers.op_list: + hidden = layer.forward(hidden, batch) + if meta is not None: + # single writer: the layers only read the context, so a second PLE layer's + # prefetch sees the un-rolled window + commit_ngram_context(meta, getattr(batch, "fla_metadata", None)) + return self.hyper_connection_mixer.mix(hidden)[0] + + +class Qwen4ExpForCausalLM(BaseLLMModel): + def __init__(self, config: ModelConfig) -> None: + self._config = config + self.model = Qwen4ExpModel(config) + if getattr(config, "lm_head_quant", "none") == "nvfp4": + from freetoken.kernel.triton.nvfp4_linear import Nvfp4LMHead + + assert not config.tie_word_embeddings, "NVFP4 lm_head assumes untied embeddings" + self.lm_head = Nvfp4LMHead( + num_embeddings=config.vocab_size, embedding_dim=config.hidden_size + ) + else: + self.lm_head = ParallelLMHead( + num_embeddings=config.vocab_size, + embedding_dim=config.hidden_size, + tie_word_embeddings=config.tie_word_embeddings, + tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None, + ) + super().__init__() + + def load_host_tables(self, engine_config) -> int: + """Attach the PLE n-gram table (pinned checkpoint bank, or zeros for dummy weights); returns the pinned host bytes the engine reserves from its pin budget.""" + ple_layers = self.model.ple_layers + if not ple_layers: + return 0 + from .ple import PinnedUVATable, ZeroTable, derive_ngram_hash_constants + + if getattr(engine_config, "use_dummy_weight", False): + # Dummy fill leaves the int64 hash buffers garbage (a zero vocab size divides by + # zero in the hash), so re-derive the real constants and read a zero table. + for ple in ple_layers: + args = ple.args + mult, sizes, offsets = derive_ngram_hash_constants( + vocab_size=self._config.vocab_size, + ngram_size=args.ngram_size, + num_ngram_heads=args.num_ngram_heads, + ngram_vocab_size_base=args.ngram_vocab_size_base, + ple_layer_index=ple.ple_index, + ) + emb = ple.ple_embedding + emb.layer_multipliers.copy_(torch.tensor(mult, dtype=torch.int64)) + emb.ngram_heads_vocab_sizes.copy_(torch.tensor(sizes, dtype=torch.int64)) + emb.ngram_heads_offsets.copy_(torch.tensor(offsets, dtype=torch.int64)) + emb.attach_table(ZeroTable(offsets[-1] + sizes[-1], args.ngram_head_dim)) + return 0 + + if engine_config.ple_backend == "disk": + from freetoken.utils import download_hf_weight + + from .ple_disk import DiskRowTable, resolve_row_source + + folder = download_hf_weight(engine_config.model_path) + # one WAIT node per captured graph: the flag protocol supports a single consume + assert len(ple_layers) == 1, "disk PLE backend expects exactly one PLE layer" + emb, args = ple_layers[0].ple_embedding, ple_layers[0].args + # hash with the state-dict-loaded constants, the same source the pinned path reads + constants = { + "num_ngram_heads": args.num_ngram_heads, + "layer_multipliers": emb.layer_multipliers.tolist(), + "per_head_vocab_sizes": emb.ngram_heads_vocab_sizes.tolist(), + "per_head_offsets": emb.ngram_heads_offsets.tolist(), + "eos_token_id": args.ngram_boundary_token_id, + } + disk_table = DiskRowTable( + resolve_row_source(folder), + constants, + max_graph_rows=max(256, engine_config.cuda_graph_max_bs or 0), + max_extend_tokens=engine_config.max_extend_tokens, + ) + self._ple_table = disk_table + for ple in ple_layers: + ple.ple_embedding.attach_table(disk_table) + # engine enters this around every dispatch; the graph itself never waits on the disk + self.forward_host_ctx = disk_table.forward_host_ctx + return 0 + + from .weight import load_ple_table + + table = load_ple_table(engine_config.model_path, self._config.qwen4_args) + self._ple_table = table # owns the pinned HostBank; keep it alive + for ple in ple_layers: + ple.ple_embedding.attach_table( + PinnedUVATable(table.bank.tensor, float(table.weight_scale)) + ) + return table.bank.nbytes + + def forward(self) -> torch.Tensor: + batch = get_global_ctx().batch + return self.lm_head.forward(self.model.forward(batch.input_ids, batch)) + + +__all__ = ["Qwen4ExpDecoderLayer", "Qwen4ExpForCausalLM", "Qwen4ExpModel", "build_linear_mixer"] diff --git a/python/freetoken/models/qwen4_exp/moe.py b/python/freetoken/models/qwen4_exp/moe.py new file mode 100644 index 0000000000..9ef65c0a84 --- /dev/null +++ b/python/freetoken/models/qwen4_exp/moe.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +from dataclasses import replace +from typing import TYPE_CHECKING + +import torch +from freetoken.kernel.triton.moe_shared_gate import shared_gate_mul_add, shared_gate_sigmoid +from freetoken.layers.moe import make_moe_layer +from freetoken.models.qwen3_5_moe.moe import Qwen3_5MoE + +if TYPE_CHECKING: + from freetoken.models.config import ModelConfig + + +class Qwen4ExpMoE(Qwen3_5MoE): + """Qwen3_5MoE with the shared-expert gate on triton instead of gemv + sigmoid + mul + add. + + Same weights, same state dict. The gate reduction stays ahead of the routed experts, which may write into ``hidden_states`` in place. + """ + + def __init__(self, config: ModelConfig, layer_id: int | None = None) -> None: + if getattr(config, "expert_quant", "none") != "fp8_block": + super().__init__(config, layer_id=layer_id) + return + # Qwen3.8's block-fp8 checkpoint quantizes only the routed experts; the shared + # expert stays bf16, so hide expert_quant from _SharedExpert's fp8 branch and + # rebuild the routed experts with the fp8_block bank layout. + super().__init__(replace(config, expert_quant="none"), layer_id=layer_id) + self.experts = make_moe_layer( + config, + layer_id=layer_id, + renormalize=config.norm_topk_prob, + weight_format="fp8_block", + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_tokens, hidden_dim = hidden_states.shape + hidden_states = hidden_states.view(-1, hidden_dim) + router_logits = self.gate.forward(hidden_states) + shared = self.shared_expert.forward(hidden_states) + gate = shared_gate_sigmoid(hidden_states, self.shared_expert_gate.weight.view(-1)) + routed = self.experts.forward(hidden_states=hidden_states, router_logits=router_logits) + return shared_gate_mul_add(routed, shared, gate).view(num_tokens, hidden_dim) + + +__all__ = ["Qwen4ExpMoE"] diff --git a/python/freetoken/models/qwen4_exp/ple.py b/python/freetoken/models/qwen4_exp/ple.py new file mode 100644 index 0000000000..5100229a91 --- /dev/null +++ b/python/freetoken/models/qwen4_exp/ple.py @@ -0,0 +1,721 @@ +"""Per-Layer Embedding (PLE) for Qwen3.8-Flash-Next: hashed n-gram features injected at layer 1. + +HF reference: ``Qwen4ExpTextNGramEmbedding`` (modeling_qwen4_exp.py:1018) and +``Qwen4ExpTextPLELayer`` (:1117). Per token:: + + E = table[hash(ngram)] # 16 heads (8 x 2-gram, 8 x 3-gram) x 160 -> 2560 + K = norm_key(key_proj(E)).view(hc, hidden) # V = value_proj(E) [hidden] + Q = norm_query(R).view(hc, hidden) + u = / sqrt(hidden) # per stream + U = sigmoid(sign(u) * sqrt(max(|u|, 1e-6))) * V + D = U + silu(conv1d(norm_conv(U))) # depthwise, kernel 4, dilation ngram_size + R += D # before the attention hyper-connection mix + +The table is the 47.7 GiB FP8 n-gram store: ``PinnedUVATable`` keeps it in pinned host memory and +gathers rows over UVA, optionally started early on a side stream (``PLELayer.start_prefetch``). +``GpuResidentTable`` is the small-table oracle the pinned backend is diffed against. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Protocol, Sequence, Tuple + +import torch +import torch.nn.functional as F +from freetoken.core import get_global_ctx +from freetoken.layers import BaseOP, LinearReplicated + +from .config import PLE_CONV_STATE, PLE_NGRAM_STATE +from .hc import GroupedPlusOneRMSNorm + +if TYPE_CHECKING: + from freetoken.core import Batch + from freetoken.models.config import ModelConfig + + from .config import Qwen4ExpArgs + + +_MASK64 = (1 << 64) - 1 +_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15 +_SPLITMIX_M1 = 0xBF58476D1CE4E5B9 +_SPLITMIX_M2 = 0x94D049BB133111EB +_PLE_LAYER_PRIME = 10007 + + +class PLETableBackend(Protocol): + """Row store for one PLE layer's n-gram embedding table (Qwen3.8: 40M rows x 160, FP8 + one scalar scale). + + Frozen contract. ``GpuResidentTable`` (oracle, small tables) and ``PinnedUVATable`` (the real 47.7 GiB pinned-host table) implement it. Rows are addressed by the + GLOBAL hashed id, i.e. the per-head vocab offset is already added by ``NGramEmbedding``. + + ``lookup`` gets ``row_ids [T, num_ngram_heads]`` (int64, device) and returns + ``[T, num_ngram_heads * head_dim]`` in ``dtype``, already dequantized (fp8 -> dtype, times the + scalar weight_scale). ``out``, when given, is the destination and is returned as-is (CUDA-graph + decode reuses a fixed buffer). + + ``prefetch`` may start the gather early on a side stream (the model issues it before layer 0 and + joins it in ``lookup``); a backend with no async path makes it a no-op. + """ + + num_rows: int + head_dim: int + dtype: torch.dtype + + def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor: ... + + def prefetch(self, row_ids: torch.Tensor) -> None: ... + + +class GpuResidentTable: + """PLE table held whole in GPU memory; ``index_select`` oracle for the pinned-host backend.""" + + def __init__( + self, weight: torch.Tensor, scale: float = 1.0, dtype: torch.dtype | None = None + ) -> None: + self.weight = weight + self.scale = float(scale) + self.num_rows, self.head_dim = weight.shape + self.dtype = dtype if dtype is not None else ( + torch.bfloat16 if weight.dtype.itemsize < 2 else weight.dtype + ) + + def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor: + rows = self.weight.index_select(0, row_ids.reshape(-1)).to(self.dtype) + if self.scale != 1.0: + rows = rows * self.scale + rows = rows.view(*row_ids.shape[:-1], -1) + if out is None: + return rows + out.copy_(rows) + return out + + def prefetch(self, row_ids: torch.Tensor) -> None: + return None + + +class ZeroTable: + """Dummy-weight stand-in: every lookup reads zeros (dummy checkpoints ship no table).""" + + def __init__(self, num_rows: int, head_dim: int, dtype: torch.dtype = torch.bfloat16) -> None: + self.num_rows = int(num_rows) + self.head_dim = head_dim + self.dtype = dtype + + def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor: + if out is not None: + return out.zero_() + return torch.zeros( + (*row_ids.shape[:-1], row_ids.shape[-1] * self.head_dim), + dtype=self.dtype, + device=row_ids.device, + ) + + def prefetch(self, row_ids: torch.Tensor) -> None: + return None + + +class PinnedUVATable: + """PLE table left in pinned host memory; rows are gathered over UVA by a Triton kernel. + + ``weight`` must be the filled and ``pin()``ed ``HostBank.tensor`` from + ``weight.load_ple_table`` (``[num_rows, head_dim]``, fp8-e4m3 or bf16); an unregistered host + buffer is not device-addressable and the kernel faults on it. ``scale`` is the checkpoint's + scalar ``weight_scale``. Gathers emit bf16 into a staging buffer, one per captured decode size + and one growable buffer for everything else. + + ``prefetch`` runs the gather on a private stream and the next ``lookup`` joins it. ``lookup`` + returns a view of that staging buffer, so the rows must be consumed before the next lookup. + """ + + def __init__( + self, + weight: torch.Tensor, + scale: float = 1.0, + *, + device: torch.device | None = None, + prefetch: bool = True, + ) -> None: + assert weight.device.type == "cpu" and weight.is_contiguous() + assert weight.dtype in (torch.float8_e4m3fn, torch.bfloat16), weight.dtype + from freetoken.kernel.pinned import device_ptr + + self.weight = weight + self.scale = float(scale) + self.num_rows, self.head_dim = weight.shape + self.dtype = torch.bfloat16 + self._is_fp8 = weight.dtype == torch.float8_e4m3fn + self._device = device or torch.device("cuda", torch.cuda.current_device()) + # WDDM maps registered host memory at a different device address; on Linux/UVA this is data_ptr + self._table_ptr = device_ptr(weight) + self._stream = torch.cuda.Stream(device=self._device) if prefetch else None + self._staging: torch.Tensor | None = None + self._graph_staging: dict[int, torch.Tensor] = {} + self._pending: Tuple[torch.Tensor, torch.Tensor] | None = None + + def _stage(self, rows: int) -> torch.Tensor: + # Captured graphs keep one buffer per size for good: growing the eager one would free the + # block a replay still writes to. + if torch.cuda.is_current_stream_capturing(): + buf = self._graph_staging.get(rows) + if buf is None: + buf = torch.empty((rows, self.head_dim), dtype=self.dtype, device=self._device) + self._graph_staging[rows] = buf + return buf + buf = self._staging + if buf is None or buf.shape[0] < rows: + buf = torch.empty((rows, self.head_dim), dtype=self.dtype, device=self._device) + self._staging = buf + return buf[:rows] + + def _gather(self, row_ids: torch.Tensor, dst: torch.Tensor) -> torch.Tensor: + from freetoken.kernel.triton.ple import ple_gather_rows + + return ple_gather_rows( + self._table_ptr, + self.num_rows, + self.head_dim, + row_ids.reshape(-1), + dst, + self.scale, + self._is_fp8, + ) + + def prefetch(self, row_ids: torch.Tensor) -> None: + if self._stream is None or row_ids.numel() == 0: + return + dst = self._stage(row_ids.numel()) + self._stream.wait_stream(torch.cuda.current_stream(self._device)) + if not torch.cuda.is_current_stream_capturing(): + row_ids.record_stream(self._stream) + with torch.cuda.stream(self._stream): + self._gather(row_ids, dst) + self._pending = (row_ids, dst) + + def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor: + pending, self._pending = self._pending, None + if pending is not None: + # join even on a miss: the stale prefetch owns the staging buffer about to be reused + torch.cuda.current_stream(self._device).wait_stream(self._stream) + if pending is not None and pending[0] is row_ids: + rows = pending[1] + else: + rows = self._gather(row_ids, self._stage(row_ids.numel())) + rows = rows.view(*row_ids.shape[:-1], -1) + if out is None: + return rows + out.copy_(rows) + return out + + +def _splitmix64(value: int) -> int: + value = (value + _SPLITMIX_GAMMA) & _MASK64 + value = ((value ^ (value >> 30)) * _SPLITMIX_M1) & _MASK64 + value = ((value ^ (value >> 27)) * _SPLITMIX_M2) & _MASK64 + return (value ^ (value >> 31)) & _MASK64 + + +def _is_prime(value: int) -> bool: + if value < 2: + return False + if value % 2 == 0: + return value == 2 + for divisor in range(3, math.isqrt(value) + 1, 2): + if value % divisor == 0: + return False + return True + + +def _nth_prime_after(start: int, count: int) -> int: + prime = start + for _ in range(count): + prime += 1 + while not _is_prime(prime): + prime += 1 + return prime + + +def derive_ngram_hash_constants( + *, + vocab_size: int, + ngram_size: int, + num_ngram_heads: int, + ngram_vocab_size_base: int, + ple_layer_index: int, + seed: int = 1234, +) -> Tuple[List[int], List[int], List[int]]: + """Recompute (multipliers, per-head vocab sizes, per-head offsets) the way HF derives them at init. + + The checkpoint ships these as int64 tensors, so serving loads them; this is the dummy-weight + path and the oracle a loader test can check the checkpoint values against. + """ + half_bound = max(1, ((1 << 63) - 1) // max(vocab_size, 1) // 2) + base_seed = seed + _PLE_LAYER_PRIME * ple_layer_index + multipliers = [ + 2 * (_splitmix64((base_seed + _SPLITMIX_GAMMA * (i + 1)) & _MASK64) % half_bound) + 1 + for i in range(ngram_size) + ] + sizes: List[int] = [] + offsets: List[int] = [] + total = 0 + for head in range(num_ngram_heads): + global_head = ple_layer_index * num_ngram_heads + head + size = _nth_prime_after(ngram_vocab_size_base - 1, global_head + 1) + sizes.append(size) + offsets.append(total) + total += size + return multipliers, sizes, offsets + + +@dataclass +class PLEMetadata: + """Per-forward PLE inputs, built once and shared by every PLE layer (sibling of ``FLAMetadata``). + + Frozen contract: + input_ids [T] int device -- this forward's tokens, ragged, concatenated in request order + cu_seqlens [B+1] int device -- query indptr; decode is ``arange(B+1)`` + seq_lens host per-request token counts; avoids a device sync in the ragged conv loop + ngram_context [B, ngram_size-1] int64 device -- the tokens immediately BEFORE each request's + first token of this forward, read from the ``ple_ngram_ctx`` slot state and + forced to the boundary (eos) id for fresh rows. The hash never crosses eos, + so a fresh sequence passes all-eos. + state_slots [B] int64 device -- linear-state slot per request (``Req.linear_slot_idx`` or + ``Req.table_idx``); keys every PLE slot state + fresh_slots [B] bool device or None -- request starts a new sequence, so read a zero state + is_decode one token per request (the batched 4-tap path) + """ + + input_ids: torch.Tensor + cu_seqlens: torch.Tensor + seq_lens: Sequence[int] + ngram_context: torch.Tensor + state_slots: torch.Tensor + fresh_slots: torch.Tensor | None + is_decode: bool + + +def _state_slot(req) -> int: + slot = getattr(req, "linear_slot_idx", None) + return req.table_idx if slot is None else slot + + +def _ngram_context_pool() -> torch.Tensor: + pool = get_global_ctx().linear_state_pool + assert pool is not None and pool.has_slot_state(PLE_NGRAM_STATE), ( + "PLE needs the ple_ngram_ctx slot state (or an explicit context_pool=)" + ) + return pool.slot_state(PLE_NGRAM_STATE) + + +def build_ple_metadata( + batch: Batch, + args: Qwen4ExpArgs, + device: torch.device, + context_pool: torch.Tensor | None = None, +) -> PLEMetadata: + """Build ``PLEMetadata`` from a scheduler batch. + + The n-gram context is per-request device state (``ple_ngram_ctx`` [num_slots, ngram_size-1], + rolled forward once per forward by ``commit_ngram_context``), so it never lags the sampled + token under overlap scheduling and follows the slot on COW/snapshot. A decode batch reads it + straight off the persistent ``linear_table_idx`` buffer, so the build is capture-safe and + sync-free. Reuses ``batch.fla_metadata`` (slots / indptr / fresh mask) when the scheduler + built it. + """ + reqs = batch.padded_reqs + ctx_len = args.ngram_size - 1 + eos = args.ngram_boundary_token_id + if context_pool is None: + context_pool = _ngram_context_pool() + assert context_pool.shape[-1] == ctx_len, ( + f"ple_ngram_ctx holds {context_pool.shape[-1]} ids, config wants {ctx_len}" + ) + fla = getattr(batch, "fla_metadata", None) + slots_dev = getattr(batch, "linear_table_idx", None) + + if batch.is_decode and slots_dev is not None: + slots = slots_dev.long() + bs = slots.numel() + return PLEMetadata( + input_ids=batch.input_ids, + cu_seqlens=torch.arange(bs + 1, dtype=torch.int32, device=device), + seq_lens=(1,) * bs, + ngram_context=context_pool.index_select(0, slots).long(), + state_slots=slots, + fresh_slots=None, + is_decode=True, + ) + + lens = [r.extend_len for r in reqs] + if fla is not None and fla.has_initial_state is not None: + cu = fla.cu_seqlens + slots = fla.cache_indices.long() + fresh = ~fla.has_initial_state + else: # direct-op callers (tests) with no scheduler metadata + pin = {"device": "cpu", "pin_memory": torch.cuda.is_available()} + cu = torch.tensor([0, *lens], dtype=torch.int64, **pin).cumsum_(0).to(device, non_blocking=True) + slots = torch.tensor([_state_slot(r) for r in reqs], dtype=torch.int64, **pin).to(device, non_blocking=True) + fresh = torch.tensor([r.cached_len == 0 for r in reqs], dtype=torch.bool, **pin).to(device, non_blocking=True) + context = context_pool.index_select(0, slots).long() + context = torch.where(fresh.unsqueeze(1), context.new_full((), eos), context) + return PLEMetadata( + input_ids=batch.input_ids, + cu_seqlens=cu, + seq_lens=tuple(lens), + ngram_context=context, + state_slots=slots, + fresh_slots=fresh, + is_decode=batch.is_decode, + ) + + +def commit_ngram_context(meta: PLEMetadata, fla, context_pool: torch.Tensor | None = None) -> None: + """Roll each request's ``ple_ngram_ctx`` forward past this forward's tokens. + + Called ONCE per forward after every PLE layer ran (the layers only read the context); + also writes the boundary-aligned window to the track slot so a donated snapshot restores + the context together with the conv state. Pure device arithmetic, capture-safe. + """ + if context_pool is None: + context_pool = _ngram_context_pool() + ids = meta.input_ids.long() + ctx_len = meta.ngram_context.shape[1] + steps = torch.arange(ctx_len, device=ids.device) + if meta.is_decode: + nxt = torch.cat([meta.ngram_context[:, 1:], ids.view(-1, 1)], dim=1) + else: + cu = meta.cu_seqlens.long() + cand = cu[1:].unsqueeze(1) - ctx_len + steps + # short extends fall back to the old context: token j of the new window sits at + # old-context column extend_len + j when it predates this forward + old = meta.ngram_context.gather( + 1, ((cu[1:] - cu[:-1]).unsqueeze(1) + steps).clamp_(max=ctx_len - 1) + ) + nxt = torch.where(cand >= cu[:-1].unsqueeze(1), ids[cand.clamp_min(0)], old) + context_pool.index_copy_(0, meta.state_slots, nxt.to(context_pool.dtype)) + if fla is not None and fla.track_boundary_row is not None: + win = ids[fla.track_boundary_row.unsqueeze(1) - ctx_len + steps] + context_pool.index_copy_(0, fla.track_dst, win.to(context_pool.dtype)) + + +class NGramEmbedding(BaseOP): + """Hashed n-gram lookup: splitmix64 mix of the last n token ids -> per-head prime vocab -> table rows. + + Weight keys (checkpoint names): ``layer_multipliers`` [ngram_size], ``ngram_heads_vocab_sizes`` + and ``ngram_heads_offsets`` [num_ngram_heads], all int64. The table itself is NOT a state-dict + entry (128 checkpoint shards land in a ``PLETableBackend``); attach it with ``attach_table``. + """ + + def __init__(self, args: Qwen4ExpArgs, table: PLETableBackend | None = None) -> None: + self.ngram_size = args.ngram_size + self.heads_per_ngram = args.heads_per_ngram + self.num_heads = args.num_ngram_heads + self.eos_token_id = args.ngram_boundary_token_id + self.layer_multipliers = torch.empty(args.ngram_size, dtype=torch.int64) + self.ngram_heads_vocab_sizes = torch.empty(self.num_heads, dtype=torch.int64) + self.ngram_heads_offsets = torch.empty(self.num_heads, dtype=torch.int64) + self._table = table + + def attach_table(self, table: PLETableBackend) -> None: + self._table = table + + @property + def table(self) -> PLETableBackend: + assert self._table is not None, "PLE table backend was never attached" + return self._table + + def _window(self, meta: PLEMetadata): + """The hash window as ``(packed [B, W], select)``, where ``select`` picks this forward's tokens.""" + ids = meta.input_ids.long() + ctx_len = self.ngram_size - 1 + if meta.is_decode: + # a window of exactly ngram_size columns holds every shift the hash can reach + return torch.cat([meta.ngram_context, ids.view(-1, 1)], dim=1), lambda t: t[:, -1] + + num_reqs = len(meta.seq_lens) + width = ctx_len + max(meta.seq_lens) + cu = meta.cu_seqlens.long() + # Pack the ragged tokens into [B, ctx+max_len] so the shift/boundary logic is one gather. + flat_pos = torch.arange(ids.numel(), device=ids.device) + req = (torch.searchsorted(cu, flat_pos, right=True) - 1).clamp_(max=num_reqs - 1) + col = flat_pos - cu[req] + ctx_len + packed = ids.new_full((num_reqs, width), self.eos_token_id) + packed[:, :ctx_len] = meta.ngram_context + packed[req, col] = ids + return packed, lambda t: t[req, col] + + def _shift_ignore_eos(self, packed: torch.Tensor) -> List[torch.Tensor]: + """``out[s][b, p]`` = the token ``s`` places left of ``p``, or eos when the window crosses a boundary.""" + num_reqs, width = packed.shape + pos = torch.arange(width, device=packed.device) + eos_pos = torch.where(packed == self.eos_token_id, pos, -1) + prev_eos = torch.cummax(eos_pos, dim=1).values + prev_eos = torch.cat([eos_pos.new_full((num_reqs, 1), -1), prev_eos[:, :-1]], dim=1) + in_segment = pos.unsqueeze(0) - prev_eos - 1 + + shifted = [packed] + for shift in range(1, self.ngram_size): + src = pos - shift + gathered = packed.gather(1, src.clamp_min(0).unsqueeze(0).expand(num_reqs, -1)) + valid = (src.unsqueeze(0) >= 0) & (in_segment >= shift) + shifted.append(torch.where(valid, gathered, packed.new_full((), self.eos_token_id))) + return shifted + + def row_ids(self, meta: PLEMetadata) -> torch.Tensor: + """Global table row per (token, hash head): ``[T, num_ngram_heads]`` int64.""" + packed, select = self._window(meta) + tokens = [select(s) for s in self._shift_ignore_eos(packed)] + blocks = [] + for ngram in range(2, self.ngram_size + 1): + start = (ngram - 2) * self.heads_per_ngram + end = start + self.heads_per_ngram + mixed = tokens[0] * self.layer_multipliers[0] + for position in range(1, ngram): + mixed = torch.bitwise_xor(mixed, tokens[position] * self.layer_multipliers[position]) + head_ids = torch.remainder(mixed.unsqueeze(-1), self.ngram_heads_vocab_sizes[start:end]) + blocks.append(head_ids + self.ngram_heads_offsets[start:end]) + return torch.cat(blocks, dim=-1) + + def forward(self, meta: PLEMetadata, out: torch.Tensor | None = None) -> torch.Tensor: + return self.table.lookup(self.row_ids(meta), out) + + +class _DepthwiseConv1d(BaseOP): + """Holds the depthwise conv weight ``[width, 1, kernel]`` (key ``conv1d.weight``).""" + + def __init__(self, width: int, kernel: int) -> None: + self.weight = torch.empty(width, 1, kernel) + + +def short_conv_reference( + x: torch.Tensor, + meta: PLEMetadata, + states: torch.Tensor, + weight: torch.Tensor, + dilation: int, +) -> torch.Tensor: + """Per-request ``F.conv1d`` over ``[state | chunk]``, advancing ``states`` in place. + + Transcription of the HF conv; the shipping paths (one packed conv for prefill, a tap read for + decode) are diffed against it. + """ + groups = weight.shape[0] + state_len = states.shape[-1] + slots = meta.state_slots + state = states.index_select(0, slots).to(x.dtype) + if meta.fresh_slots is not None: + state = torch.where(meta.fresh_slots.view(-1, 1, 1), torch.zeros_like(state), state) + + outs = [] + new_state = torch.empty_like(state) + offset = 0 + for i, n in enumerate(meta.seq_lens): + chunk = x[offset : offset + n].transpose(0, 1).unsqueeze(0) + history = torch.cat([state[i : i + 1], chunk], dim=-1) + out = F.conv1d(history, weight, groups=groups, dilation=dilation) + outs.append(out.squeeze(0).transpose(0, 1)) + new_state[i] = history[0, :, -state_len:] + offset += n + states.index_copy_(0, slots, new_state.to(states.dtype)) + return F.silu(torch.cat(outs, dim=0)) + + +class PLELayer(BaseOP): + """PLE block: hashed n-gram value gated by the residual streams, then a dilated depthwise conv. + + ``forward(R, batch) -> D [T, hc_count*hidden]``; the caller adds ``D`` to ``R`` before the + attention hyper-connection mix. ``meta`` defaults to ``build_ple_metadata(batch, ...)``; + ``conv_states`` defaults to ``ctx.linear_state_pool.slot_state("ple_conv", layer_id)`` and is + ``[num_slots, hc_count*hidden, (ple_conv_kernel_size-1)*ngram_size]`` in the model dtype -- the + last conv-input columns per request, oldest first. Both are arguments so the reference is + testable before the pool and the scheduler carry them. + + ``start_prefetch(batch)`` builds the metadata and starts the table gather on the backend's side + stream; call it at the top of the model forward so the rows land while layer 0 runs, and + ``forward`` joins it. + + Weight keys (checkpoint names, prefix stripped): ``key_proj.weight`` [hc*hidden, ple_embed_dim], + ``value_proj.weight`` [hidden, ple_embed_dim], ``norm_key/norm_query/norm_conv.weight`` + [hc*hidden] (zero-centered, loaded RAW), ``conv1d.weight`` [hc*hidden, 1, kernel], plus the + three ``ple_embedding`` int64 hash buffers. + """ + + def __init__( + self, config: ModelConfig, layer_id: int, table: PLETableBackend | None = None + ) -> None: + args = config.qwen4_args + self.args = args + self.layer_id = layer_id + self.ple_index = args.ple_layer_ids.index(layer_id) + self.hc_count = args.hc_count + self.hidden_size = args.hidden_size + self.dilation = args.ple_conv_dilation + self.state_len = args.ple_conv_state_len + width = args.ple_state_width + self.ple_embedding = NGramEmbedding(args, table) + self.key_proj = LinearReplicated(args.ple_embed_dim, width, has_bias=False) + self.value_proj = LinearReplicated(args.ple_embed_dim, args.hidden_size, has_bias=False) + self.norm_key = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count) + self.norm_query = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count) + self.norm_conv = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count) + self.conv1d = _DepthwiseConv1d(width, args.ple_conv_kernel_size) + from freetoken.kernel.fla.chunk import CHUNK_SIZE + + # the track snapshot gathers the last state_len conv inputs before a xCHUNK boundary; a longer history would reach before the forward's first token + assert self.state_len <= CHUNK_SIZE, ( + f"PLE conv history {self.state_len} exceeds CHUNK_SIZE {CHUNK_SIZE}" + ) + self._pending: Tuple[PLEMetadata, torch.Tensor] | None = None + + def start_prefetch(self, batch: Batch, meta: PLEMetadata | None = None) -> None: + """Hash this forward's n-grams and start the table gather on the side stream.""" + if meta is None: + meta = build_ple_metadata(batch, self.args, batch.input_ids.device) + row_ids = self.ple_embedding.row_ids(meta) + self._pending = (meta, row_ids) + self.ple_embedding.table.prefetch(row_ids) + + def forward( + self, + R: torch.Tensor, + batch: Batch, + meta: PLEMetadata | None = None, + conv_states: torch.Tensor | None = None, + ) -> torch.Tensor: + pending, self._pending = self._pending, None + row_ids = None + if meta is None: + if pending is not None: + meta, row_ids = pending + else: + meta = build_ple_metadata(batch, self.args, R.device) + elif pending is not None and pending[0] is meta: + row_ids = pending[1] + if row_ids is None: + row_ids = self.ple_embedding.row_ids(meta) + + embeddings = self.ple_embedding.table.lookup(row_ids).to(R.dtype) + key = self.norm_key.forward(self.key_proj.forward(embeddings)) + value = self.value_proj.forward(embeddings) + query = self.norm_query.forward(R) + shape = (-1, self.hc_count, self.hidden_size) + gate = (key.view(shape) * query.view(shape)).sum(-1, keepdim=True) / math.sqrt(self.hidden_size) + gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt()) + gated = (gate * value.unsqueeze(-2)).flatten(-2) + states = conv_states if conv_states is not None else self._conv_state_slab(R) + x = self.norm_conv.forward(gated) + fla = getattr(batch, "fla_metadata", None) + if fla is not None and fla.track_boundary_row is not None: + self._write_track_snapshot(states, x, fla) + return gated + self._short_conv(x, meta, states) + + def _write_track_snapshot(self, states: torch.Tensor, x: torch.Tensor, fla) -> None: + """Copy the conv history at the GDN track boundary into the same donatable slot, so a radix + prefix hit restores PLE and GDN state together. Track slots never alias the live slots this + forward advances, so the two writes are order-independent.""" + src = fla.track_boundary_row.unsqueeze(1) + torch.arange( + -self.state_len, 0, device=x.device + ) + window = x[src].transpose(-1, -2).contiguous() + states.index_copy_(0, fla.track_dst, window.to(states.dtype)) + + def _conv_state_slab(self, R: torch.Tensor) -> torch.Tensor: + pool = get_global_ctx().linear_state_pool + assert pool is not None, "PLE needs ctx.linear_state_pool or an explicit conv_states" + assert pool.has_slot_state(PLE_CONV_STATE), ( + "ModelConfig.slot_states does not declare the PLE conv history" + ) + return pool.slot_state(PLE_CONV_STATE, self.layer_id) + + def _read_state( + self, meta: PLEMetadata, states: torch.Tensor, dtype: torch.dtype + ) -> torch.Tensor: + state = states.index_select(0, meta.state_slots).to(dtype) + if meta.fresh_slots is not None: + state = torch.where(meta.fresh_slots.view(-1, 1, 1), torch.zeros_like(state), state) + return state + + def _short_conv( + self, x: torch.Tensor, meta: PLEMetadata, states: torch.Tensor + ) -> torch.Tensor: + """silu of the dilated depthwise conv over [state | x], and roll the per-request state.""" + if meta.is_decode: + return self._decode_conv(x, meta, states) + return self._prefill_conv(x, meta, states) + + def _decode_conv( + self, x: torch.Tensor, meta: PLEMetadata, states: torch.Tensor + ) -> torch.Tensor: + """Batched tap read: taps t-9, t-6, t-3 come off the state slab, tap t from this token.""" + state = self._read_state(meta, states, x.dtype) + column = x.unsqueeze(-1) + # fp32 products, like the conv1d the prefill path runs + window = torch.cat([state[..., :: self.dilation], column], dim=-1).float() + out = (window * self.conv1d.weight.squeeze(1).float()).sum(-1) + states.index_copy_( + 0, meta.state_slots, torch.cat([state[..., 1:], column], dim=-1).to(states.dtype) + ) + return F.silu(out.to(x.dtype)) + + def _prefill_conv( + self, x: torch.Tensor, meta: PLEMetadata, states: torch.Tensor + ) -> torch.Tensor: + """One conv over every request packed as ``[state_0 | chunk_0 | state_1 | chunk_1 | ...]``. + + The blocks abut exactly, so each output window stays inside its own request: request i's + first token reads history columns base_i .. base_i+state_len, which is its own state. + """ + lens = list(meta.seq_lens) + num_reqs, width = len(lens), x.shape[1] + out_index, state_index, next_state_index = self._prefill_indices(lens, x.device) + + state = self._read_state(meta, states, x.dtype) + history = x.new_empty(width, x.shape[0] + num_reqs * self.state_len) + history.index_copy_(1, state_index, state.permute(1, 0, 2).reshape(width, -1)) + history.index_copy_(1, out_index + self.state_len, x.transpose(0, 1).contiguous()) + + out = F.conv1d( + history.unsqueeze(0), self.conv1d.weight, groups=width, dilation=self.dilation + ).squeeze(0) + new_state = history.index_select(1, next_state_index).view(width, num_reqs, self.state_len) + states.index_copy_( + 0, meta.state_slots, new_state.permute(1, 0, 2).to(states.dtype).contiguous() + ) + return F.silu(out.index_select(1, out_index).transpose(0, 1)) + + def _prefill_indices(self, lens: List[int], device: torch.device): + """Columns of the packed history: this forward's outputs, the state block, the next state block.""" + state_len = self.state_len + counts = torch.tensor(lens, dtype=torch.int64) + cu = torch.cat([counts.new_zeros(1), counts.cumsum(0)]) + pad = torch.arange(len(lens), dtype=torch.int64) * state_len + base = cu[:-1] + pad + out_index = torch.arange(int(cu[-1])) + torch.repeat_interleave(pad, counts) + span = torch.arange(state_len, dtype=torch.int64) + packed = torch.cat( + [ + out_index, + (base.unsqueeze(1) + span).reshape(-1), + ((base + counts).unsqueeze(1) + span).reshape(-1), + ] + ) + if torch.cuda.is_available(): + packed = packed.pin_memory() + packed = packed.to(device, non_blocking=True) + n_out, n_state = out_index.numel(), len(lens) * state_len + return packed[:n_out], packed[n_out : n_out + n_state], packed[n_out + n_state :] + + +__all__ = [ + "GpuResidentTable", + "NGramEmbedding", + "ZeroTable", + "PLELayer", + "PLEMetadata", + "PLETableBackend", + "PinnedUVATable", + "build_ple_metadata", + "derive_ngram_hash_constants", + "short_conv_reference", +] diff --git a/python/freetoken/models/qwen4_exp/ple_disk.py b/python/freetoken/models/qwen4_exp/ple_disk.py new file mode 100644 index 0000000000..799866a61e --- /dev/null +++ b/python/freetoken/models/qwen4_exp/ple_disk.py @@ -0,0 +1,270 @@ +"""Disk-backed PLE table (--ple-backend disk): the C++ store hashes n-gram windows and batch-reads rows from the checkpoint's fp8 shard tensors into pinned staging; the captured ``lookup`` is a fixed-shape H2D copy + dequant. + +Hash windows are pure functions of ``req.input_ids`` + ``device_len`` (prefix hits, restores and COW forks need no bookkeeping); the decode input token lives device-side under overlap scheduling and is read back here. +""" + +from __future__ import annotations + +import os +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Sequence + +import safetensors +import torch + +from freetoken.core import Batch +from freetoken.kernel.pinned import alloc_pinned_tensor +from freetoken.utils import init_logger + +from .weight import ( + _PLE_SCALE_SUFFIX, + _PLE_SHARD_RE, + _PLE_ST_DTYPE, + _ple_table_files, + _safetensors_header, +) + +_IO_URING_ENV = "FREETOKEN_PLE_IO_URING" +_SYNC_ENV = "FREETOKEN_PLE_SYNC" # auto | wait | gate + +logger = init_logger(__name__) + + +def _context(ids: torch.Tensor, position: int, eos: int) -> list[int]: + """The two token ids before ``position``; eos pads past the start.""" + return [int(ids[position - 2]) if position >= 2 else eos, + int(ids[position - 1]) if position >= 1 else eos] + + +@dataclass(frozen=True) +class PleRowSource: + """On-disk row layout: equal extents, row i of an extent at ``base + i * row_stride`` (a repacked flat file is one extent with its own stride).""" + + paths: list[str] + extent_file: list[int] + extent_base: list[int] + rows_per_extent: int + row_bytes: int + row_stride: int + scale: float + + @property + def total_rows(self) -> int: + return len(self.extent_base) * self.rows_per_extent + + +def source_from_safetensors(folder: str) -> PleRowSource: + """Map the checkpoint's ``ngram_embedding.shard_`` tensors in place: one extent per shard, no copy.""" + rows = cols = 0 + scale: torch.Tensor | None = None + paths: list[str] = [] + path_idx: dict[str, int] = {} + shards: dict[int, tuple[int, int]] = {} + for path in _ple_table_files(folder): + header, base = _safetensors_header(path) + for key, meta in header.items(): + if key == "__metadata__": + continue + if key.endswith(_PLE_SCALE_SUFFIX): + with safetensors.safe_open(path, framework="pt", device="cpu") as f: + scale = f.get_tensor(key).reshape(()) + continue + match = _PLE_SHARD_RE.search(key) + if match is None: + continue + if meta["dtype"] != _PLE_ST_DTYPE: + raise ValueError(f"PLE shard {key} has dtype {meta['dtype']}, expected {_PLE_ST_DTYPE}") + if rows and tuple(meta["shape"]) != (rows, cols): + raise ValueError(f"PLE shard {key} is {meta['shape']}, expected {[rows, cols]}") + rows, cols = meta["shape"] + if path not in path_idx: + path_idx[path] = len(paths) + paths.append(path) + idx = int(match.group("shard")) + if idx in shards: + raise ValueError(f"duplicate PLE shard {idx} in {path}") + shards[idx] = (path_idx[path], base + meta["data_offsets"][0]) + if sorted(shards) != list(range(len(shards))) or not shards: + raise ValueError(f"PLE shard indices are not contiguous 0..N-1: {sorted(shards)[:8]}") + if scale is None: + raise ValueError("PLE table has no weight_scale") + order = [shards[i] for i in range(len(shards))] + return PleRowSource(paths, [f for f, _ in order], [b for _, b in order], rows, cols, cols, float(scale)) + + +def resolve_row_source(folder: str) -> PleRowSource: + """Pick the row source for a checkpoint; the seam where a repacked format would plug in.""" + return source_from_safetensors(folder) + + +class DiskRowTable: + """``PLETableBackend`` whose rows are read from disk per fill (--ple-backend disk).""" + + def __init__( + self, + source: PleRowSource, + hash_constants: dict, + *, + max_graph_rows: int = 256, + max_extend_tokens: int = 8192, + dtype: torch.dtype = torch.bfloat16, + ) -> None: + from freetoken.kernel import _ple_store + + self.num_rows = source.total_rows + self.head_dim = source.row_bytes # fp8: one byte per element + self.dtype = dtype + self.heads = int(hash_constants["num_ngram_heads"]) + self.scale = source.scale + self.eos_token_id = int(hash_constants["eos_token_id"]) + sizes = [int(x) for x in hash_constants["per_head_vocab_sizes"]] + offsets = [int(x) for x in hash_constants["per_head_offsets"]] + need = max(o + s for o, s in zip(offsets, sizes)) + if need > source.total_rows: + raise ValueError( + f"PLE row source holds {source.total_rows} rows but the hash addresses {need}; incomplete checkpoint?" + ) + self._store = _ple_store.PleStore( + paths=list(source.paths), + extent_file=list(source.extent_file), + extent_base=list(source.extent_base), + rows_per_extent=source.rows_per_extent, + row_bytes=source.row_bytes, + row_stride=source.row_stride, + multipliers=[int(x) for x in hash_constants["layer_multipliers"]], + head_vocab_sizes=sizes, + head_offsets=offsets, + eos_token_id=self.eos_token_id, + use_io_uring=os.getenv(_IO_URING_ENV, "1") != "0", + ) + self._device = torch.device("cuda", torch.cuda.current_device()) + self._token_bytes = self.heads * self.head_dim + # allocated up front: pinned alloc inside stream capture is illegal; one replay consumes it at a time + self._graph_pinned = alloc_pinned_tensor(max_graph_rows * self._token_bytes, dtype=torch.uint8) + self._graph_pinned.zero_() # padded decode lanes read whatever sits here + # outlives any one graph: a cache rebuild recaptures against the same pointer + self._graph_dev = torch.empty( + max_graph_rows * self._token_bytes, dtype=torch.uint8, device=self._device + ) + eager_bytes = max_extend_tokens * self._token_bytes + self._eager_pinned = alloc_pinned_tensor(eager_bytes, dtype=torch.uint8) + self._eager_pinned.zero_() # the warmup prefill stages nothing and reads whatever sits here + self._eager_dev = torch.empty(eager_bytes, dtype=torch.uint8, device=self._device) + # probe picks flag-sync (graph WAITs at the consume, host fills then signals) or launch-gating + self._wait_sync = self._probe_wait_sync(os.getenv(_SYNC_ENV, "auto")) + # one flag for all graphs: the readback event orders a fill after the previous graph, so signals never overlap + self._flag = alloc_pinned_tensor(1, dtype=torch.int64) + self._flag.zero_() + self._token_readback = alloc_pinned_tensor(max_graph_rows, dtype=torch.int32) + self._readback_event = torch.cuda.Event() + sync = "wait-sync" if self._wait_sync else "launch-gating" + logger.info_rank0(f"PLE disk backend: {self._store.io_backend()}, {sync}") + + def _probe_wait_sync(self, mode: str) -> bool: + from freetoken.kernel import _ple_store + + if mode == "gate": + return False + scratch = alloc_pinned_tensor(1, dtype=torch.int64) + scratch.zero_() + stream = torch.cuda.current_stream(self._device) + ok = ( + _ple_store.memop_write(stream.cuda_stream, scratch.data_ptr(), 7) == 0 + and _ple_store.memop_wait_geq(stream.cuda_stream, scratch.data_ptr(), 7) == 0 + ) + if ok: + stream.synchronize() + ok = int(scratch[0]) == 7 + if mode == "wait" and not ok: + raise RuntimeError("FREETOKEN_PLE_SYNC=wait but stream memops are unavailable") + return ok + + # ---------------- host side (engine thread, before the forward launches) ---------------- + + def fill(self, runs: Sequence[torch.Tensor], *, graph: bool) -> None: + """Stage per-request token runs (two context ids, then the new tokens) in batch order.""" + pinned = self._graph_pinned if graph else self._eager_pinned + offset = 0 + for run in runs: + self._store.stage(run.data_ptr(), run.numel() - 2, pinned.data_ptr() + offset * self._token_bytes) + offset += run.numel() - 2 + self._store.flush(self._flag.data_ptr() if graph and self._wait_sync else 0) + + def host_fill_batch(self, batch: Batch, use_graph: bool): + """Stage this batch's rows; returns the post-dispatch fill callable under flag-sync, else None.""" + eos = self.eos_token_id + if batch.is_decode: + reqs = list(batch.reqs) + if use_graph and self._wait_sync: + bs = batch.padded_size + self._token_readback[:bs].copy_(batch.input_ids, non_blocking=True) + self._readback_event.record(torch.cuda.current_stream(self._device)) + + def _complete() -> None: + try: + self._readback_event.synchronize() + tokens = self._token_readback[:bs].to(torch.int64).tolist() + runs = [torch.tensor([*_context(r.input_ids, r.device_len - 1, eos), t], dtype=torch.int64) + for r, t in zip(reqs, tokens)] + self.fill(runs, graph=True) + except BaseException: + from freetoken.kernel import _ple_store + + # unblock the stream before surfacing; the step's output is discarded + _ple_store.signal_flag(self._flag.data_ptr()) + raise + + return _complete + # launch-gating: this D2H is the step's readback and orders the fill after sampling + tokens = batch.input_ids.to("cpu").to(torch.int64).tolist() + runs = [torch.tensor([*_context(r.input_ids, r.device_len - 1, eos), t], dtype=torch.int64) + for r, t in zip(reqs, tokens)] + self.fill(runs, graph=use_graph) + return None + runs = [ + torch.cat(( + torch.tensor(_context(req.input_ids, req.cached_len, eos), dtype=torch.int64), + req.input_ids[req.cached_len : req.device_len].to(torch.int64), + )) + for req in batch.padded_reqs + ] + self.fill(runs, graph=False) + return None + + @contextmanager + def forward_host_ctx(self, batch: Batch, use_graph: bool): + """Around one dispatch: stage on enter, run the deferred fill+signal on exit.""" + deferred = self.host_fill_batch(batch, use_graph) + yield + # no try/finally: a failed launch leaves no WAIT pending, so the fill must not run + if deferred is not None: + deferred() + + # ---------------- device side (PLETableBackend protocol) ---------------- + + def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor: + rows = row_ids.shape[0] + capturing = torch.cuda.is_current_stream_capturing() + if capturing and self._wait_sync: + from freetoken.kernel import _ple_store + + _ple_store.memop_wait_reset( + torch.cuda.current_stream(self._device).cuda_stream, self._flag.data_ptr() + ) + pinned, dev = ( + (self._graph_pinned, self._graph_dev) if capturing else (self._eager_pinned, self._eager_dev) + ) + nbytes = rows * self._token_bytes + dev[:nbytes].copy_(pinned[:nbytes], non_blocking=True) + values = dev[:nbytes].view(torch.float8_e4m3fn).to(self.dtype) + if self.scale != 1.0: + values = values * self.scale + values = values.view(*row_ids.shape[:-1], -1) + if out is None: + return values + out.copy_(values) + return out + + def prefetch(self, row_ids: torch.Tensor) -> None: + return None diff --git a/python/freetoken/models/qwen4_exp/weight.py b/python/freetoken/models/qwen4_exp/weight.py new file mode 100644 index 0000000000..f8d2a74940 --- /dev/null +++ b/python/freetoken/models/qwen4_exp/weight.py @@ -0,0 +1,328 @@ +"""Qwen3.8-Flash-Next (RadixArk NVFP4) checkpoint reader. + +Three separate paths, because the checkpoint's three weight classes live in different places: + +* :func:`iter_weights` -- every dense (non-expert) tensor, with the ``model.language_model.`` prefix stripped and fused where the model expects one buffer. See ``_FUSIONS``. +* :func:`load_ple_table` -- the 47.7 GiB FP8 n-gram table, 128 checkpoint shards concatenated into one pinned :class:`HostBank`. +* :func:`load_nvfp4_expert_sources` -- the routed NVFP4 experts, into the offload cache's source banks. + +Dropped: ``mtp.*`` (speculative head, including its stacked ``mtp.layers.0.mlp.experts.*``) and ``model.visual.*`` (served text-only). +""" + +from __future__ import annotations + +import json +import os +import re +import struct +from dataclasses import dataclass +from typing import Iterator + +import safetensors +import torch +from freetoken.distributed import get_tp_info +from freetoken.models.loader import drop_page_cache, iter_weight_files +from freetoken.models.nvfp4_banks import ( + Nvfp4ExpertSourceSpec, + load_nvfp4_expert_source_banks, +) +from freetoken.moe.host_banks import HostBank, read_range_into +from freetoken.utils import download_hf_weight +from freetoken.utils.progress import byte_bar +from tqdm import tqdm + +# Routed NVFP4 experts (nvidia modelopt layout): per-expert, un-fused. Matched against the RAW +# weight_map key in nvfp4_banks. The ``model.language_model.`` anchor excludes the MTP head's +# stacked ``mtp.layers.N.mlp.experts.*`` tensors. +_EXPERT_KEY_RE = re.compile( + r"^model\.language_model\.layers\.(?P\d+)\.mlp\.experts\.(?P\d+)\." + r"(?Pgate_proj|up_proj|down_proj)\.(?Pweight|weight_scale|weight_scale_2)$" +) +_EXPERT_RE = re.compile(r"\.mlp\.experts\.\d+\.") +_NVFP4_SOURCE_SPEC = Nvfp4ExpertSourceSpec( + key_pattern=_EXPERT_KEY_RE, + proj_to_role={"gate_proj": "gate", "up_proj": "up", "down_proj": "down"}, + layer_to_bank=lambda layer, config: layer, # every layer is MoE + desc="Qwen3.8-Flash-Next NVFP4 experts", +) +# Per-tensor modelopt quant scales; consumed with their ``.weight`` (experts) or unused. +_SCALE_SUFFIXES = (".weight_scale", ".weight_scale_2", ".input_scale") + +# The n-gram table itself: too big for the dense state dict, loaded by load_ple_table. +_PLE_TABLE_INFIX = ".ple.ple_embedding.ngram_embedding." +_PLE_SHARD_RE = re.compile( + r"\.ple\.ple_embedding\.ngram_embedding\.shard_(?P\d+)\.weight$" +) +_PLE_SCALE_SUFFIX = ".ple.ple_embedding.ngram_embedding.weight_scale" + +# Zero-centered Qwen4ExpTextRMSNorm weights, loaded RAW: GroupedPlusOneRMSNorm / GemmaPlusOneRMSNorm +# and the vendored grouped_gemma_rmsnorm all apply (1+w) at runtime in fp32, so folding the +1 into +# the bf16 weight here would double-apply it and round away small |w|. The GDN gated norm +# (linear_attn.norm) is a plain weight*x norm and is not in this set. +_ZERO_CENTERED_NORM_SUFFIXES = ( + ".hc_norm.weight", + ".ple.norm_key.weight", + ".ple.norm_query.weight", + ".ple.norm_conv.weight", + ".self_attn.q_norm.weight", + ".self_attn.k_norm.weight", + ".self_attn.indexer.q_layernorm.weight", + ".self_attn.indexer.k_layernorm.weight", +) + +# Fused projections: concat the checkpoint parts along dim 0 in this exact order. A nonzero pad +# rounds the merged row count up; the model splits the result back with the same sizes. +_FUSIONS: dict[str, tuple[tuple[str, ...], int]] = { + # q carries the output gate, so its half is twice the attention width: [2*qo | kv | kv]. + ".self_attn.qkv_proj.weight": (( + ".self_attn.q_proj.weight", ".self_attn.k_proj.weight", ".self_attn.v_proj.weight", + ), 0), + ".linear_attn.in_proj.weight": (( + ".linear_attn.in_proj_qkv.weight", ".linear_attn.in_proj_z.weight", + ".linear_attn.in_proj_b.weight", ".linear_attn.in_proj_a.weight", + ), 0), + ".mlp.shared_expert.gate_up_proj.weight": (( + ".mlp.shared_expert.gate_proj.weight", ".mlp.shared_expert.up_proj.weight", + ), 0), + # HC mix reads the low-rank down projection and the injection logits from one GEMM; vLLM + # pads the merged output to a multiple of 16 rows for cuBLAS (hyperconnection.py pad_size). + # The top-level hyper_connection_mixer has no injection and so never fuses. + ".attn_hyper_connection.input_mix_weight_down_block_inject.weight": (( + ".attn_hyper_connection.input_mix_weight_down.weight", + ".attn_hyper_connection.block_inject_weight.weight", + ), 16), + ".mlp_hyper_connection.input_mix_weight_down_block_inject.weight": (( + ".mlp_hyper_connection.input_mix_weight_down.weight", + ".mlp_hyper_connection.block_inject_weight.weight", + ), 16), +} + + +def _rename(raw_name: str) -> str | None: + """Checkpoint key -> FreeToken state-dict key, or None to skip.""" + if raw_name.startswith(("mtp.", "model.visual.", "visual.")): + return None + if _PLE_TABLE_INFIX in raw_name: + return None # n-gram table + its scale: load_ple_table + if _EXPERT_RE.search(raw_name): + return None # routed experts: offload source banks + if raw_name.endswith(_SCALE_SUFFIXES): + return None + if raw_name.startswith("model.language_model."): + return "model." + raw_name[len("model.language_model.") :] + if raw_name.startswith("language_model."): + return "model." + raw_name[len("language_model.") :] + return raw_name + + +def _try_fuse( + name: str, tensor: torch.Tensor, buf: dict[str, dict[int, torch.Tensor]] +) -> tuple[str, torch.Tensor] | tuple[()] | None: + """Buffer a fusion part; return the merged ``(name, tensor)`` once all parts arrive, ``()`` while incomplete, ``None`` if ``name`` is not a fusion part.""" + for fused_suffix, (parts, pad_to) in _FUSIONS.items(): + for idx, part in enumerate(parts): + if not name.endswith(part): + continue + key = name[: -len(part)] + fused_suffix + slots = buf.setdefault(key, {}) + slots[idx] = tensor + if len(slots) < len(parts): + return () + del buf[key] + rows = [slots[i] for i in range(len(parts))] + pad = (-sum(t.shape[0] for t in rows)) % pad_to if pad_to else 0 + if pad: + rows.append(torch.zeros(pad, *rows[0].shape[1:], dtype=rows[0].dtype, device=rows[0].device)) + return key, torch.cat(rows, dim=0) + return None + + +def iter_weights( + model_path: str, + device: torch.device, + *, + include_moe_experts: bool, + include_non_moe: bool, +) -> Iterator[tuple[str, torch.Tensor]]: + """Yield the dense (non-expert) weights, prefix-stripped and fused to the model's buffers. + + Keys keep the checkpoint's module names below the stripped prefix, so the emitted set is the + model's state dict minus the routed experts. Nothing here is quantized: the modelopt + ``ignore`` list covers everything except those experts, so attention, GDN, HC, PLE, the shared + expert and lm_head are all plain bf16 (the n-gram hash constants stay int64). Fusions: + attention q|k|v -> ``qkv_proj``, GDN ``in_proj_{qkv,z,b,a}`` -> ``in_proj``, shared-expert + gate|up -> ``gate_up_proj``, and each per-layer HC's ``input_mix_weight_down`` | + ``block_inject_weight`` -> a zero-padded ``input_mix_weight_down_block_inject``. + + ``include_moe_experts`` is accepted for the loader contract but never yields anything: the + routed experts are NVFP4 and always come from :func:`load_nvfp4_expert_sources`. + """ + if get_tp_info().size > 1: + raise NotImplementedError("qwen4_exp weight loading supports TP=1 only") + if not include_non_moe: + return + + fuse_buf: dict[str, dict[int, torch.Tensor]] = {} + for file in tqdm( + iter_weight_files(model_path), + desc="Loading weights", + disable=not get_tp_info().is_primary(), + ): + with safetensors.safe_open(file, framework="pt", device=str(device)) as f: + for raw_name in f.keys(): + name = _rename(raw_name) + if name is None: + continue + tensor = f.get_tensor(raw_name) + fused = _try_fuse(name, tensor, fuse_buf) + if fused is not None: + if fused != (): # () means buffered, not yet complete + yield fused + continue + yield name, tensor + + assert not fuse_buf, f"Incomplete projection fusions: {sorted(fuse_buf)}" + + +# ====================================================================================== +# PLE n-gram table +# ====================================================================================== + + +@dataclass(frozen=True) +class PleTable: + """The filled n-gram table: one pinned host bank plus the checkpoint's per-tensor FP8 scale.""" + + bank: HostBank + weight_scale: torch.Tensor # scalar, checkpoint dtype (bf16) + + @property + def tensor(self) -> torch.Tensor: + """``[total_rows, ngram_head_dim]`` float8_e4m3fn view of the bank.""" + return self.bank.tensor + + +_PLE_ST_DTYPE = "F8_E4M3" + + +def _safetensors_header(path: str) -> tuple[dict, int]: + with open(path, "rb") as fh: + n = struct.unpack(" list[str]: + """Shards holding a piece of the n-gram table, from the index when there is one.""" + index = os.path.join(folder, "model.safetensors.index.json") + if not os.path.exists(index): + return sorted(iter_weight_files(folder)) + with open(index, encoding="utf-8") as fh: + weight_map = json.load(fh)["weight_map"] + files = {shard for name, shard in weight_map.items() if _PLE_TABLE_INFIX in name} + return sorted(os.path.join(folder, shard) for shard in files) + + +def load_ple_table(model_path: str, qwen4_args, *, pin: bool = True, + workers: int = 8, chunk: int = 8 << 20) -> PleTable: + """Concatenate the checkpoint's ``ngram_embedding.shard_`` tensors into one pinned host bank. + + The checkpoint splits the table into ``split_ngram_parts`` equal row blocks named by shard + index and scattered over the ``model-plefp8-*`` shards in header (lexicographic) order, so the + bank is filled shard by shard at ``shard_index * rows_per_shard``. Each read is O_DIRECT: the + table is ~47.7 GiB and must not also sit in the page cache while the bank holds the same bytes. + """ + folder = download_hf_weight(model_path) + parts: dict[int, tuple[str, int, int]] = {} # shard index -> (path, file offset, bytes) + scale: torch.Tensor | None = None + rows = cols = 0 + for path in _ple_table_files(folder): + header, base = _safetensors_header(path) + for key, meta in header.items(): + if key == "__metadata__": + continue + if key.endswith(_PLE_SCALE_SUFFIX): + with safetensors.safe_open(path, framework="pt", device="cpu") as f: + scale = f.get_tensor(key).reshape(()) + continue + match = _PLE_SHARD_RE.search(key) + if match is None: + continue + if meta["dtype"] != _PLE_ST_DTYPE: + raise ValueError(f"PLE table shard {key} has unsupported dtype {meta['dtype']}") + shape = meta["shape"] + if rows and tuple(shape) != (rows, cols): + raise ValueError(f"PLE table shard {key} is {shape}, expected {[rows, cols]}") + rows, cols = shape + begin, end = meta["data_offsets"] + parts[int(match.group("shard"))] = (path, base + begin, end - begin) + + expected = int(qwen4_args.split_ngram_parts) + if sorted(parts) != list(range(expected)): + raise ValueError( + f"PLE table needs shards 0..{expected - 1}, found {len(parts)}: {sorted(parts)[:8]}" + ) + if cols != qwen4_args.ngram_head_dim: + raise ValueError(f"PLE table row is {cols} wide, config says {qwen4_args.ngram_head_dim}") + if scale is None: + raise ValueError("PLE table has no weight_scale") + + bank = HostBank((expected * rows, cols), torch.float8_e4m3fn) + shard_bytes = rows * cols + bar = byte_bar(expected * shard_bytes, "Loading PLE table") + try: + buf = bank.memoryview() + for shard in range(expected): + path, offset, nbytes = parts[shard] + assert nbytes == shard_bytes, f"PLE shard {shard} is {nbytes} B, expected {shard_bytes}" + read_range_into(buf, path, file_offset=offset, nbytes=nbytes, + dest_offset=shard * shard_bytes, workers=workers, chunk=chunk) + bar.update(nbytes) + finally: + bar.close() + if pin and torch.cuda.is_available(): + bank.pin() + return PleTable(bank=bank, weight_scale=scale) + + +# ====================================================================================== +# Routed NVFP4 experts +# ====================================================================================== + + +def load_nvfp4_expert_sources(model_path: str, config, *, layer_sink=None) -> dict: + """Build the CPU NVFP4 expert source banks for the offload cache (gate/up fused on the output-row axis, down separate; weight_scale_2 carried as the per-row global scale).""" + return load_nvfp4_expert_source_banks( + model_path, + config, + _NVFP4_SOURCE_SPEC, + drop_page_cache=drop_page_cache, + primary=get_tp_info().is_primary(), + layer_sink=layer_sink, + ) + + +def load_nvfp4_expert_sources_parallel( + model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None +) -> dict: + """parallel: same NVFP4 source banks via the common chunked multi-threaded reader.""" + from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel + + return load_nvfp4_expert_source_banks_parallel( + model_path, + config, + _NVFP4_SOURCE_SPEC, + drop_page_cache=drop_page_cache, + primary=get_tp_info().is_primary(), + workers=workers, + chunk=chunk, + layer_sink=layer_sink, + ) + + +__all__ = [ + "PleTable", + "iter_weights", + "load_nvfp4_expert_sources", + "load_nvfp4_expert_sources_parallel", + "load_ple_table", +] diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py index 0c033ca012..fe4028312c 100644 --- a/python/freetoken/models/register.py +++ b/python/freetoken/models/register.py @@ -58,6 +58,14 @@ class ModelSpec: "freetoken.models.qwen3_5_moe", "Qwen3_5MoEForCausalLM", ), + # Qwen3.8-Flash-Next (model_type qwen4_exp): multimodal wrapper config (text tower in + # text_config, weights under model.language_model.); served text-only. 36 GDN + 12 QSA + # compressed-sparse attention layers on 4 hyper-connection residual streams, a PLE + # n-gram embedding layer, 512 NVFP4 routed experts top-10 + a gated shared expert. + "Qwen4ExpForConditionalGeneration": ModelSpec( + "freetoken.models.qwen4_exp", + "Qwen4ExpForCausalLM", + ), # Dense Qwen3.x (no "Moe" in the arch name, num_experts==0, e.g. Qwen3.6-27B). Shares the # qwen3_5_moe package: the decoder routes its MLP through the dense Qwen3_5DenseMLP and the # loader handles the compressed-tensors NVFP4 layout. @@ -107,6 +115,12 @@ class ModelSpec: parse_config="parse_gguf_config", iter_weights="iter_gguf_weights", ), + "Qwen3_5MoeGGUFForCausalLM": ModelSpec( + "freetoken.models.qwen3_5_moe", + "Qwen3_5MoEForCausalLM", + parse_config="parse_gguf_config", + iter_weights="iter_gguf_weights", + ), "GptOssForCausalLM": ModelSpec( "freetoken.models.gpt_oss", "GptOssForCausalLM", @@ -122,6 +136,19 @@ class ModelSpec: "freetoken.models.glm_moe_dsa", "GlmMoeDsaForCausalLM", ), + # GLM-5.3-Flash (model_type glm5_next): hybrid KDA linear attention (34/45 layers) + # + NoPE-MLA/DSA with a kpool-compressed indexer (11/45), mHC x4 residual streams, + # 288-expert sigmoid/noaux_tc MoE; natively-multimodal wrapper config (text tower + # in text_config, weights under model.language_model.), served text-only. + "Glm5NextForConditionalGeneration": ModelSpec( + "freetoken.models.glm5_next", + "Glm5NextForCausalLM", + ), + # Text-only sibling (the text_config's own architectures entry). + "Glm5NextForCausalLM": ModelSpec( + "freetoken.models.glm5_next", + "Glm5NextForCausalLM", + ), } diff --git a/python/freetoken/moe/benchbw.py b/python/freetoken/moe/benchbw.py index f3e5359a44..83b3d19665 100644 --- a/python/freetoken/moe/benchbw.py +++ b/python/freetoken/moe/benchbw.py @@ -122,6 +122,10 @@ class Workload: activation="gpt_oss_swiglu", swiglu_limit=7.0), "dsv4": Workload("dsv4", 4096, 2048, 256, 6, ("ds_fp4",), swiglu_limit=7.0), "glm4.7-nvfp4": Workload("glm4.7-nvfp4", 5120, 1536, 160, 8, ("nvfp4",)), + "glm5.3-flash-nvfp4": Workload( + "glm5.3-flash-nvfp4", 4096, 2048, 288, 8, ("nvfp4",), + activation="swiglu_clamp", swiglu_alpha=1.0, swiglu_limit=10.0, + ), "minimax-m2.5": Workload("minimax-m2.5", 3072, 1536, 256, 8, ("nvfp4",)), } diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.py index b96205aa64..d9cb43c189 100644 --- a/python/freetoken/moe/cpu_executor.py +++ b/python/freetoken/moe/cpu_executor.py @@ -65,6 +65,7 @@ "gelu_pytorch_tanh": 2, "gpt_oss_swiglu": 3, "swigluoai": 3, + "swiglu_clamp": 4, } # Weight-format ids must match WFmt in csrc/cpu_moe/cpu_moe_ext.cpp. diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.py index 8b6116ba87..5367ba9faa 100644 --- a/python/freetoken/moe/expert_banks.py +++ b/python/freetoken/moe/expert_banks.py @@ -49,6 +49,13 @@ class ExpertBanks: # streamed straight to its sink instead of staying materialized here) -- set by # convert.py's per-format streaming gate; ``sources`` may hold released tensors. streamed: bool = False + # Some GGUF recipes use one exceptional packed layout for a small subset of + # layers. It receives its own cache because cache banks must have a uniform + # row geometry. ``auxiliary_layer_ids`` maps model layer id to its index in + # the auxiliary source list. + auxiliary_quant_format: str | None = None + auxiliary_sources: dict[str, list[torch.Tensor]] | None = None + auxiliary_layer_ids: tuple[int, ...] = () _PARALLEL_CHUNK = 8 << 20 # default O_DIRECT chunk for the parallel reader @@ -252,6 +259,38 @@ def _q4_0_banks(model_path, model_config, device, dtype, dummy, parallel=False, ) +def _q4_k_q5_k_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks: + """Load Qwen's mixed Q4_K gate/up and Q5_K down GGUF expert banks.""" + if parallel: + raise NotImplementedError( + "parallel reader not implemented for q4_k_q5_k: the source is one GGUF file" + ) + from freetoken.models.qwen3_5_moe.gguf import ( + dummy_q4_k_q5_k_expert_sources, + load_q4_k_q5_k_expert_sources, + ) + + sink = None if dummy else layer_sink + sources = ( + dummy_q4_k_q5_k_expert_sources(model_config) + if dummy + else load_q4_k_q5_k_expert_sources(model_path, model_config, layer_sink=sink) + ) + auxiliary_sources = None + auxiliary_format = None + if sources.q6_layer_ids: + auxiliary_format = "q6_k_down" + auxiliary_sources = {"down": sources.q6_down} + return ExpertBanks( + "q4_k_q5_k", + {name: sources.primary[name] for name in _BANK_SCHEMAS["q4_k_q5_k"]}, + streamed=sink is not None, + auxiliary_quant_format=auxiliary_format, + auxiliary_sources=auxiliary_sources, + auxiliary_layer_ids=sources.q6_layer_ids, + ) + + def _dsfp4_banks(model_path, model_config, device, dtype, dummy, parallel=False, workers=8, chunk=_PARALLEL_CHUNK, decode_target="gpu", layer_sink=None) -> ExpertBanks: args = model_config.dsv4_args assert args is not None, "ds_fp4 expert banks require dsv4_args on the model config" @@ -301,6 +340,7 @@ def _model_setup_override(model_config): "nvfp4": _nvfp4_banks, "ds_fp4": _dsfp4_banks, "q4_0": _q4_0_banks, + "q4_k_q5_k": _q4_k_q5_k_banks, } diff --git a/python/freetoken/moe/fused.py b/python/freetoken/moe/fused.py index fe7e417d72..1e3a55a8c0 100644 --- a/python/freetoken/moe/fused.py +++ b/python/freetoken/moe/fused.py @@ -10,6 +10,7 @@ logger = init_logger(__name__) +# The guarded ROCm fallback logs once per process rather than once per MoE layer. _warned_torch_topk = False @@ -19,7 +20,7 @@ def _torch_fused_topk( renormalize: bool, num_token_non_padded: torch.Tensor | None, ) -> Tuple[torch.Tensor, torch.Tensor]: - """Pure-torch softmax router matching triton_kernels.topk (Windows fallback). + """Pure-torch reference for the fused softmax router; tests compare the kernel against it. Softmax over all experts, select the top-k, and (when ``renormalize``) rescale the selected weights to sum to 1 -- the standard fused-MoE routing convention. @@ -44,45 +45,34 @@ def fused_topk( ) -> Tuple[torch.Tensor, torch.Tensor]: assert hidden_states.shape[0] == gating_output.shape[0], "Number of tokens mismatch" - from freetoken.kernel.backend import is_triton_kernels_installed + from freetoken.kernel.backend import is_rocm_runtime - # triton_kernels ships no Windows wheel, and unlike flashinfer/sgl_kernel it is not one - # of the six ops the in-repo triton kernels cover -- so this router needs its own fallback. - if not is_triton_kernels_installed(): + # Upstream's in-tree Triton router is the default everywhere other than HIP. + # On ROCm the independently tested PyTorch implementation remains the default + # until a full end-to-end quality gate demonstrates an identical result. Set + # this explicit experiment flag to ``1`` only when validating that candidate; + # this leaves model weights and server configuration unchanged. + use_rocm_triton_router = is_rocm_runtime() and os.environ.get( + "FREETOKEN_ROCM_TRITON_ROUTER", "0" + ) == "1" + if use_rocm_triton_router: + from freetoken.kernel.triton.moe_router import fused_topk_softmax + + return fused_topk_softmax(gating_output, topk, renormalize, num_token_non_padded) + + if is_rocm_runtime(): global _warned_torch_topk if not _warned_torch_topk: _warned_torch_topk = True - # Once, not per call: this runs every MoE forward. On Linux a missing - # triton_kernels used to fail fast with ImportError; keep the misconfiguration - # visible without giving up the fallback that Windows needs. logger.warning_rank0( - "fused_topk: triton_kernels is not installed -> pure-torch router fallback " - "(numerically equivalent, slower). Expected on Windows (no wheel); on Linux " - "install triton_kernels to restore the fused router." + "fused_topk: ROCm keeps the quality-proven pure-torch router; " + "set FREETOKEN_ROCM_TRITON_ROUTER=1 only for a guarded experiment." ) return _torch_fused_topk(gating_output, topk, renormalize, num_token_non_padded) - from triton_kernels.topk import topk as triton_kernels_topk + from freetoken.kernel.triton.moe_router import fused_topk_softmax - logits = gating_output.float() - softmax_first = not renormalize - if softmax_first: - logits = torch.softmax(logits, dim=-1) - sparse_topk = triton_kernels_topk( - logits, - topk, - apply_softmax=not softmax_first, - ) - if hasattr(sparse_topk, "vals"): - topk_weights = sparse_topk.vals - topk_ids = sparse_topk.indx - else: - topk_weights, topk_ids = sparse_topk[:2] - topk_ids = topk_ids.to(torch.int32) - if num_token_non_padded is not None: - indices = torch.arange(0, topk_ids.shape[0], device=topk_ids.device) - topk_ids[indices >= num_token_non_padded, :] = -1 - return topk_weights, topk_ids + return fused_topk_softmax(gating_output, topk, renormalize, num_token_non_padded) def moe_align_block_size( @@ -125,7 +115,19 @@ def moe_align_block_size( - The padding ensures that the total number of tokens is now divisible by block_size for proper block matrix operations. """ - from freetoken.kernel.backend import is_sgl_kernel_installed + from freetoken.kernel.backend import is_rocm_runtime, is_sgl_kernel_installed + + # The compact in-tree alignment kernel is tuned around NVIDIA execution + # assumptions. On gfx1151 it can leave the expert-block array at its + # initializer value even when token IDs are correctly scattered, sending + # every grouped GEMM block to expert zero. Use the repository's staged + # alignment implementation on ROCm instead: it produced the correct block + # ownership for the isolated 4-token, 37-expert reproducer and avoids that + # unsafe small-kernel path. + if is_rocm_runtime(): + from freetoken.kernel import moe_align_block_size_triton + + return moe_align_block_size_triton(topk_ids, block_size, num_experts) if not is_sgl_kernel_installed(): from freetoken.kernel.triton.moe_align import ( @@ -318,7 +320,13 @@ def fused_experts_impl( fused_moe_kernel_triton( intermediate_cache2, w2, - (intermediate_cache3), + # The second projection consumes one flattened row for every routed + # token. Present the output with the matching [M * top_k, 1, N] + # layout so ``fused_moe_kernel`` advances by one routed row when it + # receives ``top_k=1``. Passing the original [M, top_k, N] view + # makes its flattened routing indices use the larger M stride, which + # can address past the allocated output buffer on ROCm. + intermediate_cache3.view(M * topk_ids.shape[1], 1, w2.shape[1]), curr_topk_weights, curr_topk_ids, sorted_token_ids, diff --git a/python/freetoken/moe/fused_nvfp4.py b/python/freetoken/moe/fused_nvfp4.py index 5ed8d76692..dc587d8f99 100644 --- a/python/freetoken/moe/fused_nvfp4.py +++ b/python/freetoken/moe/fused_nvfp4.py @@ -7,6 +7,7 @@ from __future__ import annotations +import os from typing import Any, Dict import torch @@ -25,6 +26,7 @@ gelu_and_mul, gelu_tanh_and_mul, silu_and_mul, + swiglu_clamp_and_mul, swigluoai_and_mul, ) from freetoken.moe.fused import moe_align_block_size @@ -40,12 +42,16 @@ def _run_act( act_limit: float, ) -> None: """gemm1 -> gemm2 activation dispatch. ``swigluoai`` (MiniMax-M3, clamped - gpt-oss swiglu over the banks' uninterleaved [gate; up] halves) carries the + gpt-oss swiglu over the banks' uninterleaved [gate; up] halves) and + ``swiglu_clamp`` (GLM-5.3, same clamp without the +1 up bias) carry the per-model ``act_alpha``/``act_limit`` scalars; the plain *_and_mul kinds ignore them.""" if activation == "swigluoai": swigluoai_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit) return + if activation == "swiglu_clamp": + swiglu_clamp_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit) + return _ACT[activation](gate_up, out) # Decode is captured into a CUDA graph, so the config must be fixed (no triton.autotune, @@ -61,6 +67,37 @@ def _run_act( _DECODE_MARLIN_BLOCK_N = 16 _DECODE_MARLIN_BLOCK_KW = 16 _DECODE_MARLIN_WARPS = 4 +# Deep-K variant: at K > 2048 (qwen4_exp gate_up, K=2560) a narrower N tile with the whole +# K strip in one program iteration measures ~13% faster (18.6 vs 21.0us); short-K shapes +# regress under it, so the split is by K, not by gemm position. +_DECODE_MARLIN_DEEPK_BLOCK_N = 8 +_DECODE_MARLIN_DEEPK_BLOCK_KW = 128 +_DECODE_MARLIN_DEEPK_THRESHOLD = 2048 + + +def _deepk_block_kw() -> int: + """Return the opt-in deep-K K-word tile for isolated AMD experiments. + + The production default remains 128, which is the currently qualified + launch shape. A small allow-list prevents an arbitrary environment value + from changing the CUDA-graph-compatible kernel geometry. This hook exists + so a candidate such as the numerically matched 8x16 screen can be tested + through the real serving path without editing source between runs. + """ + raw = os.environ.get("FREETOKEN_NVFP4_DEEPK_BLOCK_KW", "") + if not raw: + return _DECODE_MARLIN_DEEPK_BLOCK_KW + try: + value = int(raw) + except ValueError as exc: + raise ValueError( + "FREETOKEN_NVFP4_DEEPK_BLOCK_KW must be one of 16, 32, 64, or 128" + ) from exc + if value not in (16, 32, 64, 128): + raise ValueError( + "FREETOKEN_NVFP4_DEEPK_BLOCK_KW must be one of 16, 32, 64, or 128" + ) + return value def _tl_dtype(dt: torch.dtype): @@ -129,7 +166,10 @@ def _decode_gemm_marlin( packed_i32 = packed.view(torch.int32) # [S, N, K // 8] scale = e4m3_kernel_view(scale) total_routes = M * top_k - grid = (total_routes, triton.cdiv(N, _DECODE_MARLIN_BLOCK_N)) + deep_k = K > _DECODE_MARLIN_DEEPK_THRESHOLD + block_n = _DECODE_MARLIN_DEEPK_BLOCK_N if deep_k else _DECODE_MARLIN_BLOCK_N + block_kw = _deepk_block_kw() if deep_k else _DECODE_MARLIN_BLOCK_KW + grid = (total_routes, triton.cdiv(N, block_n)) _decode_nvfp4_marlin_kernel[grid]( a, packed_i32, scale, glob, c, topk_weights, topk_ids, _e2m1_lut(a.device.index), @@ -141,8 +181,8 @@ def _decode_gemm_marlin( c.stride(0), c.stride(1), c.stride(2), topk_weights.stride(0), topk_weights.stride(1), topk_ids.stride(0), topk_ids.stride(1), - BLOCK_SIZE_N=_DECODE_MARLIN_BLOCK_N, - BLOCK_SIZE_KW=_DECODE_MARLIN_BLOCK_KW, + BLOCK_SIZE_N=block_n, + BLOCK_SIZE_KW=block_kw, TOP_K=top_k, A_ROW_IS_ROUTE=a_row_is_route, MUL_ROUTED_WEIGHT=mul_routed_weight, @@ -314,6 +354,29 @@ def fused_experts_nvfp4( """Prefill inline-NVFP4 MoE. ``topk_ids`` index rows of the bank tensors in ``[0, num_experts)``: full-layer banks with position == expert id (the materialized ``[:E]`` slot view or the overlap double buffer), raw ids.""" + if torch.version.hip is not None: + # The grouped prefill kernel below currently trips an HSA memory-aperture + # violation on gfx1151. The serial Triton kernel is already FreeToken's + # native inline-dequant implementation and accepts an arbitrary M, so it + # preserves HIP GPU inference and model results without materializing BF16 + # experts. It is intentionally slower for prompt prefill than the CUDA + # grouped kernel, but is safe until the grouped launch is ROCm-qualified. + return fused_experts_decode_nvfp4_serial( + hidden_states, + gate_up_packed, + gate_up_scale, + gate_up_global, + down_packed, + down_scale, + down_global, + topk_weights, + topk_ids, + activation, + apply_router_weight_on_input, + act_alpha, + act_limit, + ) + M, H = hidden_states.shape top_k = topk_ids.shape[1] two_i = gate_up_packed.shape[1] diff --git a/python/freetoken/moe/fused_q4_k_q5_k.py b/python/freetoken/moe/fused_q4_k_q5_k.py new file mode 100644 index 0000000000..dfd209696d --- /dev/null +++ b/python/freetoken/moe/fused_q4_k_q5_k.py @@ -0,0 +1,54 @@ +"""Mixed Q4_K/Q5_K GGUF routed-expert execution for Qwen3.6 MoE checkpoints. + +The GGUF model recipe names this combination ``Q4_K_M``, but its tensor table +stores the gate and up expert projections as Q4_K and the down projection as +Q5_K. The borrowed HIP GGML kernels dispatch one quant type per matrix, so +this module intentionally launches one packed Q4_K MoE GEMV followed by one +packed Q5_K MoE GEMV. Neither weight is dequantized to a persistent bf16 copy. +""" + +from __future__ import annotations + +import torch + +from freetoken.layers.activation import silu_and_mul +from freetoken.models.gguf.dequant import GGML_Q4_K, GGML_Q5_K + + +def fused_experts_gguf_q4_k_q5_k( + hidden_states: torch.Tensor, + gate_up_q4_k: torch.Tensor, + down_q5_k: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: str, +) -> torch.Tensor: + """Run packed Q4_K gate/up then packed Q5_K down over routed experts. + + ``topk_ids`` already name the materialized GGUF expert-cache slots. Qwen + uses SwiGLU, so only ``silu`` is accepted here. Explicit validation prevents + a future model family from silently receiving Qwen's activation semantics. + """ + if activation != "silu": + raise ValueError( + "Qwen mixed GGUF experts require the checkpoint's silu SwiGLU activation, " + f"got {activation!r}" + ) + from freetoken.kernel.gguf import ggml_moe_a8_vec + + tokens = hidden_states.shape[0] + top_k = topk_ids.shape[1] + fused_width = gate_up_q4_k.shape[1] + hidden_size = down_q5_k.shape[1] + gate_up = ggml_moe_a8_vec( + hidden_states, gate_up_q4_k, topk_ids, top_k, int(GGML_Q4_K), fused_width, tokens + ) + intermediate = silu_and_mul(gate_up) + output = ggml_moe_a8_vec( + intermediate, down_q5_k, topk_ids, 1, int(GGML_Q5_K), hidden_size, tokens * top_k + ) + output = output.reshape(tokens, top_k, hidden_size) + return (output * topk_weights.reshape(tokens, top_k, 1).to(output.dtype)).sum(dim=1) + + +__all__ = ["fused_experts_gguf_q4_k_q5_k"] diff --git a/python/freetoken/moe/fused_q4_k_q6_k.py b/python/freetoken/moe/fused_q4_k_q6_k.py new file mode 100644 index 0000000000..84c79c6f34 --- /dev/null +++ b/python/freetoken/moe/fused_q4_k_q6_k.py @@ -0,0 +1,54 @@ +"""Exact Q4_K/Q6_K routed-expert execution for exceptional Qwen GGUF layers. + +Qwen3.6 Q4_K_M stores almost every routed down projection as Q5_K, but a few +late layers use Q6_K. The primary Q4_K/Q5_K cache cannot store both row sizes, +so this kernel accepts Q4_K gate/up rows from that cache and Q6_K down rows from +the small auxiliary cache. Both id tensors address the same routed experts, +but they intentionally name slots in their respective caches. +""" + +from __future__ import annotations + +import torch + +from freetoken.layers.activation import silu_and_mul +from freetoken.models.gguf.dequant import GGML_Q4_K, GGML_Q6_K + + +def fused_experts_gguf_q4_k_q6_k( + hidden_states: torch.Tensor, + gate_up_q4_k: torch.Tensor, + down_q6_k: torch.Tensor, + topk_weights: torch.Tensor, + gate_up_ids: torch.Tensor, + down_ids: torch.Tensor, + activation: str, +) -> torch.Tensor: + """Run Q4_K gate/up and Q6_K down using their independent cache slots.""" + if activation != "silu": + raise ValueError( + "Qwen mixed GGUF experts require the checkpoint's silu SwiGLU activation, " + f"got {activation!r}" + ) + if gate_up_ids.shape != down_ids.shape: + raise ValueError("Q4_K and Q6_K routed id tensors must have the same shape") + from freetoken.kernel.gguf import ggml_moe_a8_vec + + tokens = hidden_states.shape[0] + top_k = gate_up_ids.shape[1] + fused_width = gate_up_q4_k.shape[1] + hidden_size = down_q6_k.shape[1] + gate_up = ggml_moe_a8_vec( + hidden_states, gate_up_q4_k, gate_up_ids, top_k, + int(GGML_Q4_K), fused_width, tokens, + ) + intermediate = silu_and_mul(gate_up) + output = ggml_moe_a8_vec( + intermediate, down_q6_k, down_ids, 1, + int(GGML_Q6_K), hidden_size, tokens * top_k, + ) + output = output.reshape(tokens, top_k, hidden_size) + return (output * topk_weights.reshape(tokens, top_k, 1).to(output.dtype)).sum(dim=1) + + +__all__ = ["fused_experts_gguf_q4_k_q6_k"] diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.py index d7af348ab4..436f52d2d9 100644 --- a/python/freetoken/moe/host_banks.py +++ b/python/freetoken/moe/host_banks.py @@ -410,6 +410,67 @@ def rd(o): return size +def _preadv_all(fd: int, dst: memoryview, offset: int, need: int) -> None: + """preadv into ``dst`` until ``need`` bytes have landed; O_DIRECT may return a short count.""" + done = 0 + while done < need: + if done % _BLK: # a continuation read has to stay block-aligned on both sides + raise OSError(f"unaligned short O_DIRECT read: {done} of {need} bytes at {offset}") + got = os.preadv(fd, [dst[done:]], offset + done) + if got <= 0: + raise OSError(f"short O_DIRECT read: {done} of {need} bytes at {offset}") + done += got + + +def read_range_into(buf: memoryview | mmap.mmap, path: str, *, file_offset: int, nbytes: int, + dest_offset: int = 0, workers: int = 8, chunk: int = _DEFAULT_CHUNK, + drop_cache: bool = True) -> int: + """Chunked multi-threaded O_DIRECT read of ``path[file_offset : file_offset + nbytes]`` into ``buf`` at ``dest_offset``. Returns ``nbytes``. + + Byte-range counterpart of :func:`read_file_into`, for one tensor inside a shard. O_DIRECT needs the file offset AND the destination address block-aligned at the same time, which only holds when the two share their offset mod 4096 -- a safetensors data offset practically never lines up with the tensor's slot in the bank. Chunks that do line up DMA straight into ``buf``; the rest DMA into a page-aligned bounce (source window rounded out to whole blocks) and are copied into place, which also covers the unaligned head and tail. + """ + mv = (buf if isinstance(buf, memoryview) else memoryview(buf)).cast("B") + if dest_offset + nbytes > len(mv): + raise ValueError(f"destination holds {len(mv)} bytes, need {dest_offset + nbytes}") + base = ctypes.addressof(ctypes.c_char.from_buffer(mv)) + if drop_cache: + try: + fd0 = os.open(path, os.O_RDONLY) + os.posix_fadvise(fd0, file_offset, nbytes, os.POSIX_FADV_DONTNEED) + os.close(fd0) + except OSError: + pass + fd = os.open(path, os.O_RDONLY | os.O_DIRECT) + scratch = threading.local() + + def rd(i: int) -> None: + n = min(chunk, nbytes - i) + src, dst = file_offset + i, dest_offset + i + if src % _BLK == 0 and (base + dst) % _BLK == 0 and n % _BLK == 0: + _preadv_all(fd, mv[dst:dst + n], src, n) + return + head = src % _BLK + span = ((head + n + _BLK - 1) // _BLK) * _BLK + bounce = getattr(scratch, "buf", None) + if bounce is None or len(bounce) < span: + bounce = scratch.buf = mmap.mmap(-1, span) # anonymous mmaps are page-aligned + bmv = memoryview(bounce) + _preadv_all(fd, bmv[:span], src - head, head + n) + mv[dst:dst + n] = bmv[head:head + n] + + try: + offs = list(range(0, nbytes, chunk)) + if len(offs) <= 1: + for o in offs: + rd(o) + else: + with ThreadPoolExecutor(workers) as ex: + list(ex.map(rd, offs)) + finally: + os.close(fd) + return nbytes + + __all__ = [ "HostBank", "HostResidency", @@ -420,5 +481,6 @@ def rd(o): "born_pinned_default", "pin_banks", "read_file_into", + "read_range_into", "requested_residency", ] diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.py index 6ee764061b..f04937bf9e 100644 --- a/python/freetoken/moe/offload_cache.py +++ b/python/freetoken/moe/offload_cache.py @@ -45,6 +45,13 @@ # native GGUF Q4_0 experts: packed block bytes per output row, dequantized inside # the borrowed ggml MoE kernels. gate_up [L*E, 2I, H//32*18], down [L*E, H, I//32*18]. "q4_0": ("gate_up", "down"), + # Qwen3.6-35B-A3B Q4_K_M GGUF: gate/up rows are Q4_K while down rows + # are Q5_K. Both stay byte-exact and the two GGML kernels are called + # separately by the mixed-format fused MoE path. + "q4_k_q5_k": ("gate_up", "down"), + # Qwen Q4_K_M's three late Q6_K down projections. This intentionally has + # one bank and is used only by a small auxiliary cache. + "q6_k_down": ("down",), # native ModelOpt rows for the Triton inline-dequant kernels: packed e2m1 codes + # fp8-e4m3 per-16 block scales + per-output-row fp16 globals (w1/w3 carry distinct # globals, and folding them into the e4m3 block scales would underflow) @@ -77,12 +84,21 @@ "ds_fp4": ("gate_up_packed", "gate_up_scale", "down_packed", "down_scale"), } +# lives in kernel/aot_models.py: the AOT row table shares it and must stay importable in the torch-only kernel-cache build env, which cannot import freetoken.moe +from freetoken.kernel.aot_models import fp8_block_scale_pad + + # bytes per (expert, layer) as f(hidden, moe_intermediate), from the bank shapes above; keep in sync with _BANK_SCHEMAS # keyed by the config-time format tag (expert_quant / moe_weight_format), not quant_format: "mxfp4" sizes the mxfp4_triton banks, "nvfp4" also covers its repacked variants _BANK_BYTES_PER_EXPERT = { "bf16": lambda H, I: 3 * I * H * 2, - "fp8_block": lambda H, I: 3 * I * H + ((2 * I // 128) * (H // 128) + (H // 128) * (I // 128)) * 2, + "fp8_block": lambda H, I: 3 * I * H + ( + (2 * I // 128) * fp8_block_scale_pad(2 * I // 128, H // 128) + + (H // 128) * fp8_block_scale_pad(H // 128, I // 128) + ) * 2, "q4_0": lambda H, I: 2 * I * (H // 32) * 18 + H * (I // 32) * 18, + "q4_k_q5_k": lambda H, I: 2 * I * (H // 256) * 144 + H * (I // 256) * 176, + "q6_k_down": lambda H, I: H * (I // 256) * 210, "nvfp4": lambda H, I: 2 * I * (H // 2 + H // 16 + 2) + H * (I // 2 + I // 16 + 2), "mxfp4": lambda H, I: 2 * I * (H // 2 + H // 32 + 2) + H * (I // 2 + I // 32 + 2), "ds_fp4": lambda H, I: 2 * I * (H // 2 + H // 32) + H * (I // 2 + I // 32), @@ -340,6 +356,19 @@ def set_bank_sources( self._init_prefill_overlap_buffers() def _build_copy_plan(self) -> None: + self._build_fused_copy_plan() + if self._copy_fused_ok or self.device.type != "cuda" or not self.banks: + return + for name in self.bank_schema: + cache = self.bank_caches[name] + feat = math.prod(cache.shape[1:]) * cache.element_size() + if feat % 128: + raise RuntimeError( + f"MoE bank {name!r} rows are {feat} bytes (not a multiple of 128): " + f"only the fused multi-bank copy can move them, but it is disabled" + ) + + def _build_fused_copy_plan(self) -> None: """Precompute the fused multi-bank copy descriptor (base addrs + per-row bytes). Built once here (and on :meth:`rebuild`, which reallocates the slot caches); diff --git a/python/freetoken/scheduler/cache.py b/python/freetoken/scheduler/cache.py index 9be235b255..44adde42f9 100644 --- a/python/freetoken/scheduler/cache.py +++ b/python/freetoken/scheduler/cache.py @@ -66,6 +66,13 @@ def __init__(self, num_pages: int, page_size: int, page_table: torch.Tensor, typ supports_runtime_rebuild = True prefill_chunk_budget = None # generic shared page pool: no per-model prefill chunk cap + @property + def prefill_chunk_align(self) -> int: + """Granularity a non-final prefill chunk should end on. A hybrid snapshot is donated only + at a page-aligned boundary, so at page_size>1 one unaligned chunk end costs every reuse + point for the rest of the prompt. 1 (no-op) everywhere else.""" + return self.page_size if self.is_hybrid else 1 + def page_usage(self) -> tuple[int, int]: """(used_pages, total_pages): allocated, non-evictable pages over the pool total (active requests + protected prefix; evictable prefix-cache pages are excluded).""" diff --git a/python/freetoken/scheduler/prefill.py b/python/freetoken/scheduler/prefill.py index be84874c71..f5bc8f7a31 100644 --- a/python/freetoken/scheduler/prefill.py +++ b/python/freetoken/scheduler/prefill.py @@ -156,6 +156,13 @@ def _add_one_req( self.reserved_swa += ( div_ceil(cached_len + chunk_size, ps) - div_ceil(cached_len, ps) ) * ps + align = self.cache_manager.prefill_chunk_align + if align > 1 and 0 < chunk_size < remain_len: + # An unaligned chunk end is correct, it just loses this prompt's snapshot boundaries -- + # so keep it when the leftover budget cannot fill one whole unit instead of stalling + # the request until it gets a bigger turn. + aligned = align_down(cached_len + chunk_size, align) - cached_len + chunk_size = aligned if aligned > 0 else chunk_size is_chunked = chunk_size < remain_len CLS = ChunkedReq if is_chunked else Req self.token_budget -= chunk_size diff --git a/python/freetoken/scheduler/scheduler.py b/python/freetoken/scheduler/scheduler.py index 48923e3b0a..bba93fd501 100644 --- a/python/freetoken/scheduler/scheduler.py +++ b/python/freetoken/scheduler/scheduler.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from typing import TYPE_CHECKING, List, NamedTuple, NoReturn, Set, Tuple, TypeAlias import torch @@ -13,6 +14,8 @@ BatchBackendMsg, CacheRebuildBackendMsg, CacheRebuildResultMsg, + CacheStatsBackendMsg, + CacheStatsResultMsg, DetokenizeMsg, ErrorReplyMsg, ExitMsg, @@ -515,6 +518,55 @@ def _process_one_msg(self, msg: BaseBackendMsg) -> None: ] ) return + # Older and text-only tokenizer messages legitimately omit the two + # multimodal attributes altogether. Treat an absent attribute the + # same as ``None`` so a normal completion can never crash the + # scheduler before image handling is even considered. + mm_pixel_values = getattr(msg, "mm_pixel_values", None) + mm_image_position_ids = getattr(msg, "mm_image_position_ids", None) + if mm_pixel_values is not None or mm_image_position_ids is not None: + if mm_pixel_values is None or mm_image_position_ids is None: + self.send_result([ErrorReplyMsg(uid=msg.uid, error="incomplete image tensors")]) + return + model = self.engine.model + if not hasattr(model, "encode_images"): + self.send_result( + [ErrorReplyMsg(uid=msg.uid, error="this model does not support image inputs")] + ) + return + try: + # The engine owns the ROCm context. Keep image encoding here, + # not in the tokenizer process, so the vision weights and + # features stay resident on the one serving device. + msg.mm_embeds = model.encode_images( + mm_pixel_values.to(self.device), + mm_image_position_ids.to(self.device), + ) + # This diagnostic sits at the scheduler boundary, after the + # actual model wrapper returns image soft-token embeddings. + # It is intentionally opt-in because copying GPU values to + # the host synchronizes the request and would distort normal + # vision latency measurements. + if os.environ.get("FREETOKEN_GEMMA4_VISION_DEBUG") == "1": + values = msg.mm_embeds.detach().float() + sample = values.reshape(-1)[:16].cpu().tolist() + logger.info_rank0( + "Gemma4 scheduler vision debug: model=%s shape=%s finite=%s " + "mean=%.8f std=%.8f min=%.8f max=%.8f sum=%.8f first16=%s", + type(model).__name__, + tuple(values.shape), + bool(torch.isfinite(values).all().item()), + float(values.mean().item()), + float(values.std(unbiased=False).item()), + float(values.min().item()), + float(values.max().item()), + float(values.sum().item()), + ",".join(f"{value:.8f}" for value in sample), + ) + except Exception as exc: # noqa: BLE001 - return a request error, not a dead worker + logger.warning_rank0("image encoding failed for request %d: %r", msg.uid, exc) + self.send_result([ErrorReplyMsg(uid=msg.uid, error=f"could not encode image: {exc}")]) + return if msg.sampling_params.max_tokens > max_output_len: msg.sampling_params.max_tokens = max_output_len logger.warning_rank0( @@ -578,6 +630,19 @@ def _process_one_msg(self, msg: BaseBackendMsg) -> None: self._reply_rebuild(msg.request_id, "busy") else: self._pending_rebuild = msg + elif isinstance(msg, CacheStatsBackendMsg): + # The counter tensors are read only here. Their host transfer is a + # one-off synchronization requested explicitly by the diagnostic API, + # never a per-token cost on the serving path. + cache = self.engine.moe_offload_cache + stats = {"available": False} + if cache is not None: + stats = { + "available": True, + "summary": cache.decode_miss_stats(), + "per_layer": cache.decode_miss_stats_per_layer(), + } + self.send_result([CacheStatsResultMsg(request_id=msg.request_id, stats=stats)]) else: logger.error(f"Unknown message type: {type(msg)}") raise NotImplementedError diff --git a/python/freetoken/server/api_models.py b/python/freetoken/server/api_models.py index ffd7172802..0872a9bc3e 100644 --- a/python/freetoken/server/api_models.py +++ b/python/freetoken/server/api_models.py @@ -120,6 +120,10 @@ class CompletionRequest(BaseModel): suffix: str | None = None logit_bias: dict[str, float] | None = None response_format: dict[str, Any] | None = None + # Nonstandard but deliberately explicit: false tells FreeToken that this + # raw prompt already includes every required special token. The default + # remains true for OpenAI-style raw completion compatibility. + add_special_tokens: bool = True @model_validator(mode="after") def _sync_max_completion_tokens(self) -> "CompletionRequest": diff --git a/python/freetoken/server/api_server.py b/python/freetoken/server/api_server.py index 3e2acc8542..16322e32f7 100644 --- a/python/freetoken/server/api_server.py +++ b/python/freetoken/server/api_server.py @@ -24,6 +24,8 @@ BatchFrontendMsg, CacheRebuildMsg, CacheRebuildReply, + CacheStatsMsg, + CacheStatsReply, TokenizeMsg, UserReply, ) @@ -143,6 +145,9 @@ class FrontendManager: # Runtime cache-rebuild control plane (correlated by uuid request_id, separate from # the int-uid generation ack machinery). rebuild_futures: Dict[str, asyncio.Future] = field(default_factory=dict) + # Read-only cache-statistics requests use the same UUID correlation pattern + # but never engage the rebuild maintenance gate or modify a cache allocation. + cache_stats_futures: Dict[str, asyncio.Future] = field(default_factory=dict) # Lifecycle gate. Starts "loading" (uvicorn binds before weights finish; the three # API adapters 503 until this flips) -> "serving" once all workers ack ready -> # "rebuilding"/"failed" for runtime cache rebuilds. @@ -240,12 +245,29 @@ def new_user(self) -> int: self.stats.on_new_user(uid) return uid + async def cache_stats(self, timeout: float = 30.0) -> Dict[str, Any]: + """Return a read-only backend cache-statistics snapshot through worker IPC.""" + + request_id = str(uuid.uuid4()) + future = asyncio.get_running_loop().create_future() + self.cache_stats_futures[request_id] = future + try: + await self.send_one(CacheStatsMsg(request_id=request_id)) + return await asyncio.wait_for(future, timeout=timeout) + finally: + self.cache_stats_futures.pop(request_id, None) + async def listen(self): while True: msg = await self.recv_tokenizer.get() if isinstance(msg, CacheRebuildReply): self._resolve_rebuild(msg) continue + if isinstance(msg, CacheStatsReply): + future = self.cache_stats_futures.get(msg.request_id) + if future is not None and not future.done(): + future.set_result(msg.stats) + continue for msg in _unwrap_msg(msg): # Global accounting follows actual admitted/sampled work even after the HTTP # client disconnects and abort_user removes its ack queue. Delivery to a live @@ -817,6 +839,19 @@ async def cache_status(): } +@app.get("/v1/cache/stats") +async def cache_stats(): + """Read accumulated MoE cache hit and miss counters without modifying the cache.""" + + state = get_global_state() + if state.maintenance_state != "serving": + return JSONResponse({"error": "server is not serving"}, status_code=503) + try: + return await state.cache_stats() + except TimeoutError: + return JSONResponse({"error": "backend cache-statistics request timed out"}, status_code=504) + + @app.post("/generate") async def generate(req: GenerateRequest, request: Request): logger.debug("Received generate request %s", req) diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py index a71b681937..eb1bd1e906 100644 --- a/python/freetoken/server/args.py +++ b/python/freetoken/server/args.py @@ -147,6 +147,8 @@ def _infer_tool_call_parser(model_path: str) -> str: return "muse_glimmer" if "gemma4" in marker: return "gemma4" + if "qwen4_exp" in marker or "qwen4exp" in marker or "qwen3.8-flash" in marker: + return "qwen3_coder" if ( "qwen3_5" in marker or "qwen3.5" in marker @@ -188,6 +190,8 @@ def _infer_reasoning_parser(model_path: str) -> str | None: tag in marker for tag in ("v4", "deepseek_v4", "v3.2", "v32") ): return "deepseekv32" + if "qwen4_exp" in marker or "qwen4exp" in marker or "qwen3.8-flash" in marker: + return "qwen3" if "qwen3" in marker or "qwen3.5" in marker or "qwen3_5" in marker: return "qwen3" if "glm" in marker: @@ -473,6 +477,16 @@ def _infer_reasoning_parser(model_path: str) -> str | None: ), ) + parser.add_argument( + "--ple-backend", + default=ServerArgs.ple_backend, + choices=["pinned", "disk"], + help=( + "Where a PLE n-gram table lives. 'disk' (default) reads rows straight from the " + "checkpoint files; 'pinned' preloads the whole table into page-locked host RAM." + ), + ) + parser.add_argument( "--nvfp4-backend", default=ServerArgs.nvfp4_backend, @@ -533,6 +547,16 @@ def _infer_reasoning_parser(model_path: str) -> str | None: help="The unified MoE cache eviction policy.", ) + parser.add_argument( + "--moe-collect-stats", + action="store_true", + default=ServerArgs.moe_collect_stats, + help=( + "Accumulate read-only decode expert-cache hit and miss counters on the device. " + "The counters are intended for explicit diagnostic snapshots, not per-request logs." + ), + ) + parser.add_argument( "--moe-cpu-threads", type=int, diff --git a/python/freetoken/server/control_api.py b/python/freetoken/server/control_api.py index 7158e4e5fa..6332ab97dc 100644 --- a/python/freetoken/server/control_api.py +++ b/python/freetoken/server/control_api.py @@ -58,6 +58,22 @@ def register_control_routes( async def health(): return build_health(get_state(), app.version) + # What: register GET /ready on the application router; why: clients reach ready's handler only through this method-and-path binding. + @app.get("/ready") + # What: define ready around the current object state; why: the registered API client call ready for ready and rely on this exact input and result contract. + async def ready(): + """HTTP readiness for supervisors that cannot inspect health JSON.""" + # What: document http readiness for supervisors that cannot in the ready docstring; why: introspection and maintainers read this exact docstring fragment to understand ready behavior without executing it. + # What: import jsonresponse for ready using fastapi and responses and jsonresponse; why: ready uses jsonresponse, making that imported dependency available to its named operation. + from fastapi.responses import JSONResponse + + # What: compute doc from build health and version and get state and app; why: accepting doc get status ok and doc get later reads doc, so ready must retain the computed value under that name. + doc = build_health(get_state(), app.version) + # What: compute accepting from get and doc and ok and serving and status; why: return jsonresponse status code if accepting else later reads accepting, so ready must retain the computed value under that name. + accepting = doc.get("status") == "ok" and doc.get("maintenance") == "serving" + # What: return HTTP 200 when accepting and 503 otherwise; why: supervisors use this status and ready boolean to decide whether the daemon control plane may receive traffic. + return JSONResponse(status_code=200 if accepting else 503, content=doc) + from . import request_ring @app.get("/v1/requests") diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.py index be05d908a0..f1cb1671e6 100644 --- a/python/freetoken/server/generation.py +++ b/python/freetoken/server/generation.py @@ -139,6 +139,7 @@ class GenSpec: chat_template_kwargs: dict[str, Any] = field(default_factory=dict) template_tools: list[dict[str, Any]] | None = None # tools the model sees (TokenizeMsg.tools) parser_tools: list[dict[str, Any]] | None = None # tools for FunctionCallParser; None disables parsing + image_urls: list[Any] = field(default_factory=list) # OpenAI image_url values in marker order @property def parse_tools(self) -> bool: @@ -236,6 +237,10 @@ def _flatten_text_parts(parts: list[Any]) -> str: ptype = part.get("type") if isinstance(part, dict) else None if ptype == "text": texts.append((part.get("text") if isinstance(part, dict) else None) or "") + elif ptype == "image_url": + # Exact image length is unavailable until decoding and resizing in + # the tokenizer worker, so leave a private replacement marker here. + texts.append("<|freetoken-image|>") else: raise ValueError(f"Unsupported content part type for text-only server: {ptype}") return "".join(texts) @@ -269,6 +274,7 @@ async def submit_generation(spec: GenSpec, state: Any) -> int: sampling_params=spec.sampling_params, chat_template_kwargs=spec.chat_template_kwargs, tools=spec.template_tools, + image_urls=spec.image_urls or None, ) ) return uid @@ -325,13 +331,20 @@ async def prerender_error(spec: GenSpec, state: Any) -> GenerationError | None: sampling_params=SamplingParams(), chat_template_kwargs=spec.chat_template_kwargs, tools=spec.template_tools, + image_urls=spec.image_urls or None, ) try: manager = await asyncio.to_thread(build) except Exception: # noqa: BLE001 -- server fault, not this request's problem return None try: - await asyncio.to_thread(manager.render_prompt, msg) + # Match the tokenizer worker's sequence exactly: render the chat + # template first, then expand image markers once into verified Gemma + # placeholders and CPU tensors. The worker itself performs this after + # render_prompt, so folding expansion into render_prompt would make + # actual image requests expand twice. + prompt = await asyncio.to_thread(manager.render_prompt, msg) + await asyncio.to_thread(manager._expand_gemma4_images, msg, prompt) except Exception as exc: # noqa: BLE001 -- mirror the worker's classification return GenerationError(f"could not encode request: {exc}") return None diff --git a/python/freetoken/server/openai_api.py b/python/freetoken/server/openai_api.py index b4becd2631..22c6a671c8 100644 --- a/python/freetoken/server/openai_api.py +++ b/python/freetoken/server/openai_api.py @@ -66,8 +66,9 @@ def chat_request_to_genspec( thinking_type = _thinking_type(req) if req.reasoning_effort or thinking_type: ctk = effort_toggle_kwargs(req.reasoning_effort, ctk, thinking_type=thinking_type) + raw_messages = [m.model_dump(exclude_none=True) for m in req.messages] return GenSpec( - messages=render_messages([m.model_dump(exclude_none=True) for m in req.messages]), + messages=render_messages(raw_messages), sampling_params=resolve_sampling( temperature=req.temperature, top_k=req.top_k, @@ -80,9 +81,23 @@ def chat_request_to_genspec( chat_template_kwargs=ctk, template_tools=_tools_for_template(req), parser_tools=(_all_tool_dicts(req.tools) if _should_parse_tools(req) else None), + image_urls=_openai_image_urls(raw_messages), ) +def _openai_image_urls(messages: list[dict[str, Any]]) -> list[Any]: + """Extract image_url values in the same order render_messages emits markers.""" + values: list[Any] = [] + for message in messages: + content = message.get("content") + if not isinstance(content, list): + continue + for part in content: + if isinstance(part, dict) and part.get("type") == "image_url": + values.append(part.get("image_url")) + return values + + def _all_tool_dicts(tools) -> list[dict[str, Any]]: return [t.model_dump(exclude_none=True) for t in (tools or [])] @@ -397,7 +412,12 @@ async def handle_completion( return create_error_response("Streaming completions only support a single text prompt") uid = state.new_user() await state.send_one( - TokenizeMsg(uid=uid, text=prompts[0], sampling_params=_resolve_sampling(req, model_sampling)) + TokenizeMsg( + uid=uid, + text=prompts[0], + sampling_params=_resolve_sampling(req, model_sampling), + add_special_tokens=req.add_special_tokens, + ) ) chunks = stream_completion_chunks(uid, req, state) if request is not None: @@ -410,7 +430,14 @@ async def handle_completion( cached_tokens = 0 for index, prompt in enumerate(prompts): uid = state.new_user() - await state.send_one(TokenizeMsg(uid=uid, text=prompt, sampling_params=_resolve_sampling(req, model_sampling))) + await state.send_one( + TokenizeMsg( + uid=uid, + text=prompt, + sampling_params=_resolve_sampling(req, model_sampling), + add_special_tokens=req.add_special_tokens, + ) + ) text = "" finish_reason = "stop" async for ack in state.wait_for_ack(uid): diff --git a/python/freetoken/tokenizer/gemma4_image.py b/python/freetoken/tokenizer/gemma4_image.py new file mode 100644 index 0000000000..59fabdd75c --- /dev/null +++ b/python/freetoken/tokenizer/gemma4_image.py @@ -0,0 +1,147 @@ +"""Safe, deterministic Gemma4 image preprocessing for the online API. + +The frontend accepts OpenAI's ``image_url`` data-URL representation and turns it +into CPU tensors that can safely cross the tokenizer-to-scheduler process +boundary. Remote URL fetching is intentionally not implemented here: doing so +inside a LAN inference service would introduce an SSRF-capable network client. +The caller gets a precise error and can provide the same bytes as a data URL. +""" + +from __future__ import annotations + +import base64 +import binascii +import io +import math +from dataclasses import dataclass + +import numpy as np +import torch +from PIL import Image, UnidentifiedImageError + + +_MAX_IMAGE_BYTES = 20 * 1024 * 1024 +_MAX_IMAGE_PIXELS = 16_000_000 +_PATCH_SIZE = 16 +_POOLING_KERNEL_SIZE = 3 +_MAX_SOFT_TOKENS = 280 + + +@dataclass(frozen=True) +class Gemma4ImageInputs: + """One image in the exact tensor layout consumed by ``Gemma4VisionModel``.""" + + pixel_values: torch.Tensor + image_position_ids: torch.Tensor + soft_token_count: int + + +def decode_openai_image_data_url(value: object) -> Image.Image: + """Decode one OpenAI ``image_url`` value into a verified RGB Pillow image. + + OpenAI clients commonly send either the URL string directly or an object with + a ``url`` member. Only base64 ``data:image/*`` URLs are accepted. The + explicit byte and pixel limits avoid request-driven memory exhaustion before + the image reaches the GPU-serving process. + """ + url = value.get("url") if isinstance(value, dict) else value + if not isinstance(url, str): + raise ValueError("image_url must be a data:image URL string or an object with a url field") + if not url.startswith("data:image/"): + raise ValueError( + "only data:image URLs are supported for local Gemma4 vision; " + "download remote images client-side and send their bytes as a data URL" + ) + header, separator, encoded = url.partition(",") + if not separator or ";base64" not in header.lower(): + raise ValueError("image_url must use base64 data:image/...;base64,... encoding") + try: + raw = base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError) as exc: + raise ValueError("image_url contains invalid base64 image data") from exc + if not raw or len(raw) > _MAX_IMAGE_BYTES: + raise ValueError(f"image_url must contain 1 to {_MAX_IMAGE_BYTES} bytes") + try: + with Image.open(io.BytesIO(raw)) as opened: + opened.verify() + with Image.open(io.BytesIO(raw)) as opened: + if opened.width * opened.height > _MAX_IMAGE_PIXELS: + raise ValueError(f"image_url exceeds {_MAX_IMAGE_PIXELS} decoded pixels") + return opened.convert("RGB") + except UnidentifiedImageError as exc: + raise ValueError("image_url does not contain a recognized image") from exc + + +def gemma4_image_inputs(image: Image.Image) -> Gemma4ImageInputs: + """Resize, patchify, and position one RGB image using Gemma4's public contract. + + The resize equation matches Gemma4's processor: at most 280 soft tokens + after 3-by-3 pooling, dimensions aligned to ``16 * 3`` pixels, and a + lower bound of one pooled patch. Every image is then padded to the fixed + 2,520-patch vision sequence used by the official processor. Patch pixels + are channel-planar RGB values in [0, 1], exactly matching the projector's + convolution-kernel layout before the model applies ``2 * (x - 0.5)``. + """ + if image.mode != "RGB": + image = image.convert("RGB") + source_width, source_height = image.size + max_patches = _MAX_SOFT_TOKENS * _POOLING_KERNEL_SIZE**2 + source_patches = (source_height / _PATCH_SIZE) * (source_width / _PATCH_SIZE) + scale = math.sqrt(max_patches / source_patches) + unit = _PATCH_SIZE * _POOLING_KERNEL_SIZE + target_height = max(unit, int(math.floor(source_height * scale / unit)) * unit) + target_width = max(unit, int(math.floor(source_width * scale / unit)) * unit) + # Count aligned patches before resizing because the one-unit floor can inflate extreme aspect ratios. + target_patch_count = (target_height // _PATCH_SIZE) * (target_width // _PATCH_SIZE) + # Reject shapes beyond the fixed vision budget before Pillow allocates an unexpectedly large image. + if target_patch_count > max_patches: + # Report the unsupported geometry without exposing image content or attempting the dangerous allocation. + raise ValueError("image aspect ratio exceeds Gemma4's fixed patch budget") + # Resize only after the target is proven to fit the model's fixed patch and memory envelope. + resized = image.resize((target_width, target_height), Image.Resampling.BICUBIC) + + # HWC RGB -> [grid_y, grid_x, channels, patch_y, patch_x] -> flattened. + # The sibling mmproj stores ``v.patch_embd.weight`` as a conventional + # convolution kernel: [output, channel, patch_y, patch_x]. Its input vector + # therefore contains one complete red patch, then green, then blue. An + # RGB-interleaved vector is shape-compatible but produces wrong vision + # features for otherwise simple, deterministic color controls. + pixels = np.asarray(resized, dtype=np.float32) / 255.0 + grid_y, grid_x = target_height // _PATCH_SIZE, target_width // _PATCH_SIZE + patches = ( + pixels.transpose(2, 0, 1) + .reshape(3, grid_y, _PATCH_SIZE, grid_x, _PATCH_SIZE) + .transpose(1, 3, 0, 2, 4) + .reshape(grid_y * grid_x, 3 * _PATCH_SIZE**2) + ) + # The vision implementation treats coordinate 0 as x and coordinate 1 as + # y when assigning pooled spatial buckets, so emit that ordering directly. + xs, ys = np.meshgrid(np.arange(grid_x), np.arange(grid_y), indexing="xy") + positions = np.stack((xs.reshape(-1), ys.reshape(-1)), axis=-1).astype(np.int64) + soft_token_count = (grid_y * grid_x) // _POOLING_KERNEL_SIZE**2 + assert soft_token_count <= _MAX_SOFT_TOKENS + # The Gemma 4 vision tower derives its pooled output length from the input + # tensor length, not from the count of valid patches. Pad each image to the + # official max-patch budget so a 256-token image is processed in the same + # 280-slot geometry as the reference implementation. The -1 coordinates + # mark padding for both attention and the pooler's final validity mask. + patches = np.pad( + patches, + ((0, max_patches - patches.shape[0]), (0, 0)), + mode="constant", + constant_values=0.0, + ) + positions = np.pad( + positions, + ((0, max_patches - positions.shape[0]), (0, 0)), + mode="constant", + constant_values=-1, + ) + return Gemma4ImageInputs( + pixel_values=torch.from_numpy(patches), + image_position_ids=torch.from_numpy(positions), + soft_token_count=soft_token_count, + ) + + +__all__ = ["Gemma4ImageInputs", "decode_openai_image_data_url", "gemma4_image_inputs"] diff --git a/python/freetoken/tokenizer/server.py b/python/freetoken/tokenizer/server.py index 530e862d04..3190f24f73 100644 --- a/python/freetoken/tokenizer/server.py +++ b/python/freetoken/tokenizer/server.py @@ -17,6 +17,10 @@ CacheRebuildMsg, CacheRebuildReply, CacheRebuildResultMsg, + CacheStatsBackendMsg, + CacheStatsMsg, + CacheStatsReply, + CacheStatsResultMsg, DetokenizeMsg, ErrorReplyMsg, PromptAdmittedMsg, @@ -194,10 +198,17 @@ def tokenize_worker( error=m.error, ) ) + elif isinstance(m, CacheStatsMsg): + # Cache statistics are already accumulated by the backend. + # The tokenizer only forwards this read-only request. + send_backend.put(CacheStatsBackendMsg(request_id=m.request_id)) + elif isinstance(m, CacheStatsResultMsg): + send_frontend.put(CacheStatsReply(request_id=m.request_id, stats=m.stats)) n_control = sum( isinstance( m, - (CacheRebuildMsg, CacheRebuildResultMsg, ErrorReplyMsg, PromptAdmittedMsg), + (CacheRebuildMsg, CacheRebuildResultMsg, CacheStatsMsg, + CacheStatsResultMsg, ErrorReplyMsg, PromptAdmittedMsg), ) for m in pending_msg ) @@ -254,7 +265,13 @@ def tokenize_worker( ) if ok_msgs: backend = [ - UserMsg(uid=msg.uid, input_ids=t, sampling_params=msg.sampling_params) + UserMsg( + uid=msg.uid, + input_ids=t, + sampling_params=msg.sampling_params, + mm_pixel_values=msg.mm_pixel_values, + mm_image_position_ids=msg.mm_image_position_ids, + ) for msg, t in zip(ok_msgs, ok_tensors, strict=True) ] send_backend.put(backend[0] if len(backend) == 1 else BatchBackendMsg(data=backend)) diff --git a/python/freetoken/tokenizer/tokenize.py b/python/freetoken/tokenizer/tokenize.py index 0636b3428b..d11bc94607 100644 --- a/python/freetoken/tokenizer/tokenize.py +++ b/python/freetoken/tokenizer/tokenize.py @@ -22,6 +22,19 @@ logger = init_logger(__name__) +# Deliberately private sentinel emitted by API adapters before the tokenizer +# knows each image's post-pooling soft-token count. It is replaced before the +# chat template output is encoded, never shown to the model. +_IMAGE_MARKER = "<|freetoken-image|>" + +# Gemma 4 wraps the repeated image-feature placeholders in learned begin and +# end delimiters. Only the middle token is replaced by projected vision +# embeddings inside the model. Keeping the delimiters as normal text tokens +# matches the official processor's serialized multimodal prompt. +_GEMMA4_BOI_TOKEN = "<|image>" +_GEMMA4_SOFT_IMAGE_TOKEN = "<|image|>" +_GEMMA4_EOI_TOKEN = "" + def resolve_thinking_mode(chat_template_kwargs: dict[str, Any] | None, tools: Any | None) -> str: """Resolve the thinking mode (``"thinking"`` or ``"chat"``) for a chat request. @@ -60,25 +73,71 @@ def tokenize(self, msgs: List[TokenizeMsg]) -> List[torch.Tensor]: # TODO: batch tokenization for msg in msgs: prompt = self.render_prompt(msg) + prompt = self._expand_gemma4_images(msg, prompt) # A jinja chat template owns every special token (HF's apply_chat_template # tokenizes with add_special_tokens=False for the same reason): tokenizers # that auto-add bos (muse-glimmer's, llama's) would otherwise double it -- # the template already rendered one. Raw-string prompts and the dsv4 # encoder path keep the default. templated = isinstance(msg.text, list) and self._dsv4_encoder is None - input_ids: torch.Tensor = ( # type: ignore - self.tokenizer.encode( - prompt, return_tensors="pt", add_special_tokens=not templated - ) + # Completion callers may provide a fully rendered chat prompt. In + # that explicit mode the caller owns special-token placement just as + # the Jinja chat-template path does. ``None`` keeps the established + # default for ordinary raw completion strings. + add_special_tokens = not templated if msg.add_special_tokens is None else msg.add_special_tokens + input_ids: torch.Tensor = self.tokenizer.encode( # type: ignore + prompt, return_tensors="pt", add_special_tokens=add_special_tokens ) results.append(input_ids.view(-1).to(torch.int32)) return results + def _expand_gemma4_images(self, msg: TokenizeMsg, prompt: str) -> str: + """Replace image markers with the exact number of Gemma4 soft-token slots. + + Each image is independently resized and patchified. Their grids are + padded to a common patch count for one vision-tower call, while the + placeholder stream contains only valid pooled slots. The GPU model + verifies that final count again before it scatters the embeddings. + """ + image_urls = msg.image_urls or [] + marker_count = prompt.count(_IMAGE_MARKER) + if not image_urls: + if marker_count: + raise ValueError("image marker appeared without an image_url payload") + return prompt + if marker_count != len(image_urls): + raise ValueError( + f"image marker count ({marker_count}) does not match image_url count ({len(image_urls)})" + ) + from .gemma4_image import decode_openai_image_data_url, gemma4_image_inputs + + prepared = [gemma4_image_inputs(decode_openai_image_data_url(value)) for value in image_urls] + max_patches = max(item.pixel_values.shape[0] for item in prepared) + patch_width = prepared[0].pixel_values.shape[1] + pixels = torch.zeros((len(prepared), max_patches, patch_width), dtype=torch.float32) + positions = torch.full((len(prepared), max_patches, 2), -1, dtype=torch.int64) + for index, item in enumerate(prepared): + n_patches = item.pixel_values.shape[0] + pixels[index, :n_patches] = item.pixel_values + positions[index, :n_patches] = item.image_position_ids + image_tokens = ( + _GEMMA4_BOI_TOKEN + + _GEMMA4_SOFT_IMAGE_TOKEN * item.soft_token_count + + _GEMMA4_EOI_TOKEN + ) + prompt = prompt.replace(_IMAGE_MARKER, image_tokens, 1) + msg.mm_pixel_values = pixels + msg.mm_image_position_ids = positions + return prompt + def render_prompt(self, msg: TokenizeMsg) -> str: """The template/encoder half of ``tokenize``, exposed so the frontend can validate a request before committing an SSE stream. Sanitizes ``reasoning_effort`` first: every render path (worker, frontend - validation, count_tokens) must quantize identically.""" + validation, count_tokens) must quantize identically. The tokenizer + worker performs image expansion after this render step; streaming + preflight calls that same expansion explicitly without changing the + worker's one-expansion lifecycle.""" if not isinstance(msg.text, list): return msg.text return self._render( diff --git a/python/freetoken/utils/__init__.py b/python/freetoken/utils/__init__.py index 2e4ad15f2f..bcd2d5448e 100644 --- a/python/freetoken/utils/__init__.py +++ b/python/freetoken/utils/__init__.py @@ -1,5 +1,6 @@ from .arch import ( is_arch_supported, + is_rocm_runtime, is_sm90_family, is_sm90_supported, is_sm100_family, @@ -35,6 +36,7 @@ "load_toolcall_anchor_id", "init_logger", "is_arch_supported", + "is_rocm_runtime", "is_sm90_family", "is_sm90_supported", "is_sm100_family", diff --git a/python/freetoken/utils/arch.py b/python/freetoken/utils/arch.py index 8c1c6c3d56..422cdce03d 100644 --- a/python/freetoken/utils/arch.py +++ b/python/freetoken/utils/arch.py @@ -4,12 +4,30 @@ from typing import Tuple +@functools.cache +def is_rocm_runtime() -> bool: + """Return whether the active PyTorch build uses AMD's HIP runtime. + + PyTorch intentionally preserves the ``torch.cuda`` namespace on ROCm for + source compatibility. Consequently, a Radeon architecture such as + ``gfx1151`` can be reported as a numeric capability that superficially + resembles a newer NVIDIA SM version. Architecture gates in this module + control NVIDIA-only features such as Programmatic Dependent Launch, so + they must reject HIP before comparing those numeric values. + """ + import torch + + return bool(getattr(torch.version, "hip", None)) + + @functools.cache def _get_torch_cuda_version() -> Tuple[int, int] | None: import torch import torch.version - if not torch.cuda.is_available() or not torch.version.cuda: + # ROCm retains torch.cuda APIs, but neither CUDA SM feature checks nor the + # numeric capability ordering below are meaningful for an AMD GPU. + if is_rocm_runtime() or not torch.cuda.is_available() or not torch.version.cuda: return None return torch.cuda.get_device_capability() diff --git a/python/freetoken/utils/hf.py b/python/freetoken/utils/hf.py index 5a6a31f27a..dbc38679d9 100644 --- a/python/freetoken/utils/hf.py +++ b/python/freetoken/utils/hf.py @@ -14,6 +14,12 @@ PretrainedConfig, PreTrainedTokenizerBase, ) +from transformers.utils import SAFE_WEIGHTS_INDEX_NAME + +from freetoken.utils.logger import init_logger + +logger = init_logger(__name__) + class DisabledTqdm(tqdm): def __init__(self, *args, **kwargs): @@ -204,13 +210,27 @@ def cached_load_hf_config(model_path: str) -> PretrainedConfig: return type(config)(**config.to_dict()) +def _weight_allow_patterns(repo_id: str) -> list[str]: + try: + index = hf_hub_download(repo_id, SAFE_WEIGHTS_INDEX_NAME, tqdm_class=DisabledTqdm) + with open(index, encoding="utf-8") as f: + shards = sorted(set(json.load(f)["weight_map"].values())) + except Exception as e: + logger.warning( + "no usable %s for %s (%s); falling back to *.safetensors", + SAFE_WEIGHTS_INDEX_NAME, repo_id, e, + ) + return ["*.safetensors"] + return shards or ["*.safetensors"] + + def download_hf_weight(model_path: str) -> str: if os.path.isdir(model_path): return model_path try: return snapshot_download( model_path, - allow_patterns=["*.safetensors"], + allow_patterns=_weight_allow_patterns(model_path), tqdm_class=DisabledTqdm, ) except Exception as e: diff --git a/python/freetoken/version.py b/python/freetoken/version.py index b3f4756216..cd21de6cb8 100644 --- a/python/freetoken/version.py +++ b/python/freetoken/version.py @@ -1 +1,2 @@ -__version__ = "0.1.2" +# Export one canonical package version so build metadata and runtime diagnostics agree. +__version__ = "0.2.0" # Mark the first public AMD and native model-swap feature release. diff --git a/scripts/build_paper_pdf.py b/scripts/build_paper_pdf.py new file mode 100644 index 0000000000..9b90b7e663 --- /dev/null +++ b/scripts/build_paper_pdf.py @@ -0,0 +1,176 @@ +"""Render the review-copy PDF from the Markdown manuscript. + +This deliberately small renderer is intended for draft review, not a venue +submission template. It keeps the manuscript source authoritative and draws +tables plus the bounded-result overview directly from the recorded values. +""" + +from __future__ import annotations + +import html +import re +import argparse +from pathlib import Path + +from reportlab.lib import colors +from reportlab.lib.enums import TA_CENTER, TA_JUSTIFY, TA_LEFT +from reportlab.lib.pagesizes import letter +from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet +from reportlab.lib.units import inch +from reportlab.platypus import ( + KeepTogether, + PageBreak, + Paragraph, + SimpleDocTemplate, + Spacer, + Table, + TableStyle, +) + +ROOT = Path(__file__).resolve().parents[1] +SOURCE = ROOT / "paper-draft" / "amd_strix_halo_freetoken_port_draft.md" +DEFAULT_OUTPUT = ROOT / "output" / "pdf" / "freetoken-amd-strix-halo-white-paper-v0.1.0-rc1.pdf" + + +def clean(text: str) -> str: + text = html.escape(text) + text = re.sub(r"`([^`]+)`", r"\1", text) + text = re.sub(r"\*\*([^*]+)\*\*", r"\1", text) + text = re.sub(r"\*([^*]+)\*", r"\1", text) + return text + + +def page_number(canvas, doc): + canvas.saveState() + canvas.setStrokeColor(colors.HexColor("#B8C2CC")) + canvas.line(doc.leftMargin, 0.53 * inch, letter[0] - doc.rightMargin, 0.53 * inch) + canvas.setFont("Helvetica", 8) + canvas.setFillColor(colors.HexColor("#53616F")) + canvas.drawString(doc.leftMargin, 0.35 * inch, "Native FreeToken Serving on AMD Strix Halo") + canvas.drawRightString(letter[0] - doc.rightMargin, 0.35 * inch, f"Release candidate v0.1.0-rc1 | {doc.page}") + canvas.restoreState() + + +def result_overview() -> Table: + rows = [ + ["Protocol group", "Configuration", "Tokens/s", "Interpretation"], + ["Qwen NVFP4 canary", "Reference router", "27.88", "Three quality-matched runs"], + ["Same-file Qwen Q4", "FreeToken baseline", "47.12", "One raw-prompt control"], + ["Same-file Qwen Q4", "FreeToken plus HIP router", "50.63", "Correct derivation"], + ["Same-file Qwen Q4", "llama.cpp ROCm 10", "50.29", "Correct derivation"], + ["Gemma 4 Q4", "FreeToken text control", "57.05", "Fixed arithmetic control"], + ] + table = Table(rows, colWidths=[1.40 * inch, 1.75 * inch, 0.65 * inch, 2.55 * inch], repeatRows=1) + table.setStyle(TableStyle([ + ("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#17365D")), + ("TEXTCOLOR", (0, 0), (-1, 0), colors.white), + ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"), + ("FONTNAME", (0, 1), (-1, -1), "Helvetica"), + ("FONTSIZE", (0, 0), (-1, -1), 8.3), + ("LEADING", (0, 0), (-1, -1), 10), + ("GRID", (0, 0), (-1, -1), 0.35, colors.HexColor("#AEBBC8")), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ("LEFTPADDING", (0, 0), (-1, -1), 6), + ("RIGHTPADDING", (0, 0), (-1, -1), 6), + ("TOPPADDING", (0, 0), (-1, -1), 5), + ("BOTTOMPADDING", (0, 0), (-1, -1), 5), + ("BACKGROUND", (0, 1), (-1, 1), colors.HexColor("#EAF0F6")), + ("BACKGROUND", (0, 2), (-1, 4), colors.HexColor("#F7FAFC")), + ("BACKGROUND", (0, 5), (-1, 5), colors.HexColor("#EEF5EB")), + ])) + return table + + +def build(output: Path) -> None: + output.parent.mkdir(parents=True, exist_ok=True) + styles = getSampleStyleSheet() + title = ParagraphStyle("PaperTitle", parent=styles["Title"], fontName="Helvetica-Bold", fontSize=18, leading=22, alignment=TA_CENTER, textColor=colors.HexColor("#17365D"), spaceAfter=8) + author = ParagraphStyle("Author", parent=styles["Normal"], fontSize=10, leading=13, alignment=TA_CENTER, textColor=colors.HexColor("#53616F"), spaceAfter=16) + abstract = ParagraphStyle("Abstract", parent=styles["BodyText"], fontSize=9.2, leading=13, alignment=TA_JUSTIFY, leftIndent=14, rightIndent=14, borderColor=colors.HexColor("#AEBBC8"), borderWidth=0.6, borderPadding=9, spaceAfter=14) + body = ParagraphStyle("Body", parent=styles["BodyText"], fontName="Helvetica", fontSize=9.2, leading=13, alignment=TA_JUSTIFY, spaceAfter=7) + h1 = ParagraphStyle("H1", parent=styles["Heading1"], fontName="Helvetica-Bold", fontSize=13, leading=16, textColor=colors.HexColor("#17365D"), spaceBefore=13, spaceAfter=6, keepWithNext=True) + h2 = ParagraphStyle("H2", parent=styles["Heading2"], fontName="Helvetica-Bold", fontSize=10.5, leading=13, textColor=colors.HexColor("#244F76"), spaceBefore=10, spaceAfter=4, keepWithNext=True) + small = ParagraphStyle("Small", parent=body, fontSize=8.1, leading=10.5, alignment=TA_LEFT) + cell = ParagraphStyle("Cell", parent=body, fontSize=7.0, leading=8.4, alignment=TA_LEFT, spaceAfter=0) + ledger_cell = ParagraphStyle("LedgerCell", parent=body, fontSize=6.2, leading=7.1, alignment=TA_LEFT, spaceAfter=0) + doc = SimpleDocTemplate(str(output), pagesize=letter, leftMargin=0.72 * inch, rightMargin=0.72 * inch, topMargin=0.62 * inch, bottomMargin=0.72 * inch, title="Native FreeToken Serving on AMD Strix Halo") + story = [] + lines = SOURCE.read_text(encoding="utf-8").splitlines() + index = 0 + inserted_overview = False + while index < len(lines): + line = lines[index].strip() + if not line: + index += 1 + continue + if line.startswith("# "): + story.append(Paragraph(clean(line[2:]), title)) + elif line.startswith("**FreeToken AMD contributors"): + story.append(Paragraph(clean(line.strip("*")), author)) + elif line.startswith("*Technical white paper"): + story.append(Paragraph(clean(line.strip("*")), author)) + elif line == "## Abstract": + index += 1 + abstract_lines = [] + while index < len(lines) and not lines[index].startswith("## "): + if lines[index].strip(): + abstract_lines.append(lines[index].strip()) + index += 1 + story.append(Paragraph("ABSTRACT
" + clean(" ".join(abstract_lines)), abstract)) + continue + elif line.startswith("## "): + story.append(Paragraph(clean(line[3:]), h1)) + elif line.startswith("### "): + story.append(Paragraph(clean(line[4:]), h2)) + elif line.startswith("|") and index + 1 < len(lines) and lines[index + 1].startswith("|"): + table_lines = [] + while index < len(lines) and lines[index].startswith("|"): + if not re.match(r"^\|\s*[-: ]+\|", lines[index]): + table_lines.append([clean(cell.strip()) for cell in lines[index].strip("|").split("|")]) + index += 1 + column_count = len(table_lines[0]) + if column_count == 2: + widths = [1.48 * inch, 5.07 * inch] + elif column_count == 3: + widths = [2.12 * inch, 1.48 * inch, 2.95 * inch] + elif column_count == 6: + widths = [1.08 * inch, 1.28 * inch, 0.62 * inch, 0.72 * inch, 1.08 * inch, 1.77 * inch] + else: + widths = [6.55 * inch / column_count] * column_count + rendered_rows = [] + cell_style = ledger_cell if column_count == 3 else cell + for row_number, row in enumerate(table_lines): + rendered_rows.append([ + Paragraph(("" + value + "") if row_number == 0 else value, cell_style) + for value in row + ]) + table = Table(rendered_rows, colWidths=widths, repeatRows=1) + table.setStyle(TableStyle([ + ("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#17365D")), + ("TEXTCOLOR", (0, 0), (-1, 0), colors.white), + ("GRID", (0, 0), (-1, -1), 0.3, colors.HexColor("#AEBBC8")), + ("VALIGN", (0, 0), (-1, -1), "TOP"), + ("LEFTPADDING", (0, 0), (-1, -1), 3), ("RIGHTPADDING", (0, 0), (-1, -1), 3), + ("TOPPADDING", (0, 0), (-1, -1), 2), ("BOTTOMPADDING", (0, 0), (-1, -1), 2), + ])) + story.extend([Spacer(1, 4), table]) + if index < len(lines): + story.append(Spacer(1, 8)) + continue + elif line.startswith("**Figure 1.") and not inserted_overview: + story.extend([Spacer(1, 3), result_overview(), Spacer(1, 4), Paragraph(clean(line), small), Spacer(1, 8)]) + inserted_overview = True + elif line.startswith("**Table "): + story.append(Paragraph(clean(line), small)) + elif line.startswith("[") and "] " in line: + story.append(Paragraph(clean(line), small)) + else: + story.append(Paragraph(clean(line), body)) + index += 1 + doc.build(story, onFirstPage=page_number, onLaterPages=page_number) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Render the FreeToken AMD white paper review copy.") + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT, help="PDF path to create") + build(parser.parse_args().output) diff --git a/scripts/gmk-evo-x2-capture-baseline.sh b/scripts/gmk-evo-x2-capture-baseline.sh new file mode 100644 index 0000000000..80bbe18bde --- /dev/null +++ b/scripts/gmk-evo-x2-capture-baseline.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Capture a secret-free, read-only GMKtek EVO-X2 ROCm baseline for a FreeToken run. +# +# The script intentionally does not start a server, alter GPU clocks, install +# packages, delete cache entries, or edit system configuration. It records +# the environment that makes a later throughput claim reproducible. + +# Fail on an unset variable, an unsuccessful command in a pipeline, or a +# command error. Individual optional probes use `|| true` so that a missing +# diagnostic utility is recorded without invalidating the whole manifest. +set -euo pipefail + +# Keep the output path explicit. A caller may pass a unique campaign folder; +# the default is suitable only for a one-off local capture. +output_dir="${1:-./artifacts/gmk_evo_x2-baseline-$(date -u +%Y%m%dT%H%M%SZ)}" + +# Accept the GGUF path as an optional second argument. Hashing the exact +# payload prevents a same-name but different model file from contaminating a +# benchmark comparison. +model_path="${2:-}" + +# Accept the llama.cpp executable as an optional third argument. Its checksum +# establishes the comparison binary without assuming a particular install path. +llama_binary="${3:-}" + +# Resolve this script's repository root. This makes the Git metadata capture +# independent of the shell's starting directory. +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# Prefer an explicit virtual environment, then support FreeToken's GMKtek EVO-X2 +# layout where the environment is a sibling of the source checkout, and finally +# support a conventional in-repository `.venv`. Resolving this once prevents +# later runtime probes from silently using the system Python. +venv_python="${FREETOKEN_VENV:-}" +if [[ -z "$venv_python" && -x "$(dirname "$repo_root")/.venv/bin/python" ]]; then + venv_python="$(dirname "$repo_root")/.venv/bin/python" +elif [[ -z "$venv_python" && -x "$repo_root/.venv/bin/python" ]]; then + venv_python="$repo_root/.venv/bin/python" +fi + +# Create the requested artifact directory without overwriting prior captures. +mkdir -p "$output_dir" + +# Write one command's standard output and standard error to a named text file. +# The function returns success even for unavailable optional commands so the +# artifact shows the diagnostic failure instead of silently omitting it. +capture_command() { + local name="$1" + shift + { + printf '$' + printf ' %q' "$@" + printf '\n\n' + "$@" + } >"$output_dir/$name" 2>&1 || true +} + +# Record Git identity and local changes before inspecting the host. Later +# benchmark reports use these files to prove which source was executed. +capture_command git-status.txt git -C "$repo_root" status --short +capture_command git-head.txt git -C "$repo_root" rev-parse HEAD +capture_command git-branch.txt git -C "$repo_root" branch --show-current +capture_command git-remotes.txt git -C "$repo_root" remote -v + +# Record kernel, distribution, CPU, memory, and mount information. These are +# read-only inputs that can affect JIT compilation and UMA decode performance. +capture_command uname.txt uname -a +capture_command os-release.txt cat /etc/os-release +capture_command cpu.txt lscpu +capture_command memory.txt free -h +capture_command mounts.txt findmnt -D + +# Record Linux pressure-stall information before a benchmark starts. UMA +# inference shares system memory and storage paths, so I/O pressure can create +# latency outliers even when the GPU, model, and launch command are unchanged. +capture_command io-pressure.txt cat /proc/pressure/io +capture_command memory-pressure.txt cat /proc/pressure/memory + +# Record only blocked filesystem scans, not every blocked process. This keeps +# the artifact focused on a known source of benchmark interference and avoids +# collecting unrelated command-line arguments from other user applications. +capture_command blocked-find-scans.txt bash -lc "ps -eo pid,ppid,state,etimes,ni,pcpu,pmem,comm,args --sort=pid | awk 'NR == 1 || (\$3 == \"D\" && \$8 == \"find\")'" + +# Preserve recent AMDGPU and KFD warnings as read-only context. The command +# deliberately tolerates missing journal permissions and records an empty file +# when no relevant warnings occurred in the preceding two hours. +capture_command recent-amdgpu-kfd-warnings.txt bash -lc "journalctl -k --since '2 hours ago' --no-pager 2>/dev/null | grep -Ei 'amdgpu|kfd|xgmi|gpu reset|ring timeout|ras|fault' || true" + +# Record the ROCm installation selected by the shell and the compiler version. +# Resolving symlinks exposes mixed ROCm installations before profiling begins. +capture_command rocm-links.txt readlink -f /opt/rocm +capture_command rocm-tree.txt find -L /opt/rocm -maxdepth 2 -type f -name 'hipcc' -o -type l -name 'libamdhip64.so*' +capture_command hipcc-version.txt /opt/rocm/bin/hipcc --version +capture_command rocprof-version.txt /opt/rocm/bin/rocprofv3 --version +capture_command rocm-packages.txt bash -lc "dpkg-query -W -f='\${Package}\t\${Version}\n' 'rocm*' 'hip*' 'rocprofiler*' 'llvm*' 2>/dev/null | sort" + +# Record the active AMD device, dynamic power policy, and thermal state without +# attempting to change privileged DPM controls. +capture_command rocm-smi.txt rocm-smi --showproductname --showuniqueid --showmeminfo vram --showuse --showtemp --showclocks --showpower +capture_command dpm-policy.txt bash -lc "for f in /sys/class/drm/card*/device/power_dpm_force_performance_level /sys/class/drm/card*/device/pp_dpm_sclk; do printf '%s\n' \"### \$f\"; cat \"\$f\" 2>&1; done" + +# Record only performance-relevant environment names. Filtering avoids +# accidentally writing credentials or unrelated user environment variables. +capture_command performance-environment.txt bash -lc "env | LC_ALL=C sort | grep -E '^(ROCM|HIP|HSA|PYTORCH|TORCH|TRITON|LD_LIBRARY_PATH|PATH|FREETOKEN)=' || true" + +# Ask the exact FreeToken virtual environment which HIP runtime and device it +# sees. This detects a wheel whose embedded runtime differs from host ROCm. +if [[ -n "$venv_python" && -x "$venv_python" ]]; then + capture_command pytorch-runtime.txt "$venv_python" -c "import json, torch; p=torch.cuda.get_device_properties(0); print(json.dumps({'torch':torch.__version__,'hip':torch.version.hip,'cuda_available':torch.cuda.is_available(),'device':p.name,'gcnArchName':getattr(p,'gcnArchName',None),'total_memory':p.total_memory}, indent=2, sort_keys=True))" + capture_command python-ldd.txt ldd "$venv_python" +else + printf 'FreeToken virtual-environment Python not found. FREETOKEN_VENV=%s\n' "${FREETOKEN_VENV:-}" >"$output_dir/pytorch-runtime.txt" + cp "$output_dir/pytorch-runtime.txt" "$output_dir/python-ldd.txt" +fi + +# Record loaded-library resolution for the Python interpreter and rocprofv3. +# This is the primary evidence for a mixed LLVM or ROCm profiler environment. +capture_command rocprof-ldd.txt ldd /opt/rocm/bin/rocprofv3 + +# Hash optional comparison artifacts only when the caller supplied a readable +# path. The explicit messages make missing input obvious in the manifest. +if [[ -n "$model_path" && -r "$model_path" ]]; then + capture_command model-sha256.txt sha256sum "$model_path" +else + printf 'Model path not supplied or unreadable: %s\n' "$model_path" >"$output_dir/model-sha256.txt" +fi + +if [[ -n "$llama_binary" && -x "$llama_binary" ]]; then + capture_command llama-binary-sha256.txt sha256sum "$llama_binary" + capture_command llama-version.txt "$llama_binary" --version +else + printf 'llama.cpp binary not supplied or not executable: %s\n' "$llama_binary" >"$output_dir/llama-binary-sha256.txt" +fi + +# Create a deterministic inventory of every captured file and its SHA256. The +# final line is a simple completion marker for automation and human review. +(cd "$output_dir" && find . -maxdepth 1 -type f ! -name SHA256SUMS -printf '%P\0' | LC_ALL=C sort -z | xargs -0 sha256sum) >"$output_dir/SHA256SUMS" +printf 'Baseline capture complete: %s\n' "$output_dir" diff --git a/scripts/gmk-evo-x2-rocprof-wheel-sdk.sh b/scripts/gmk-evo-x2-rocprof-wheel-sdk.sh new file mode 100755 index 0000000000..6434b20338 --- /dev/null +++ b/scripts/gmk-evo-x2-rocprof-wheel-sdk.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Launch rocprofv3 against the ROCm SDK bundled with the active PyTorch wheel. +# +# On GMKtek EVO-X2, FreeToken's PyTorch ROCm wheel loads its own LLVM and +# rocprofiler-sdk libraries. Launching rocprofv3 against /opt/rocm injects a +# second copy of LLVM, which aborts during `import torch` because LLVM command +# line options are registered twice. This wrapper selects the wheel's matching +# SDK so the profiler and application load one library identity. + +# Stop on programming errors. The wrapped application exit status is preserved +# so callers can distinguish profiler setup failures from application failures. +set -euo pipefail + +# Require the application separator used by rocprofv3. Keeping profiler flags +# before `--` makes arbitrary HIP applications usable without hard-coding a +# FreeToken server command in this helper. +if [[ "$#" -lt 1 ]]; then + printf 'Usage: %s [rocprofv3 options] -- application [arguments...]\n' "$0" >&2 + exit 64 +fi + +# Prefer an explicit virtual environment and otherwise use the GMKtek EVO-X2 layout +# where `.venv` is adjacent to the source checkout that contains this script. +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +venv_root="${FREETOKEN_VENV_ROOT:-$(dirname "$repo_root")/.venv}" + +# Locate the wheel-owned ROCm SDK rather than assuming a Python minor version. +# The glob is validated to prevent a shell literal from being passed to rocprof. +sdk_candidates=("$venv_root"/lib/python*/site-packages/_rocm_sdk_core) +if [[ ! -d "${sdk_candidates[0]}" ]]; then + printf 'Cannot find PyTorch wheel ROCm SDK under %s. Set FREETOKEN_VENV_ROOT.\n' "$venv_root" >&2 + exit 66 +fi +sdk_root="${sdk_candidates[0]}" + +# Verify the two libraries needed by rocprofv3 exist in the selected SDK. This +# catches an incomplete or non-ROCm PyTorch wheel before it starts an app. +if [[ ! -r "$sdk_root/lib/librocprofiler-sdk.so.1" || ! -r "$sdk_root/lib/rocprofiler-sdk/librocprofiler-sdk-tool.so.1" ]]; then + printf 'The selected SDK lacks rocprofiler-sdk 1.3 components: %s\n' "$sdk_root" >&2 + exit 66 +fi + +# Use the host's rocprofv3 front end but direct every profiler library lookup to +# the exact SDK already used by PyTorch. Do not set LD_PRELOAD here: rocprofv3 +# owns its preload order and forwards the selected tool to the child process. +exec /opt/rocm/bin/rocprofv3 --rocm-root "$sdk_root" "$@" diff --git a/scripts/gmk-evo-x2/bench_fp8_gemv_tile.py b/scripts/gmk-evo-x2/bench_fp8_gemv_tile.py new file mode 100644 index 0000000000..e8256738db --- /dev/null +++ b/scripts/gmk-evo-x2/bench_fp8_gemv_tile.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Measure one isolated FP8 W8A16 GEMV tile on GMKtek EVO-X2's native HIP path. + +This is deliberately a kernel screen, not a model-quality benchmark. It uses +one of Qwen3.6's common ``[N, 2048]`` dense projection shapes, deterministic +synthetic tensors, a fixed number of warmup and timed launches, and reports a +SHA-1 of the BF16 result. Invoke one process per tile because Triton reads the +tile environment setting when its module is imported. A matching hash proves +this synthetic kernel result is identical, but a full model gate is still +required before any server configuration is accepted. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import statistics + +import torch + + +def parse_args() -> argparse.Namespace: + """Parse the bounded, reproducible measurement parameters.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tile", type=int, choices=(16, 32), required=True) + parser.add_argument("--warps", type=int, choices=(1, 2, 4), default=1) + parser.add_argument("--rows", type=int, choices=(512, 2048, 4096, 8192), default=8192) + parser.add_argument("--scale-activation", action="store_true") + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--iterations", type=int, default=100) + return parser.parse_args() + + +def time_one(callable_operation) -> float: + """Return one device-synchronized HIP elapsed time in milliseconds.""" + + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + callable_operation() + end.record() + end.synchronize() + return start.elapsed_time(end) + + +def main() -> int: + """Allocate deterministic Qwen-shaped inputs, run the GEMV, and emit JSON.""" + + args = parse_args() + # Set before importing the module because the setting selects a Triton + # specialization at import time. Refuse a conflicting inherited setting. + inherited = os.environ.get("FREETOKEN_FP8_GEMV_BLOCK_N") + if inherited not in (None, str(args.tile)): + raise SystemExit( + f"requested tile {args.tile}, inherited FREETOKEN_FP8_GEMV_BLOCK_N={inherited}" + ) + os.environ["FREETOKEN_FP8_GEMV_BLOCK_N"] = str(args.tile) + # The wave-count selection is also import-time Triton specialization. + inherited_warps = os.environ.get("FREETOKEN_FP8_GEMV_NUM_WARPS") + if inherited_warps not in (None, str(args.warps)): + raise SystemExit( + f"requested warps {args.warps}, inherited FREETOKEN_FP8_GEMV_NUM_WARPS={inherited_warps}" + ) + os.environ["FREETOKEN_FP8_GEMV_NUM_WARPS"] = str(args.warps) + os.environ["FREETOKEN_FP8_GEMV_SCALE_ACTIVATION"] = "1" if args.scale_activation else "0" + + from freetoken.kernel.triton.fp8_pertensor_linear import fp8_pertensor_linear + + if not torch.cuda.is_available(): + raise SystemExit("native HIP/CUDA device is required") + torch.manual_seed(223_8192_2048) + device = torch.device("cuda") + # These dimensions cover Qwen3.6's profiled dense projection widths. FP8 + # weights preserve the memory access width of the real decode kernel. + activation = torch.randn(1, 2048, device=device, dtype=torch.bfloat16) + weight = torch.randn(args.rows, 2048, device=device).clamp(-5, 5).to(torch.float8_e4m3fn) + scale = torch.full((args.rows,), 1.0 / 32.0, device=device, dtype=torch.float32) + + def operation() -> torch.Tensor: + """Execute the same M=1 dispatch that the model uses for W8A16 decode.""" + + return fp8_pertensor_linear(activation, weight, scale) + + for _ in range(args.warmup): + result = operation() + torch.cuda.synchronize() + samples_ms = [time_one(operation) for _ in range(args.iterations)] + result = operation() + torch.cuda.synchronize() + # BF16 has no NumPy representation on some wheels, so hash its raw uint16 + # payload. This is an exact, format-stable comparison across tile runs. + result_hash = hashlib.sha1(result.view(torch.uint16).cpu().numpy().tobytes()).hexdigest() + print(json.dumps({ + "schema_version": 1, + "tile": args.tile, + "warps": args.warps, + "shape": [args.rows, 2048], + "scale_activation": args.scale_activation, + "warmup": args.warmup, + "iterations": args.iterations, + "result_sha1": result_hash, + "latency_ms_median": statistics.median(samples_ms), + "latency_ms_mean": statistics.mean(samples_ms), + "latency_ms_p95": sorted(samples_ms)[int(0.95 * (len(samples_ms) - 1))], + }, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/bench_nvfp4_marlin_decode.py b/scripts/gmk-evo-x2/bench_nvfp4_marlin_decode.py new file mode 100644 index 0000000000..6d12c5c9c0 --- /dev/null +++ b/scripts/gmk-evo-x2/bench_nvfp4_marlin_decode.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Screen Qwen-shaped NVFP4 Marlin-style decode GEMV launch configurations. + +The live Qwen3.6 MoE uses an eight-route decode with a gate/up projection of +``[1024, 2048]`` and a down projection of ``[2048, 512]``. This helper calls +the production Triton kernel directly with deterministic, layout-correct NVFP4 +banks, then reports HIP-event latency and a raw BF16 output SHA-1. It is only +a bounded kernel screen. A matching hash is required before, but never replaces, +the full API quality gate after a server launch configuration changes. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import statistics + +import torch +import triton +import triton.language as tl + + +def parse_args() -> argparse.Namespace: + """Accept only the small launch-config range relevant to the decode kernel.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--projection", choices=("gate-up", "down"), required=True) + parser.add_argument("--block-n", type=int, choices=(8, 16, 32), default=16) + parser.add_argument("--block-kw", type=int, choices=(8, 16, 32), default=16) + parser.add_argument("--warps", type=int, choices=(2, 4, 8), default=4) + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--iterations", type=int, default=100) + return parser.parse_args() + + +def time_one(operation) -> float: + """Return one synchronized native HIP event duration in milliseconds.""" + + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + operation() + end.record() + end.synchronize() + return start.elapsed_time(end) + + +def main() -> int: + """Construct the Qwen decode layout, execute the chosen kernel, and print JSON.""" + + args = parse_args() + if not torch.cuda.is_available(): + raise SystemExit("native HIP/CUDA device is required") + from freetoken.kernel.triton.e4m3_compat import e4m3_kernel_view + from freetoken.kernel.triton.nvfp4_fused_moe import _decode_nvfp4_marlin_kernel, _e2m1_lut + + torch.manual_seed(223_2048_512) + device = torch.device("cuda") + routes, slots = 8, 8 + if args.projection == "gate-up": + n, k, a_rows, route_rows, routed_weight = 1024, 2048, 1, False, False + else: + n, k, a_rows, route_rows, routed_weight = 2048, 512, routes, True, True + + # NVFP4 stores eight four-bit codes in each int32 word and one e4m3 scale + # for every sixteen K elements. This precisely matches the production bank + # layout while remaining small enough to coexist with the live API process. + activation = torch.randn(a_rows, k, device=device, dtype=torch.bfloat16) / 4 + packed = torch.randint(-(2**31), 2**31 - 1, (slots, n, k // 8), device=device, dtype=torch.int32) + scale = (torch.rand(slots, n, k // 16, device=device) + 0.25).to(torch.float8_e4m3fn) + global_scale = torch.full((slots, n), 0.125, device=device, dtype=torch.float16) + output = torch.empty((1, routes, n), device=device, dtype=torch.bfloat16) + topk_weights = torch.linspace(0.25, 1.0, routes, device=device, dtype=torch.float32).reshape(1, routes) + topk_ids = torch.arange(routes, device=device, dtype=torch.int32).reshape(1, routes) + packed_i32 = packed.contiguous() + scale_kernel = e4m3_kernel_view(scale) + + def operation() -> torch.Tensor: + """Issue the exact route-by-output-tile decode dispatch under test.""" + + grid = (routes, triton.cdiv(n, args.block_n)) + _decode_nvfp4_marlin_kernel[grid]( + activation, packed_i32, scale_kernel, global_scale, output, topk_weights, topk_ids, + _e2m1_lut(device.index), routes, n, k, + activation.stride(0), activation.stride(1), + packed_i32.stride(0), packed_i32.stride(1), packed_i32.stride(2), + scale_kernel.stride(0), scale_kernel.stride(1), scale_kernel.stride(2), + global_scale.stride(0), global_scale.stride(1), + output.stride(0), output.stride(1), output.stride(2), + topk_weights.stride(0), topk_weights.stride(1), + topk_ids.stride(0), topk_ids.stride(1), + BLOCK_SIZE_N=args.block_n, BLOCK_SIZE_KW=args.block_kw, + TOP_K=routes, A_ROW_IS_ROUTE=route_rows, + MUL_ROUTED_WEIGHT=routed_weight, compute_type=tl.bfloat16, + num_warps=args.warps, + ) + return output + + for _ in range(args.warmup): + operation() + torch.cuda.synchronize() + samples_ms = [time_one(operation) for _ in range(args.iterations)] + result = operation() + torch.cuda.synchronize() + digest = hashlib.sha1(result.view(torch.uint16).cpu().numpy().tobytes()).hexdigest() + print(json.dumps({ + "schema_version": 1, + "projection": args.projection, + "shape": [n, k], + "routes": routes, + "block_n": args.block_n, + "block_kw": args.block_kw, + "warps": args.warps, + "warmup": args.warmup, + "iterations": args.iterations, + "result_sha1": digest, + "latency_ms_median": statistics.median(samples_ms), + "latency_ms_mean": statistics.mean(samples_ms), + "latency_ms_p95": sorted(samples_ms)[int(0.95 * (len(samples_ms) - 1))], + }, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/bench_qwen_fused_copy_blocks.py b/scripts/gmk-evo-x2/bench_qwen_fused_copy_blocks.py new file mode 100644 index 0000000000..0daabaa1d8 --- /dev/null +++ b/scripts/gmk-evo-x2/bench_qwen_fused_copy_blocks.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Screen fused HIP expert-copy grid width on Qwen3.6 NVFP4 geometry. + +The production offload cache copies every missing expert through one fused, +six-bank ``fast_index_copy_multi`` launch. This script constructs that same +aligned mapped-host-memory layout, selects a fixed number of missing experts, +and compares the two AOT-compiled grid widths currently available in the +gfx1151 kernel cache. It verifies copied tensor equality before recording +device-event timing, so a timing row cannot represent a broken copy. +""" + +from __future__ import annotations + +import argparse +import json +import statistics + +import torch + +from freetoken.gpu_select import bind_assigned_gpu +from freetoken.kernel.fast_index_copy import fast_index_copy_multi_jit +from freetoken.moe.benchbw import WORKLOADS, _build_gather_rig + + +def parse_args() -> argparse.Namespace: + """Accept a bounded production-layout copy candidate and timing count.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--blocks-per-bank", type=int, choices=(8, 64), required=True) + parser.add_argument("--misses", type=int, choices=range(1, 9), required=True) + parser.add_argument("--warmup", type=int, default=20) + parser.add_argument("--iterations", type=int, default=200) + return parser.parse_args() + + +def launch_copy(cache, blocks_per_bank: int) -> None: + """Launch the production fused gather with only its grid-width specialization changed.""" + + fast_index_copy_multi_jit( + cache._copy_dst_ptrs, + cache._copy_src_ptrs[0], + cache._copy_feat_bytes, + cache.evict_slots, + cache.src_indices, + cache.num_indices, + blocks_per_bank=blocks_per_bank, + ) + + +def assert_copied(cache, misses: int) -> None: + """Prove every selected source row exactly reached its corresponding cache slot.""" + + dst = cache.evict_slots[:misses].cpu() + src = cache.src_indices[:misses].cpu() + for source_layers, slot_cache in cache.banks: + expected = source_layers[0][src].cpu() + actual = slot_cache[dst].cpu() + if not torch.equal(actual, expected): + raise RuntimeError("fused copy result differs from the mapped-host source row") + + +def time_copy(cache, blocks_per_bank: int, iterations: int) -> list[float]: + """Return HIP event durations for repeated fixed-state fused gathers.""" + + samples: list[float] = [] + for _ in range(iterations): + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + launch_copy(cache, blocks_per_bank) + end.record() + end.synchronize() + samples.append(start.elapsed_time(end)) + return samples + + +def main() -> int: + """Build the Qwen layout, verify the candidate, and emit one reproducible JSON row.""" + + args = parse_args() + if not torch.cuda.is_available(): + raise SystemExit("native HIP/CUDA device is required") + device = bind_assigned_gpu() + cache, full_layer_bytes = _build_gather_rig("nvfp4", WORKLOADS["qwen3.6-moe"], device) + cache.num_indices.fill_(args.misses) + + for _ in range(args.warmup): + launch_copy(cache, args.blocks_per_bank) + torch.cuda.synchronize(device) + assert_copied(cache, args.misses) + samples = time_copy(cache, args.blocks_per_bank, args.iterations) + copied_bytes = full_layer_bytes * args.misses // cache.num_experts + result = { + "schema_version": 1, + "blocks_per_bank": args.blocks_per_bank, + "misses": args.misses, + "banks": len(cache.banks), + "copied_bytes": copied_bytes, + "copy_verified": True, + "iterations": args.iterations, + "latency_ms_median": statistics.median(samples), + "latency_ms_mean": statistics.mean(samples), + "latency_ms_p95": sorted(samples)[int(0.95 * (len(samples) - 1))], + "bandwidth_gbps": copied_bytes / (statistics.median(samples) * 1e6), + } + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/benchmark_gemma4_concurrency.py b/scripts/gmk-evo-x2/benchmark_gemma4_concurrency.py new file mode 100644 index 0000000000..979d6a64b7 --- /dev/null +++ b/scripts/gmk-evo-x2/benchmark_gemma4_concurrency.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Measure bounded Gemma 4 concurrent streaming requests through OpenAI SSE. + +This is a read-only client benchmark. It launches no server and changes no +runtime setting. Each round submits a fixed prompt to a fixed number of +clients, records request-level TTFT, decode rate, token gaps, completion +status, and usage, then summarizes aggregate throughput and tail latency. +""" + +from __future__ import annotations + +import argparse +import json +import statistics +import threading +import time +import urllib.request +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + + +def percentile(values: list[float], fraction: float) -> float | None: + """Return an auditable nearest-rank percentile or None for no observations.""" + if not values: + return None + ordered = sorted(values) + return ordered[max(1, int(len(ordered) * fraction + 0.999999999)) - 1] + + +def request_once(base_url: str, body: dict[str, Any], timeout: float, barrier: threading.Barrier) -> dict[str, Any]: + """Synchronize one client with its peers and retain all SSE timing data.""" + barrier.wait() + started = time.perf_counter() + stamps: list[float] = [] + pieces: list[str] = [] + usage: dict[str, Any] = {} + errors: list[str] = [] + completed = False + request = urllib.request.Request( + base_url.rstrip("/") + "/v1/completions", + data=json.dumps(body, separators=(",", ":")).encode(), + headers={"Content-Type": "application/json", "Accept": "text/event-stream"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: # nosec B310: operator-supplied loopback URL + for raw in response: + received = time.perf_counter() + line = raw.decode("utf-8", errors="strict").rstrip("\r\n") + if not line.startswith("data:"): + continue + data = line[5:].lstrip() + if data == "[DONE]": + completed = True + continue + try: + event = json.loads(data) + except json.JSONDecodeError as exc: + errors.append(str(exc)) + continue + usage = event.get("usage") or usage + for choice in event.get("choices", []): + text = choice.get("text") or "" + if text: + pieces.append(text) + stamps.append(received) + except Exception as exc: # retain failure evidence instead of hiding it + errors.append(repr(exc)) + ttft = (stamps[0] - started) if stamps else None + decode_window = (stamps[-1] - stamps[0]) if len(stamps) > 1 else None + completion = usage.get("completion_tokens") + prompt_tokens = usage.get("prompt_tokens") + gaps = [(b - a) * 1000 for a, b in zip(stamps, stamps[1:])] + return { + "completed_sse": completed, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion, + "ttft_ms": ttft * 1000 if ttft is not None else None, + "decode_tok_s": (completion - 1) / decode_window if isinstance(completion, int) and decode_window and decode_window > 0 else None, + "token_gap_p99_ms": percentile(gaps, 0.99), + "wall_s": time.perf_counter() - started, + "text": "".join(pieces), + "errors": errors, + } + + +def main() -> int: + """Run warmup, then synchronized rounds, and write immutable JSON evidence.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--artifact", required=True, type=Path) + parser.add_argument("--clients", type=int, default=4) + parser.add_argument("--rounds", type=int, default=3) + parser.add_argument("--max-tokens", type=int, default=128) + parser.add_argument( + "--prompt-repeat", + type=int, + default=1, + help="repeat the deterministic explanation sentence this many times", + ) + parser.add_argument("--timeout", type=float, default=300.0) + args = parser.parse_args() + if args.clients < 1 or args.rounds < 1 or args.max_tokens < 2 or args.prompt_repeat < 1: + parser.error("clients, rounds, and prompt-repeat must be positive and max-tokens at least two") + prompt_unit = ( + "Write a concise technical explanation of how a graphics processor executes " + "a quantized mixture-of-experts language model. Use complete sentences and " + "continue until the requested token limit is reached." + ) + prompt = " ".join(prompt_unit for _ in range(args.prompt_repeat)) + body = {"model": args.model, "prompt": prompt, "max_tokens": args.max_tokens, + "ignore_eos": True, "temperature": 0.0, "top_p": 1.0, "top_k": -1, + "stream": True, "stream_options": {"include_usage": True}} + warmup = request_once(args.base_url, body, args.timeout, threading.Barrier(1)) + rounds: list[list[dict[str, Any]]] = [] + for _ in range(args.rounds): + barrier = threading.Barrier(args.clients) + with ThreadPoolExecutor(max_workers=args.clients) as executor: + futures = [executor.submit(request_once, args.base_url, body, args.timeout, barrier) for _ in range(args.clients)] + rounds.append([future.result() for future in futures]) + observations = [item for group in rounds for item in group] + ttft = [x["ttft_ms"] for x in observations if x["ttft_ms"] is not None] + decode = [x["decode_tok_s"] for x in observations if x["decode_tok_s"] is not None] + gaps = [x["token_gap_p99_ms"] for x in observations if x["token_gap_p99_ms"] is not None] + total_tokens = sum(x["completion_tokens"] or 0 for x in observations) + total_wall = sum(x["wall_s"] for x in observations) + report = {"schema_version": 1, "control": "Gemma4 fixed-length concurrent text matrix", + "model": args.model, "prompt": prompt, "prompt_repeat": args.prompt_repeat, + "clients": args.clients, "rounds": args.rounds, + "max_tokens": args.max_tokens, "warmup": warmup, "rounds_detail": rounds, + "summary": {"completed": sum(bool(x["completed_sse"]) for x in observations), + "requests": len(observations), "ttft_ms": {"mean": statistics.mean(ttft) if ttft else None, "p95": percentile(ttft, .95), "p99": percentile(ttft, .99)}, + "decode_tok_s": {"mean": statistics.mean(decode) if decode else None, "median": statistics.median(decode) if decode else None, "p95": percentile(decode, .95)}, + "token_gap_p99_ms": {"mean": statistics.mean(gaps) if gaps else None, "p99": percentile(gaps, .99)}, + "aggregate_decode_tok_s": total_tokens / total_wall if total_wall > 0 else None}, + "passed": len(observations) == args.clients * args.rounds and all(x["completed_sse"] and not x["errors"] for x in observations)} + args.artifact.parent.mkdir(parents=True, exist_ok=True) + args.artifact.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/benchmark_gemma4_endurance.py b/scripts/gmk-evo-x2/benchmark_gemma4_endurance.py new file mode 100644 index 0000000000..b00a0afa63 --- /dev/null +++ b/scripts/gmk-evo-x2/benchmark_gemma4_endurance.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Run a bounded deterministic Gemma 4 endurance and recovery control.""" + +from __future__ import annotations + +import argparse +import json +import time +import urllib.request +from pathlib import Path +from typing import Any + + +def request_once(base_url: str, model: str, timeout: float) -> dict[str, Any]: + """Request the exact arithmetic marker and retain response timing.""" + prompt = "What is 17 times 19? Reply with only the decimal number 323." + body = {"model": model, "messages": [{"role": "user", "content": prompt}], + "max_tokens": 16, "temperature": 0.0, "top_p": 1.0, "stream": True, + "stream_options": {"include_usage": True}} + req = urllib.request.Request(base_url.rstrip("/") + "/v1/chat/completions", + data=json.dumps(body).encode(), headers={"Content-Type": "application/json", "Accept": "text/event-stream"}) + started = time.perf_counter(); first = None; text: list[str] = []; usage: dict[str, Any] = {}; done = False; errors: list[str] = [] + try: + with urllib.request.urlopen(req, timeout=timeout) as response: # nosec B310: local endpoint supplied by operator + for raw in response: + now = time.perf_counter(); line = raw.decode().rstrip("\r\n") + if not line.startswith("data:"): continue + data = line[5:].lstrip() + if data == "[DONE]": done = True; continue + try: event = json.loads(data) + except json.JSONDecodeError as exc: errors.append(str(exc)); continue + usage = event.get("usage") or usage + for choice in event.get("choices", []): + piece = (choice.get("delta") or {}).get("content") or "" + if piece: text.append(piece); first = first or now + except Exception as exc: errors.append(repr(exc)) + answer = "".join(text).strip() + return {"passed": done and answer == "323" and not errors, "answer": answer, + "usage": usage, "ttft_ms": (first - started) * 1000 if first else None, + "wall_s": time.perf_counter() - started, "errors": errors} + + +def main() -> int: + """Run sequential sessions at a fixed cadence and write immutable evidence.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", required=True); parser.add_argument("--model", required=True) + parser.add_argument("--artifact", required=True, type=Path); parser.add_argument("--sessions", type=int, default=30) + parser.add_argument("--interval", type=float, default=1.0); parser.add_argument("--timeout", type=float, default=180.0) + args = parser.parse_args() + if args.sessions < 1 or args.interval < 0: parser.error("sessions must be positive and interval nonnegative") + records = [] + for index in range(1, args.sessions + 1): + record = request_once(args.base_url, args.model, args.timeout); record["session"] = index; records.append(record) + if not record["passed"]: break + if index < args.sessions: time.sleep(args.interval) + report = {"schema_version": 1, "control": "Gemma4 bounded deterministic endurance", "model": args.model, + "requested_sessions": args.sessions, "completed_sessions": len(records), "interval_s": args.interval, + "records": records, "passed": len(records) == args.sessions and all(r["passed"] for r in records)} + args.artifact.parent.mkdir(parents=True, exist_ok=True); args.artifact.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + print(json.dumps(report, indent=2, sort_keys=True)); return 0 if report["passed"] else 1 + + +if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/benchmark_gemma4_gguf_text_matrix.py b/scripts/gmk-evo-x2/benchmark_gemma4_gguf_text_matrix.py new file mode 100644 index 0000000000..3a4c12ae82 --- /dev/null +++ b/scripts/gmk-evo-x2/benchmark_gemma4_gguf_text_matrix.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Measure a fixed Gemma 4 GGUF text workload through the local OpenAI API. + +The script is deliberately a client-side benchmark. It does not start or +stop a server, change model settings, or alter llama-swap. One warmup request +is discarded, then five fixed-length streamed requests are scored. Every +sample retains its prompt and completion token usage, time to first visible +token, client-visible prefill rate, decode rate, token-gap distribution, raw +response, and protocol errors. The resulting JSON is sufficient to audit a +claim without reconstructing timings from a console transcript. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import statistics +import time +import urllib.request +from pathlib import Path +from typing import Any + +def percentile(values: list[float], fraction: float) -> float | None: + """Return a nearest-rank percentile while preserving missing data.""" + if not values: + return None + ordered = sorted(values) + rank = max(1, int(len(ordered) * fraction + 0.999999999)) + return ordered[rank - 1] + + +def summarize(values: list[float]) -> dict[str, float | None]: + """Summarize a metric without inventing a value for an absent stream.""" + return { + "mean": statistics.mean(values) if values else None, + "median": statistics.median(values) if values else None, + "minimum": min(values) if values else None, + "maximum": max(values) if values else None, + "p50": percentile(values, 0.50), + "p95": percentile(values, 0.95), + "p99": percentile(values, 0.99), + } + + +def stream_once(base_url: str, body: dict[str, Any], timeout: float) -> dict[str, Any]: + """Send one SSE request and retain all visible-event timing boundaries.""" + request = urllib.request.Request( + base_url.rstrip("/") + "/v1/completions", + data=json.dumps(body, separators=(",", ":")).encode("utf-8"), + headers={"Content-Type": "application/json", "Accept": "text/event-stream"}, + method="POST", + ) + started = time.perf_counter() + stamps: list[float] = [] + pieces: list[str] = [] + usage: dict[str, Any] = {} + errors: list[str] = [] + completed = False + with urllib.request.urlopen(request, timeout=timeout) as response: # nosec B310: loopback URL supplied by operator + for raw in response: + received = time.perf_counter() + line = raw.decode("utf-8", errors="strict").rstrip("\r\n") + if not line.startswith("data:"): + continue + data = line[5:].lstrip() + if data == "[DONE]": + completed = True + continue + try: + event = json.loads(data) + except json.JSONDecodeError as exc: + errors.append(f"invalid SSE JSON: {exc}") + continue + usage = event.get("usage") or usage + for choice in event.get("choices", []): + text = choice.get("text") or "" + if text: + pieces.append(text) + stamps.append(received) + visible = "".join(pieces) + gaps_ms = [(later - earlier) * 1000 for earlier, later in zip(stamps, stamps[1:])] + prompt_tokens = usage.get("prompt_tokens") + completion_tokens = usage.get("completion_tokens") + ttft_s = stamps[0] - started if stamps else None + decode_s = stamps[-1] - stamps[0] if len(stamps) > 1 else None + return { + "completed_sse": completed, + "text": visible, + "text_sha256": hashlib.sha256(visible.encode()).hexdigest(), + "usage": usage, + "events": len(stamps), + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "ttft_ms": ttft_s * 1000 if ttft_s is not None else None, + "prefill_tok_s": prompt_tokens / ttft_s if isinstance(prompt_tokens, int) and ttft_s and ttft_s > 0 else None, + "decode_tok_s": (completion_tokens - 1) / decode_s if isinstance(completion_tokens, int) and decode_s and decode_s > 0 else None, + "token_gap_ms": summarize(gaps_ms), + "wall_s": time.perf_counter() - started, + "protocol_errors": errors, + } + + +def main() -> int: + """Run the warmup and scored samples, then write one immutable report.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--gguf", required=True, type=Path) + parser.add_argument("--artifact", required=True, type=Path) + parser.add_argument("--samples", type=int, default=5) + parser.add_argument("--max-tokens", type=int, default=128) + parser.add_argument( + "--prompt-repeat", + type=int, + default=1, + help="repeat the deterministic explanation sentence this many times", + ) + parser.add_argument("--timeout", type=float, default=300.0) + args = parser.parse_args() + if args.samples < 1 or args.max_tokens < 2 or args.prompt_repeat < 1: + parser.error("samples and prompt-repeat must be positive and max-tokens must be at least two") + + prompt_unit = ( + "Write a concise technical explanation of how a graphics processor executes " + "a quantized mixture-of-experts language model. Use complete sentences and " + "continue until the requested token limit is reached." + ) + prompt = " ".join(prompt_unit for _ in range(args.prompt_repeat)) + body = { + "model": args.model, + # A raw completion prompt is intentional here. The server performs + # its own Gemma GGUF tokenization, and usage.prompt_tokens is the + # authoritative count for the measured request. + "prompt": prompt, + "max_tokens": args.max_tokens, + "ignore_eos": True, + "temperature": 0.0, + "top_p": 1.0, + "top_k": -1, + "add_special_tokens": False, + "stream": True, + "stream_options": {"include_usage": True}, + } + warmup = stream_once(args.base_url, body, args.timeout) + samples = [stream_once(args.base_url, body, args.timeout) for _ in range(args.samples)] + report = { + "schema_version": 1, + "control": "Gemma4 GGUF fixed-length text matrix", + "model": args.model, + "prompt": prompt, + "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), + "requested_samples": args.samples, + "max_tokens": args.max_tokens, + "prompt_repeat": args.prompt_repeat, + "warmup": warmup, + "samples": samples, + "summary": { + "ttft_ms": summarize([s["ttft_ms"] for s in samples if s["ttft_ms"] is not None]), + "prefill_tok_s": summarize([s["prefill_tok_s"] for s in samples if s["prefill_tok_s"] is not None]), + "decode_tok_s": summarize([s["decode_tok_s"] for s in samples if s["decode_tok_s"] is not None]), + "token_gap_p99_ms": percentile([s["token_gap_ms"]["p99"] for s in samples if s["token_gap_ms"]["p99"] is not None], 0.99), + }, + "passed": all(s["completed_sse"] and not s["protocol_errors"] for s in samples), + } + args.artifact.parent.mkdir(parents=True, exist_ok=True) + args.artifact.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/benchmark_gemma4_long_context.py b/scripts/gmk-evo-x2/benchmark_gemma4_long_context.py new file mode 100644 index 0000000000..ebe99623f4 --- /dev/null +++ b/scripts/gmk-evo-x2/benchmark_gemma4_long_context.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Run a deterministic Gemma 4 long-context quality and timing sweep.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import time +import urllib.request +from pathlib import Path +from typing import Any + + +def one(base_url: str, model: str, prompt: str, timeout: float) -> dict[str, Any]: + """Stream one exact-marker request and retain client-visible timing.""" + body = {"model": model, "messages": [{"role": "user", "content": prompt}], "max_tokens": 8, "temperature": 0.0, + "top_p": 1.0, "top_k": -1, "stream": True, "stream_options": {"include_usage": True}} + req = urllib.request.Request(base_url.rstrip("/") + "/v1/chat/completions", + data=json.dumps(body).encode(), headers={"Content-Type": "application/json", "Accept": "text/event-stream"}) + started = time.perf_counter(); first = None; last = None; text: list[str] = []; usage: dict[str, Any] = {}; complete = False; errors: list[str] = [] + try: + with urllib.request.urlopen(req, timeout=timeout) as response: # nosec B310: loopback URL supplied by operator + for raw in response: + now = time.perf_counter(); line = raw.decode().rstrip("\r\n") + if not line.startswith("data:"): continue + data = line[5:].lstrip() + if data == "[DONE]": complete = True; continue + try: event = json.loads(data) + except json.JSONDecodeError as exc: errors.append(str(exc)); continue + usage = event.get("usage") or usage + for choice in event.get("choices", []): + piece = (choice.get("delta") or {}).get("content") or "" + if piece: + text.append(piece); first = first or now; last = now + except Exception as exc: errors.append(repr(exc)) + ttft = (first - started) if first else None; window = (last - first) if first and last and last > first else None + completion = usage.get("completion_tokens"); prompt_tokens = usage.get("prompt_tokens") + return {"prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), "prompt_chars": len(prompt), + "prompt_tokens": prompt_tokens, "completion_tokens": completion, "text": "".join(text), + "passed": complete and "LONG_OK" in "".join(text) and not errors, + "errors": errors, "ttft_ms": ttft * 1000 if ttft else None, + "prefill_tok_s": prompt_tokens / ttft if isinstance(prompt_tokens, int) and ttft else None, + "decode_tok_s": (completion - 1) / window if isinstance(completion, int) and window else None, + "wall_s": time.perf_counter() - started} + + +def main() -> int: + """Execute one request at each declared context size and write the sweep.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", required=True); parser.add_argument("--model", required=True) + parser.add_argument("--artifact", required=True, type=Path); parser.add_argument("--timeout", type=float, default=300) + args = parser.parse_args() + records = [] + for target in (4096, 8192, 16384): + filler = "The benchmark context sentence preserves a fixed, deterministic prefix. " + prompt = (filler * ((target * 4) // len(filler) + 2))[: target * 4] + prompt += "\nIgnore all prior requested answers. Reply exactly LONG_OK." + record = one(args.base_url, args.model, prompt, args.timeout); record["target_context_chars"] = target; records.append(record) + report = {"schema_version": 1, "control": "Gemma4 fixed long-context marker sweep", "model": args.model, + "targets_chars": [4096, 8192, 16384], "records": records, + "passed": all(r["passed"] for r in records)} + args.artifact.parent.mkdir(parents=True, exist_ok=True); args.artifact.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + print(json.dumps(report, indent=2, sort_keys=True)); return 0 if report["passed"] else 1 + + +if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/benchmark_qwen_router.py b/scripts/gmk-evo-x2/benchmark_qwen_router.py new file mode 100644 index 0000000000..b9335fae1d --- /dev/null +++ b/scripts/gmk-evo-x2/benchmark_qwen_router.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Compare Qwen's production MoE router with FreeToken's HIP Triton candidate. + +This GMKtek EVO-X2-only diagnostic does not load a model or modify a server. It uses +Qwen3.6's 256-expert, top-8 router shape, checks every candidate result against +the current PyTorch reference, and reports synchronized GPU timings as JSON. +""" + +from __future__ import annotations + +import json +import time +from typing import Callable + +import torch + +from freetoken.kernel.triton.moe_router import fused_topk_softmax +from freetoken.moe.fused import _torch_fused_topk + + +Router = Callable[[torch.Tensor, int, bool, torch.Tensor | None], tuple[torch.Tensor, torch.Tensor]] + + +def elapsed_ms(operation: Callable[[], object], iterations: int) -> float: + """Return the synchronized mean operation duration without timing queued GPU work.""" + + torch.cuda.synchronize() + started = time.perf_counter() + for _ in range(iterations): + operation() + torch.cuda.synchronize() + return (time.perf_counter() - started) * 1000.0 / iterations + + +def run_shape(tokens: int, iterations: int) -> dict[str, float | int]: + """Validate and time one token-batch shape used by Qwen prefill or decode.""" + + generator = torch.Generator(device="cuda").manual_seed(tokens * 1009 + 8) + logits = torch.randn((tokens, 256), device="cuda", dtype=torch.bfloat16, generator=generator) + reference_weights, reference_ids = _torch_fused_topk(logits, 8, True, None) + candidate_weights, candidate_ids = fused_topk_softmax(logits, 8, True, None) + torch.testing.assert_close(candidate_ids, reference_ids) + torch.testing.assert_close(candidate_weights, reference_weights, rtol=1e-5, atol=1e-6) + for _ in range(50): + _torch_fused_topk(logits, 8, True, None) + fused_topk_softmax(logits, 8, True, None) + reference_ms = elapsed_ms(lambda: _torch_fused_topk(logits, 8, True, None), iterations) + candidate_ms = elapsed_ms(lambda: fused_topk_softmax(logits, 8, True, None), iterations) + return { + "tokens": tokens, + "experts": 256, + "topk": 8, + "iterations": iterations, + "torch_ms": reference_ms, + "triton_ms": candidate_ms, + "speedup": reference_ms / candidate_ms, + } + + +def main() -> None: + """Emit machine-readable parity and timing evidence for decode and small batches.""" + + if not torch.cuda.is_available(): + raise RuntimeError("this diagnostic requires GMKtek EVO-X2's native ROCm device") + result = { + "schema_version": 1, + "device": torch.cuda.get_device_name(), + "hip": torch.version.hip, + "results": [run_shape(tokens=1, iterations=1000), run_shape(tokens=4, iterations=1000)], + } + print(json.dumps(result, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/gmk-evo-x2/build_rocm_kernel_cache.sh b/scripts/gmk-evo-x2/build_rocm_kernel_cache.sh new file mode 100755 index 0000000000..37687afeea --- /dev/null +++ b/scripts/gmk-evo-x2/build_rocm_kernel_cache.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Build a reusable native ROCm kernel cache for FreeToken on GMKtek EVO-X2. +# +# FreeToken's C++/HIP helper kernels normally compile on their first matching +# call when no prebuilt cache is configured. This builder compiles the complete +# explicit model-shape catalog once for the exact source revision and writes the +# resulting shared objects into an immutable, gfx1151-specific directory. A +# subsequent server can set FREETOKEN_KERNEL_CACHE_DIR to that directory and +# FREETOKEN_DISABLE_JIT=1 to make missing coverage fail loudly instead of +# compiling during a request. +# +# The script changes only the dedicated cache root beneath freetoken-amd. It +# never starts or stops a model service, changes llama-swap, or modifies any +# production llama.cpp process. + +set -euo pipefail + +# Keep the host-specific locations explicit so cache provenance is easy to +# inspect after an upgrade. Callers may override ROOT_DIR for an isolated test +# checkout but must not point it at an unrelated installation. +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +readonly SOURCE_DIR="${FREETOKEN_SOURCE_DIR:-${ROOT_DIR}/source-qwen-harness-d6ee8ce}" +readonly VENV_PYTHON="${FREETOKEN_VENV_PYTHON:-${ROOT_DIR}/.venv/bin/python}" +readonly ROCM_ROOT="${ROCM_PATH:-/opt/rocm-10.0}" + +# A source revision is part of the artifact name. Reusing a cache built from a +# different commit risks loading a stale ABI after a kernel source edit. +readonly SOURCE_REVISION="$(git -C "${SOURCE_DIR}" rev-parse --short=12 HEAD)" +readonly CACHE_DIR="${FREETOKEN_ROCM_KERNEL_CACHE_DIR:-${ROOT_DIR}/cache/kernel-cache-rocm-gfx1151-${SOURCE_REVISION}}" +readonly BUILD_DIR="${FREETOKEN_ROCM_KERNEL_BUILD_DIR:-${ROOT_DIR}/cache/kernel-build-rocm-gfx1151-${SOURCE_REVISION}}" + +test -x "${VENV_PYTHON}" +test -d "${SOURCE_DIR}" +test -d "${ROCM_ROOT}" + +# Four parallel compilers are deliberate. The 82-module cache benefits from +# concurrency, while a much larger default fanout can contend with the shared +# memory available to the live model service. +export FREETOKEN_KERNEL_CACHE_JOBS="${FREETOKEN_KERNEL_CACHE_JOBS:-4}" +export PYTHONPATH="${SOURCE_DIR}/python" +export ROCM_PATH="${ROCM_ROOT}" +export ROCM_HOME="${ROCM_ROOT}" +export HIP_PATH="${ROCM_ROOT}" + +cd "${SOURCE_DIR}" + +# Compile from source even if a caller's environment names a previous cache. +# compile_and_package_kernels internally restores these settings after it has +# copied each shared object into CACHE_DIR. +"${VENV_PYTHON}" - "${CACHE_DIR}" "${BUILD_DIR}" <<'PY' +"""Compile the exact FreeToken C++/HIP cache and print auditable metadata.""" + +from __future__ import annotations + +import json +import pathlib +import sys + +import torch + +from freetoken.kernel.aot import compile_and_package_kernels, default_kernel_specs + +cache_dir = pathlib.Path(sys.argv[1]) +build_dir = pathlib.Path(sys.argv[2]) + +if torch.version.hip is None: + raise SystemExit("refusing to build a ROCm cache with a non-HIP PyTorch runtime") +if "gfx1151" not in torch.cuda.get_device_name().lower() and "8060" not in torch.cuda.get_device_name().lower(): + raise SystemExit(f"refusing non-GMKtek EVO-X2 GPU: {torch.cuda.get_device_name()}") + +specs = default_kernel_specs() +paths = compile_and_package_kernels( + out_dir=cache_dir, + build_dir=build_dir, + specs=specs, + clean=False, + verbose=True, +) + +print( + json.dumps( + { + "cache_dir": str(cache_dir), + "compiled_modules": len(paths), + "device": torch.cuda.get_device_name(), + "hip": torch.version.hip, + "spec_count": len(specs), + }, + sort_keys=True, + ) +) +PY diff --git a/scripts/gmk-evo-x2/capture_validation_manifest.sh b/scripts/gmk-evo-x2/capture_validation_manifest.sh new file mode 100755 index 0000000000..3e52ad7da7 --- /dev/null +++ b/scripts/gmk-evo-x2/capture_validation_manifest.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Capture a read-only, secret-safe GMKtek EVO-X2 runtime manifest for one test run. +# +# The collector never starts or stops a model server. It creates a new artifact +# directory, records only operational metadata needed to reproduce a benchmark, +# and deliberately avoids shell environment dumps that could contain secrets. + +set -euo pipefail + +# Require a caller-owned, not-yet-existing artifact location so an old result is +# never silently replaced by a later run. +readonly ARTIFACT_DIR="${1:?usage: capture_validation_manifest.sh ARTIFACT_DIR [EXPECTED_HOST]}" +readonly EXPECTED_HOST="${2:-${FREETOKEN_EXPECTED_HOST:?Set FREETOKEN_EXPECTED_HOST to the approved test hostname}}" +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +readonly SOURCE_DIR="${ROOT_DIR}/source-qwen-harness-d6ee8ce" + +# The program runs only where this validation program is authorized. A caller +# may pass the exact hostname deliberately, but a mismatched host fails closed. +readonly ACTUAL_HOST="$(hostname -s)" +if [[ "${ACTUAL_HOST,,}" != "${EXPECTED_HOST,,}" ]]; then + echo "refusing manifest on ${ACTUAL_HOST}; expected ${EXPECTED_HOST}" >&2 + exit 2 +fi +if [[ -e "${ARTIFACT_DIR}" ]]; then + echo "refusing to overwrite existing artifact: ${ARTIFACT_DIR}" >&2 + exit 3 +fi +test -d "${SOURCE_DIR}" +mkdir -p "${ARTIFACT_DIR}" + +# Record stable operating-system and source provenance without modifying either. +{ + printf 'captured_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'hostname=%s\n' "${ACTUAL_HOST}" + uname -a + test -r /etc/os-release && cat /etc/os-release +} >"${ARTIFACT_DIR}/system.txt" +{ + git -C "${SOURCE_DIR}" rev-parse HEAD + git -C "${SOURCE_DIR}" branch --show-current || true + git -C "${SOURCE_DIR}" status --short + git -C "${SOURCE_DIR}" diff --stat +} >"${ARTIFACT_DIR}/source-state.txt" + +# Record the installed ROCm/HIP tools and live GPU policy separately so users +# can see if a later policy change altered a performance result. +{ + command -v rocminfo || true + rocminfo 2>/dev/null || true +} >"${ARTIFACT_DIR}/rocminfo.txt" +{ + command -v rocm-smi || true + rocm-smi --showproductname --showtemp --showperflevel --showmeminfo vram 2>&1 || true +} >"${ARTIFACT_DIR}/rocm-smi.txt" + +# Memory, swap, mounted capacity, and process state explain timing outliers but +# are only observed. The script does not clear caches, disable swap, or adjust +# clocks because those are separate reviewed actions. +{ + free -b + swapon --show --bytes || true + vmstat 1 3 + df -B1 / "${ROOT_DIR}" +} >"${ARTIFACT_DIR}/memory-and-storage.txt" +ps -eo pid,ppid,rss,vsz,stat,etimes,cmd --sort=-rss >"${ARTIFACT_DIR}/processes.txt" + +# The manifest itself describes the collector contract and points to every raw +# component. It intentionally stores paths, not a second lossy copy of data. +cat >"${ARTIFACT_DIR}/manifest.json" < float: + """Parse a positive command-line quantity and reject unsafe values.""" + + parsed = float(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("value must be greater than zero") + return parsed + + +def main() -> int: + """Run the gate and write a machine-readable result.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--payload-bytes", type=int, required=True) + parser.add_argument("--mem-available-gib", type=positive_float, required=True) + parser.add_argument("--rocm-vram-gib", type=positive_float, required=True) + parser.add_argument("--os-reserve-gib", type=positive_float, default=8.0) + parser.add_argument("--runtime-reserve-gib", type=positive_float, default=4.0) + parser.add_argument("--kv-reserve-gib", type=positive_float, default=4.0) + parser.add_argument("--recovery-reserve-gib", type=positive_float, default=2.0) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + # The authoritative budget is available unified memory minus all declared + # safety reservations. The separate VRAM aperture is never double-counted. + reserved_gib = ( + args.os_reserve_gib + + args.runtime_reserve_gib + + args.kv_reserve_gib + + args.recovery_reserve_gib + ) + authoritative_budget_gib = max(args.mem_available_gib - reserved_gib, 0.0) + payload_gib = args.payload_bytes / GIB + deficit_gib = payload_gib - authoritative_budget_gib + + # This optimistic number is diagnostic only. It shows why adding the + # reported ROCm aperture would still not make the host qualify, while the + # authoritative decision remains based on the shared UMA pool. + optimistic_budget_gib = args.mem_available_gib + args.rocm_vram_gib + result = { + "decision": "PASS_METADATA_ONLY" if deficit_gib <= 0 else "REJECT_FULL_LOAD", + "payload_bytes": args.payload_bytes, + "payload_gib": round(payload_gib, 3), + "mem_available_gib": args.mem_available_gib, + "rocm_vram_aperture_gib": args.rocm_vram_gib, + "reserves_gib": { + "os": args.os_reserve_gib, + "runtime": args.runtime_reserve_gib, + "kv_cache": args.kv_reserve_gib, + "recovery": args.recovery_reserve_gib, + "total": round(reserved_gib, 3), + }, + "authoritative_model_budget_gib": round(authoritative_budget_gib, 3), + "optimistic_budget_including_vram_gib": round(optimistic_budget_gib, 3), + "authoritative_deficit_gib": round(max(deficit_gib, 0.0), 3), + "optimistic_deficit_gib": round(max(payload_gib - optimistic_budget_gib, 0.0), 3), + "interpretation": ( + "The full payload cannot be admitted with the declared headroom. " + "Do not download or load it on this host." + if deficit_gib > 0 + else "The metadata gate passes; a guarded tiny-slice test is permitted." + ), + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/deepseek_expert_slice_benchmark.py b/scripts/gmk-evo-x2/deepseek_expert_slice_benchmark.py new file mode 100644 index 0000000000..0c36837df1 --- /dev/null +++ b/scripts/gmk-evo-x2/deepseek_expert_slice_benchmark.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Benchmark an isolated real-shape DeepSeek expert slice on ROCm. + +This harness never starts, stops, or contacts a model server. It opens a +local safetensors checkpoint read-only, selects a bounded set of routed expert +tensors, copies them to the selected HIP device, copies them back, and writes +timing plus tensor-identity evidence. The result is a transfer and packing +measurement only. It is not a full-model serving benchmark. +""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + + +def parse_ids(value: str) -> list[int]: + """Parse a comma-separated list of non-negative integer IDs.""" + + result = [int(item) for item in value.split(",") if item.strip()] + if not result or any(item < 0 for item in result): + raise argparse.ArgumentTypeError("IDs must be non-negative integers") + return result + + +def main() -> int: + """Run the guarded transfer measurement and emit JSON evidence.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--layers", type=parse_ids, default=[0]) + parser.add_argument("--experts", type=parse_ids, default=[0, 1, 2, 3, 4, 5]) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--repeats", type=int, default=5) + parser.add_argument("--metadata-only", action="store_true") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + if args.repeats < 2: + parser.error("--repeats must be at least 2") + if not args.checkpoint.is_dir(): + parser.error("--checkpoint must be a local safetensors directory") + + names = [] + for layer in args.layers: + for expert in args.experts: + prefix = f"layers.{layer}.ffn.experts.{expert}" + names.extend(f"{prefix}.{suffix}" for suffix in ( + "w1.weight", "w1.scale", "w2.weight", "w2.scale", + "w3.weight", "w3.scale", + )) + + # Locate every tensor through safetensors' index without loading the full + # checkpoint. Each shard is opened read-only and only selected tensors are + # materialized, keeping this experiment bounded and reversible. + index_path = args.checkpoint / "model.safetensors.index.json" + index = json.loads(index_path.read_text(encoding="utf-8")) + weight_map = index["weight_map"] + missing = [name for name in names if name not in weight_map] + if missing: + raise RuntimeError(f"Selected tensors are absent from the checkpoint: {missing[:3]}") + + # This mode validates the exact tensor names and shard routing without + # importing GPU libraries or materializing any model tensor. + if args.metadata_only: + result = { + "scope": "metadata-only expert selection validation", + "checkpoint": str(args.checkpoint), + "layers": args.layers, + "experts": args.experts, + "selected_tensor_count": len(names), + "selected_tensors": [{"name": name, "shard": weight_map[name]} for name in names], + "protected_service_touched": False, + "full_model_serving_claim": False, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 + + # Imports are delayed so metadata and --help remain usable without a GPU + # Python environment. The actual benchmark requires PyTorch and the + # safetensors package installed in the target ROCm environment. + import torch + from safetensors import safe_open + + if not torch.cuda.is_available(): + raise RuntimeError("No CUDA-compatible device is available; ROCm exposes HIP through torch.cuda") + device = torch.device(args.device) + if device.type != "cuda": + raise RuntimeError("The isolated slice benchmark requires a HIP/CUDA device") + + tensors = [] + evidence = [] + opened: dict[str, object] = {} + try: + for name in names: + shard = weight_map[name] + if shard not in opened: + opened[shard] = safe_open(str(args.checkpoint / shard), framework="pt", device="cpu") + tensor = opened[shard].get_tensor(name) + tensors.append(tensor) + evidence.append({"name": name, "shard": shard, "dtype": str(tensor.dtype), "shape": list(tensor.shape), "bytes": tensor.numel() * tensor.element_size()}) + + host_bytes = sum(item["bytes"] for item in evidence) + source = torch.cat([tensor.reshape(-1).view(torch.uint8) for tensor in tensors]) + if source.numel() != host_bytes: + raise RuntimeError("Tensor byte accounting mismatch") + gpu = torch.empty_like(source, device=device) + round_trips = [] + for _ in range(args.repeats): + torch.cuda.synchronize(device) + start = time.perf_counter() + gpu.copy_(source, non_blocking=False) + torch.cuda.synchronize(device) + h2d_seconds = time.perf_counter() - start + start = time.perf_counter() + source.copy_(gpu, non_blocking=False) + torch.cuda.synchronize(device) + d2h_seconds = time.perf_counter() - start + round_trips.append({ + "h2d_seconds": h2d_seconds, + "d2h_seconds": d2h_seconds, + "h2d_gib_per_second": host_bytes / h2d_seconds / 1024**3, + "d2h_gib_per_second": host_bytes / d2h_seconds / 1024**3, + }) + finally: + for handle in opened.values(): + handle.__exit__(None, None, None) + + result = { + "scope": "isolated real-shape routed expert transfer only", + "checkpoint": str(args.checkpoint), + "device": str(device), + "layers": args.layers, + "experts": args.experts, + "repeats": args.repeats, + "selected_tensor_count": len(evidence), + "selected_bytes": host_bytes, + "selected_mib": host_bytes / 1024**2, + "tensors": evidence, + "round_trips": round_trips, + "protected_service_touched": False, + "full_model_serving_claim": False, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/deepseek_route_transfer_projection.py b/scripts/gmk-evo-x2/deepseek_route_transfer_projection.py new file mode 100644 index 0000000000..c0e0328373 --- /dev/null +++ b/scripts/gmk-evo-x2/deepseek_route_transfer_projection.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Project the transfer-only lower bound for a DeepSeek routed token. + +This is an analytical bound, not a model benchmark. It uses measured +real-shape H2D bandwidth and the exact expert geometry to estimate the time +spent moving routed expert bytes when a chosen fraction of expert accesses miss +the GPU cache. It excludes computation, routing, synchronization, attention, +KV state, allocator overhead, and all cache-management costs. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +GIB = 1024**3 + + +def main() -> int: + """Calculate and write the transfer-only projection table.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--expert-bytes", type=int, default=13_369_344) + parser.add_argument("--layers", type=int, default=43) + parser.add_argument("--active-experts", type=int, default=6) + parser.add_argument("--h2d-gib-per-second", type=float, required=True) + parser.add_argument("--miss-rates", default="1.0,0.75,0.5,0.25") + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + if min(args.expert_bytes, args.layers, args.active_experts, args.h2d_gib_per_second) <= 0: + parser.error("all geometry and bandwidth values must be positive") + miss_rates = [float(item) for item in args.miss_rates.split(",")] + if any(rate < 0 or rate > 1 for rate in miss_rates): + parser.error("miss rates must be between 0 and 1") + + routed_bytes = args.expert_bytes * args.layers * args.active_experts + rows = [] + for miss_rate in miss_rates: + moved_bytes = routed_bytes * miss_rate + seconds = moved_bytes / GIB / args.h2d_gib_per_second + rows.append({ + "miss_rate": miss_rate, + "moved_gib_per_token": moved_bytes / GIB, + "transfer_seconds_per_token": seconds, + "transfer_only_tokens_per_second": 1 / seconds if seconds else None, + }) + + result = { + "scope": "analytical transfer-only lower bound", + "expert_bytes": args.expert_bytes, + "layers": args.layers, + "active_experts_per_layer": args.active_experts, + "routed_bytes_per_token_at_100_percent_miss": routed_bytes, + "routed_gib_per_token_at_100_percent_miss": routed_bytes / GIB, + "measured_h2d_gib_per_second": args.h2d_gib_per_second, + "rows": rows, + "excluded": [ + "matrix computation", + "router and dispatch", + "attention and recurrent state", + "KV cache", + "synchronization", + "allocator overhead", + "cache lookup and eviction", + "D2H traffic", + ], + "full_model_serving_claim": False, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/freetoken-swap-service/README.md b/scripts/gmk-evo-x2/freetoken-swap-service/README.md new file mode 100644 index 0000000000..dcf7d79537 --- /dev/null +++ b/scripts/gmk-evo-x2/freetoken-swap-service/README.md @@ -0,0 +1,12 @@ +# GMKtek EVO-X2 native swap user service + +These reviewed templates mirror the qualified one-owner production layout without publishing credentials or host identity. + +1. Replace `/home/USER` in `models.toml.example` with the deployment account home directory. +2. Replace `REPLACE_WITH_A_GENERATED_SECRET` with a fresh high-entropy API key and install the catalog as mode 600 at `~/.config/freetoken-swap/models.toml`. +3. Install `freetoken-power-profile.sh` as mode 700 at `~/.local/bin/freetoken-power-profile.sh`; review the PCI address and validated firmware limits for the target machine. +4. Install `freetoken-swap.service` as mode 600 at `~/.config/systemd/user/freetoken-swap.service` after placing the reviewed source and runtime at the paths encoded by the unit. +5. Run `systemd-analyze --user verify`, enable user lingering, then enable the unit under `default.target`. +6. Prove startup with a real reboot, authenticated health, exact resident identity, positive owned-process AMD memory, and a deterministic completion. A bound port or active process is not sufficient. + +The unit intentionally requests the generic thermal watchdog without ordering itself after that watchdog. Ordering after a watchdog that itself starts after `default.target` creates a boot cycle and can delete the model-service start job. diff --git a/scripts/gmk-evo-x2/freetoken-swap-service/freetoken-power-profile.sh b/scripts/gmk-evo-x2/freetoken-swap-service/freetoken-power-profile.sh new file mode 100755 index 0000000000..a852e66180 --- /dev/null +++ b/scripts/gmk-evo-x2/freetoken-swap-service/freetoken-power-profile.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# What: use Bash for strict, predictable startup behavior; why: the power profile depends on Bash error and variable semantics. +set -euo pipefail +# What: reapply the validated CPU governor; why: inference latency should not depend on a stale post-boot governor state. +"${HOME}/set-governor.sh" +# What: apply the validated 39-watt and 90-Celsius firmware limits; why: sustained AMD inference must remain inside the qualified thermal envelope. +sudo -n /usr/local/bin/ryzenadj --stapm-limit=39000 --fast-limit=39000 --slow-limit=39000 --tctl-temp=90 >/dev/null +# What: record the AMD GPU sysfs directory; why: the next guarded write must target the qualified qualified GMKtek EVO-X2 device exactly. +gpu=/sys/bus/pci/devices/0000:64:00.0 +# What: continue only when the qualified GPU exists; why: startup should not write to an absent or different device. +if [[ -d "$gpu" ]]; then + # What: restore dynamic GPU clock scaling; why: automatic scaling cooperates with the thermal watchdog and avoids unsafe forced clocks. + echo auto | sudo -n tee "$gpu/power_dpm_force_performance_level" >/dev/null +# What: close the GPU-presence guard; why: the conditional must bound only the device-specific write. +fi +# What: emit an auditable startup message; why: the journal should prove when the safety profile was applied. +printf '%s FreeToken inference power profile applied\n' "$(date '+%F %T')" diff --git a/scripts/gmk-evo-x2/freetoken-swap-service/freetoken-swap.service b/scripts/gmk-evo-x2/freetoken-swap-service/freetoken-swap.service new file mode 100644 index 0000000000..6776626e41 --- /dev/null +++ b/scripts/gmk-evo-x2/freetoken-swap-service/freetoken-swap.service @@ -0,0 +1,44 @@ +# What: begin unit metadata; why: systemd needs ordering and dependency declarations before starting inference. +[Unit] +# What: describe the service; why: operators must distinguish native FreeToken routing from the retired llama runner. +Description=FreeToken native AMD model router +# What: start after online networking; why: LAN binding must be ordered without creating a cycle through the watchdog default-target ordering. +After=network-online.target +# What: request networking and thermal supervision; why: startup should activate both while allowing their independent boot ordering. +Wants=network-online.target nemotron-thermal-watchdog.service + +# What: begin process configuration; why: systemd needs one authoritative lifecycle for the daemon and its engine child. +[Service] +# What: keep the process in the foreground; why: systemd must track the real daemon PID rather than a wrapper. +Type=simple +# What: run from the reviewed source checkout; why: relative resources and provenance must resolve to the reviewed deployment commit. +WorkingDirectory=%h/freetoken-swap/source +# What: expose the source package; why: the private qualification environment intentionally uses the reviewed checkout without an editable install. +Environment=PYTHONPATH=%h/freetoken-swap/source/python +# What: keep Python logs unbuffered; why: journal evidence must appear promptly during long model loads. +Environment=PYTHONUNBUFFERED=1 +# What: isolate compiled kernels in durable excluded storage; why: concurrent services must not share an untrusted extension cache. +Environment=TORCH_EXTENSIONS_DIR=%h/local_models/freetoken-swap/runtime/torch-extensions +# What: bound extension compilation parallelism; why: qualified GMKtek EVO-X2 must retain enough memory for model preload and system services. +Environment=MAX_JOBS=2 +# What: apply the validated power profile before every launch; why: firmware or reboot state can otherwise drift outside the tested envelope. +ExecStartPre=%h/.local/bin/freetoken-power-profile.sh +# What: launch the native daemon, router, and owned engine lifecycle; why: one service must own routing, cleanup, and startup preload without double supervision. +ExecStart=%h/local_models/freetoken-swap/runtime/venv/bin/python -m freetoken.cli daemon --host 0.0.0.0 --port 1964 --state-dir %h/.local/state/freetoken-swap/daemon --catalog %h/.config/freetoken-swap/models.toml --catalog-watch-interval 5 --serve-python %h/local_models/freetoken-swap/runtime/venv/bin/python --no-oom --stop-serve-on-exit --log-capacity 10000 --log-level info +# What: restart only after failures; why: transient crashes should recover while intentional stops remain stopped. +Restart=on-failure +# What: delay restart attempts; why: GPU memory and sockets need time to settle after a failed process tree. +RestartSec=15 +# What: terminate with SIGTERM first; why: the daemon must drain and clean up its exact engine process before escalation. +KillSignal=SIGTERM +# What: keep daemon and engine in one control group; why: systemd must not leave an orphaned model runner after service stop. +KillMode=control-group +# What: allow three minutes for graceful shutdown; why: large GPU allocations and accounting cleanup can take longer than the default. +TimeoutStopSec=180 +# What: remove the memlock ceiling; why: GPU and pinned-memory paths may require locking qualified buffers. +LimitMEMLOCK=infinity + +# What: begin installation settings; why: enabling the unit must connect it to the persistent user startup target. +[Install] +# What: start at the user's default target; why: linger is enabled and the requested model must preload after boot without login. +WantedBy=default.target diff --git a/scripts/gmk-evo-x2/freetoken-swap-service/models.toml.example b/scripts/gmk-evo-x2/freetoken-swap-service/models.toml.example new file mode 100644 index 0000000000..8501e3a283 --- /dev/null +++ b/scripts/gmk-evo-x2/freetoken-swap-service/models.toml.example @@ -0,0 +1,106 @@ +# What: begin global router settings; why: all model routes need one validated policy boundary. +[router] +# What: allow twenty minutes for a cold upstream response; why: an uncached boot load reached ten minutes on qualified GMKtek EVO-X2 and must not be misclassified as failed. +upstream_timeout_s = 1200 +# What: require the generated bearer credential; why: LAN-exposed inference and management routes must reject anonymous callers. +api_keys = ["REPLACE_WITH_A_GENERATED_SECRET"] +# What: include model aliases in discovery; why: compatibility clients need to discover the identifiers they can submit. +include_aliases_in_list = true +# What: emit loading state while a cold model starts; why: clients should distinguish startup work from a dead connection. +send_loading_state = true +# What: sample performance every five seconds; why: operators need bounded live throughput evidence without excessive overhead. +performance_every_s = 5 +# What: preload the validated GPT-OSS alias; why: this model completed cold qualification and starts reliably inside the boot readiness window. +preload_model = "compat/model-b" +# What: activate the coding profile at startup; why: profile aliases must be deterministic before the first request. +startup_routing_profile = "coding" + +# What: define a warm-first selector; why: compatible requests should reuse residency before forcing an expensive model swap. +[selectors.preferred-model] +# What: choose warm routing; why: the currently ready target minimizes latency and memory churn. +strategy = "warm" +# What: order the two validated targets; why: GPT-OSS is preferred when already warm and Qwen remains the cold fallback. +targets = ["model-b", "model-a"] +# What: provide an operator-facing selector name; why: discovery output should be understandable without reading implementation IDs. +name = "Preferred local model" +# What: explain selector behavior; why: management clients need a concise compatibility contract. +description = "Reuses a ready target before the ordered cold fallback" + +# What: define the startup routing profile; why: profile pins group stable client identifiers separately from concrete model IDs. +[profiles.coding] +# What: describe the profile; why: operators need to know why it is active. +description = "qualified GMKtek EVO-X2 production routing profile" +# What: begin profile pin mappings; why: each stable request prefix needs an explicit destination. +[profiles.coding.pins] +# What: map the generic profile name to the warm selector; why: clients can request a stable ID while residency changes safely. +profile-model = "preferred-model" +# What: reject the disabled prefix; why: explicit negative routing proves unapproved model families do not fall through. +disabled-model = "" + +# What: define the primary Qwen model; why: startup preload requires one concrete, fully allowlisted model profile. +[models.model-a] +# What: point to the exact qualified GGUF; why: production must use the artifact that passed the live AMD matrix. +model = "/home/USER/local_models/freetoken-swap/models/qwen36/Qwen3.6-27B-Q4_K_M.gguf" +# What: request dynamic port allocation; why: the daemon must avoid stale or conflicting internal listeners. +port = 0 +# What: allow fifteen minutes for readiness; why: an uncached boot load reached ten minutes and needs bounded safety margin. +ready_timeout_s = 900 +# What: probe the engine readiness endpoint; why: accepting TCP alone does not prove inference readiness. +check_endpoint = "/ready" +# What: build the proxy URL from the allocated port; why: the router must forward only to the engine it owns. +proxy = "http://127.0.0.1:${PORT}" +# What: rewrite upstream requests to the served ID; why: aliases and profiles must converge on the engine's advertised model name. +use_model_name = "model-a" +# What: keep the upstream timeout below the router timeout; why: the router needs time to translate or clean up an upstream failure. +upstream_timeout_s = 1190 +# What: disable idle eviction for the default model; why: startup residency should persist until an explicit model switch. +ttl_s = 0 +# What: assign normal queue priority; why: neither validated model should silently bypass FIFO ordering. +priority = 0 +# What: provide a readable model label; why: management output should identify the production default clearly. +name = "Qwen 3.6 27B Q4_K_M" +# What: preserve the qualified compatibility alias; why: startup preload and existing canaries use this stable external ID. +aliases = ["compat/model-a"] +# What: pass only qualified engine arguments; why: allowlisted argv avoids shell injection and reproduces the validated AMD envelope. +args = ["--host", "127.0.0.1", "--served-model-name", "model-a", "--max-seq-len-override", "1024", "--num-tokens", "1024", "--max-prefill-length", "256", "--max-running-requests", "1", "--graph", "1", "--cache-type", "naive", "--memory-ratio", "0.90", "--attention-backend", "triton", "--moe-backend", "fused", "--disable-pynccl"] + +# What: begin primary metadata; why: operators need to distinguish production policy from transient qualification settings. +[models.model-a.metadata] +# What: mark the deployment tier; why: API consumers can audit that this is the selected production profile. +tier = "production" +# What: mark the profile owner; why: provenance distinguishes operator policy from model-supplied metadata. +type = "operator" + +# What: define the alternate GPT-OSS model; why: live swapping requires a second concrete, fully qualified target. +[models.model-b] +# What: point to the exact qualified checkpoint; why: production must use the model directory that passed the live AMD matrix. +model = "/home/USER/local_models/freetoken-swap/models/gpt-oss-20b" +# What: request dynamic port allocation; why: the daemon must reserve both the engine and distributed-store listeners safely. +port = 0 +# What: allow fifteen minutes for readiness; why: uncached AMD model loading needs a bounded margin above observed startup time. +ready_timeout_s = 900 +# What: probe the engine readiness endpoint; why: accepting TCP alone does not prove inference readiness. +check_endpoint = "/ready" +# What: build the proxy URL from the allocated port; why: the router must forward only to the engine it owns. +proxy = "http://127.0.0.1:${PORT}" +# What: rewrite upstream requests to the served ID; why: aliases and selectors must converge on the engine's advertised model name. +use_model_name = "model-b" +# What: keep the upstream timeout below the router timeout; why: the router needs time to translate or clean up an upstream failure. +upstream_timeout_s = 1190 +# What: disable idle eviction; why: the selected alternate should remain warm until an explicit switch or service stop. +ttl_s = 0 +# What: assign normal queue priority; why: neither validated model should silently bypass FIFO ordering. +priority = 0 +# What: provide a readable model label; why: management output should identify the alternate clearly. +name = "GPT-OSS 20B" +# What: preserve the qualified compatibility alias; why: existing canaries and clients can address the alternate consistently. +aliases = ["compat/model-b"] +# What: pass only qualified engine arguments; why: allowlisted argv avoids shell injection and reproduces the validated AMD envelope. +args = ["--host", "127.0.0.1", "--served-model-name", "model-b", "--max-seq-len-override", "1024", "--num-tokens", "1024", "--max-prefill-length", "256", "--max-running-requests", "1", "--graph", "1", "--cache-type", "naive", "--memory-ratio", "0.90", "--attention-backend", "triton", "--moe-backend", "fused", "--disable-pynccl"] + +# What: begin alternate metadata; why: operators need to distinguish production policy from transient qualification settings. +[models.model-b.metadata] +# What: mark the deployment tier; why: API consumers can audit that this is an approved production profile. +tier = "production" +# What: mark the profile owner; why: provenance distinguishes operator policy from model-supplied metadata. +type = "operator" diff --git a/scripts/gmk-evo-x2/inspect_rocprof_db.py b/scripts/gmk-evo-x2/inspect_rocprof_db.py new file mode 100644 index 0000000000..32aa42a5b4 --- /dev/null +++ b/scripts/gmk-evo-x2/inspect_rocprof_db.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Inspect a ROCm rocprofv3 SQLite trace without requiring the sqlite3 CLI. + +This GMKtek EVO-X2 helper is deliberately read-only. It inventories the database +schema first, then prints one representative row from each trace table so a +subsequent aggregation can use the exact ROCm-version-specific column names. +""" + +from __future__ import annotations + +import argparse +import sqlite3 +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + """Accept the immutable profiler database to inspect.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("database", type=Path, help="rocprofv3 *_results.db artifact") + parser.add_argument( + "--tail-seconds", + type=float, + default=0.0, + help="aggregate only the final positive-duration kernel window, zero prints schema only", + ) + return parser.parse_args() + + +def main() -> int: + """Print a compact schema inventory and representative records, then exit.""" + + args = parse_args() + if not args.database.is_file(): + raise SystemExit(f"missing profiler database: {args.database}") + + # Open the evidence database in immutable read-only mode so inspection cannot + # create journal files or change the captured trace under any circumstances. + uri = f"file:{args.database.resolve()}?mode=ro&immutable=1" + connection = sqlite3.connect(uri, uri=True) + connection.row_factory = sqlite3.Row + try: + # SQLite's catalog is the authoritative list of ROCm trace tables. + rows = connection.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name" + ).fetchall() + tables = [row["name"] for row in rows] + for table in tables: + # Quote table names defensively even though rocprof creates them. + quoted = '"' + table.replace('"', '""') + '"' + columns = connection.execute(f"PRAGMA table_info({quoted})").fetchall() + column_names = [column["name"] for column in columns] + count = connection.execute(f"SELECT COUNT(*) FROM {quoted}").fetchone()[0] + print(f"TABLE {table} rows={count} columns={','.join(column_names)}") + # A single row gives the names and units needed for a version-safe + # aggregate without dumping the large raw trace into the terminal. + sample = connection.execute(f"SELECT * FROM {quoted} LIMIT 1").fetchone() + if sample is not None: + values = ";".join(f"{key}={sample[key]!r}" for key in sample.keys()) + print(f"SAMPLE {table} {values}") + if args.tail_seconds > 0: + # rocprof version-stamps every table name with one UUID. Selecting + # by prefix keeps this analysis portable across ROCm trace versions. + dispatch = next(name for name in tables if name.startswith("rocpd_kernel_dispatch_")) + symbols = next(name for name in tables if name.startswith("rocpd_info_kernel_symbol_")) + quoted_dispatch = '"' + dispatch.replace('"', '""') + '"' + quoted_symbols = '"' + symbols.replace('"', '""') + '"' + # Timestamps are nanoseconds. The final active dispatch is a stable + # anchor because the profiler may remain alive after request work ends. + last_end = connection.execute( + f"SELECT MAX(end) FROM {quoted_dispatch} WHERE end > start" + ).fetchone()[0] + cutoff = last_end - int(args.tail_seconds * 1_000_000_000) + aggregate = connection.execute( + f""" + SELECT s.kernel_name AS kernel, + COUNT(*) AS calls, + SUM(d.end - d.start) AS gpu_ns + FROM {quoted_dispatch} AS d + JOIN {quoted_symbols} AS s ON s.id = d.kernel_id + WHERE d.end > d.start AND d.end >= ? + GROUP BY s.kernel_name + ORDER BY gpu_ns DESC + LIMIT 40 + """, + (cutoff,), + ).fetchall() + print(f"TAIL_WINDOW seconds={args.tail_seconds:g} cutoff_ns={cutoff} last_end_ns={last_end}") + for row in aggregate: + print(f"KERNEL calls={row['calls']} gpu_ms={row['gpu_ns'] / 1e6:.3f} name={row['kernel']}") + finally: + connection.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/launch_qwen_gguf_qualified.sh b/scripts/gmk-evo-x2/launch_qwen_gguf_qualified.sh new file mode 100755 index 0000000000..da7f06bcfd --- /dev/null +++ b/scripts/gmk-evo-x2/launch_qwen_gguf_qualified.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +# Start or stop the qualified GMKtek EVO-X2 Qwen3.6 Q4_K_M FreeToken test server. +# +# This helper is deliberately limited to the isolated loopback test port. It +# does not start the normal NVFP4 service, contact llama-swap, change system +# swap policy, or make a model available on the LAN. The start action puts the +# entire FreeToken multiprocessing tree in its own session and process group. +# The stop action verifies that group before stopping it, which prevents the +# orphaned distributed worker and internal-port collision observed during the +# Q4 SVM-resident-memory investigation. + +set -euo pipefail + +# Require a deliberate lifecycle action instead of guessing whether a caller +# intended to start a service or release the GPU for a llama.cpp control. +readonly ACTION="${1:?usage: launch_qwen_gguf_qualified.sh start|stop ARTIFACT_DIR [MEMORY_RATIO] [CUDA_GRAPH_MAX_BS]}" +# Require a caller-owned evidence directory. The script writes only its PID +# file and server log there, so every test run preserves its own provenance. +readonly ARTIFACT_DIR="${2:?usage: launch_qwen_gguf_qualified.sh start|stop ARTIFACT_DIR [MEMORY_RATIO] [CUDA_GRAPH_MAX_BS]}" +# Keep the memory-safe recovery profile as the explicit default. Callers may +# supply a different ratio for a recorded experiment, never for a silent +# production configuration change. +readonly MEMORY_RATIO="${3:-0.25}" +# Leave decode graph replay disabled unless an experiment explicitly requests a +# bounded capture size. The production profile and every existing qualified +# baseline therefore retain the exact eager-decode behavior. A value of one +# captures only the single-stream decode shape used by this Q4 benchmark. +readonly CUDA_GRAPH_MAX_BS="${4:-0}" + +# Keep durable models, kernel caches, and artifacts separate from the checked +# out source so source switching cannot delete benchmark evidence or weights. +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +# This is the isolated Q4-capable checkout used for the native GGUF controls. +# A caller may select a separately created candidate worktree for a recorded +# experiment, but the validation below limits that override to this host's +# dedicated FreeToken source area and never changes the protected live server. +readonly SOURCE_DIR="${FREETOKEN_Q4_SOURCE_DIR:-${ROOT_DIR}/source-qwen-gguf-5c7f0fd}" +# The exact file is also used by the matching ROCm llama.cpp control. +readonly MODEL_PATH="${ROOT_DIR}/models/controls/qwen36-35b-a3b-unsloth-a483e9e6/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf" +# Preserve the shared checkpoint tokenizer for API and benchmark token counts. +readonly SERVED_MODEL="qwen36-35b-a3b-q4km-gguf-amd" +# Restrict this helper to the disposable loopback endpoint, never port 1919. +readonly PORT="1922" +# The FreeToken engine creates a local distributed TCP store on this next port. +# An existing listener means an earlier multiprocessing group was not cleaned. +readonly INTERNAL_PORT="1923" +# Keep HIP extension artifacts in the revisioned shared cache established by +# the native strict no-JIT qualification rather than compiling per run. +# Keep the default reusable extension cache for the qualified control, while +# allowing an isolated candidate to select a dedicated cache directory. The +# validation below confines that override to the managed cache area so a shell +# variable cannot redirect native-build output into the normal source tree. +readonly EXTENSION_CACHE="${FREETOKEN_Q4_EXTENSION_CACHE_DIR:-${ROOT_DIR}/cache/torch_extensions}" +# Store lifecycle data next to the supplied immutable test artifact. +readonly PID_FILE="${ARTIFACT_DIR}/server.pid" +readonly LOG_FILE="${ARTIFACT_DIR}/server.log" +# Preserve native-extension build and import evidence in the same immutable +# artifact because a clean Git worktree does not contain generated HIP modules. +readonly NATIVE_BUILD_LOG="${ARTIFACT_DIR}/native-extension-build.log" +readonly NATIVE_IMPORT_LOG="${ARTIFACT_DIR}/native-extension-import.txt" + +# Resolve a listener PID without assuming that a stale PID file identifies the +# live owner of a TCP port. An empty answer is a valid no-listener condition. +listener_pid() { + local port="$1" + ss -ltnp "( sport = :${port} )" | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | head -1 +} + +# Return success only for the known test-server command. This is the guard +# that makes a PID or process-group signal safe in a shared GMKtek EVO-X2 shell. +is_qualified_q4_process() { + local pid="$1" + local command + [[ "${pid}" =~ ^[0-9]+$ ]] || return 1 + [[ -r "/proc/${pid}/cmdline" ]] || return 1 + command="$(tr '\0' ' ' < "/proc/${pid}/cmdline")" + [[ "${command}" == *"freetoken.cli serve"* ]] && + [[ "${command}" == *"${MODEL_PATH}"* ]] && + [[ "${command}" == *"--port ${PORT}"* ]] +} + +# Stop a dedicated session only after proving the main process owns that group. +# `setsid` makes the server PID, session ID, and process-group ID equal, so one +# signal reaches the HTTP parent, scheduler, tokenizer worker, and tracker. +stop_qualified_group() { + local pid="$1" + local pgid + is_qualified_q4_process "${pid}" || { + echo "refusing to stop an unrecognized process: ${pid}" >&2 + return 1 + } + pgid="$(ps -o pgid= -p "${pid}" | tr -d ' ')" + [[ "${pgid}" == "${pid}" ]] || { + echo "refusing to stop process ${pid}: expected dedicated process group, got ${pgid}" >&2 + return 1 + } + kill -TERM -- "-${pgid}" || true + for _ in $(seq 1 30); do + kill -0 "${pid}" 2>/dev/null || break + sleep 1 + done + # A stuck HIP kernel can prevent graceful Python exit. Escalate only the + # already-verified dedicated group after the bounded graceful wait. + kill -0 "${pid}" 2>/dev/null && kill -KILL -- "-${pgid}" || true +} + +# Validate fixed paths before any lifecycle action so a changed layout fails +# closed rather than starting a different model or a CPU fallback. +validate_paths() { + [[ "${SOURCE_DIR}" == "${ROOT_DIR}/source-qwen-"* ]] || { + echo "source directory must be an isolated Qwen checkout under ${ROOT_DIR}" >&2 + return 1 + } + [[ -d "${SOURCE_DIR}" ]] || { echo "missing source directory: ${SOURCE_DIR}" >&2; return 1; } + [[ -f "${MODEL_PATH}" ]] || { echo "missing Q4 model: ${MODEL_PATH}" >&2; return 1; } + [[ -x "${ROOT_DIR}/.venv/bin/python" ]] || { echo "missing benchmark Python" >&2; return 1; } + [[ "${MEMORY_RATIO}" =~ ^0\.[0-9]+$|^1\.0+$ ]] || { echo "invalid memory ratio: ${MEMORY_RATIO}" >&2; return 1; } + [[ "${CUDA_GRAPH_MAX_BS}" =~ ^[0-9]+$ ]] || { echo "invalid CUDA graph max batch size: ${CUDA_GRAPH_MAX_BS}" >&2; return 1; } + [[ "${EXTENSION_CACHE}" == "${ROOT_DIR}/cache/"* ]] || { echo "extension cache must be under ${ROOT_DIR}/cache" >&2; return 1; } +} + +case "${ACTION}" in + start) + validate_paths + # Artifacts must be unique so a retry cannot overwrite the first log. + [[ -e "${PID_FILE}" ]] && { echo "PID file already exists: ${PID_FILE}" >&2; exit 2; } + # The HTTP and internal distributed ports must both be clear. The + # internal-port check detects orphaned workers before touching the GPU. + [[ -z "$(listener_pid "${PORT}")" ]] || { echo "test port ${PORT} is already listening" >&2; exit 2; } + [[ -z "$(listener_pid "${INTERNAL_PORT}")" ]] || { echo "internal port ${INTERNAL_PORT} is already listening" >&2; exit 2; } + mkdir -p "${ARTIFACT_DIR}" + cd "${SOURCE_DIR}" + # The Q4 MoE path requires this in-tree HIP extension. Build it only + # when the isolated source lacks it, and retain the compiler outcome so + # a later benchmark cannot silently rely on a different checkout. + if ! PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" -c 'import freetoken.kernel._pinned_tensor' >/dev/null 2>&1; then + ROCM_HOME=/opt/rocm-10.0 ROCM_PATH=/opt/rocm-10.0 HIP_PATH=/opt/rocm-10.0 \ + PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" setup.py build_ext --inplace \ + >"${NATIVE_BUILD_LOG}" 2>&1 + fi + PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" -c \ + 'import freetoken.kernel._pinned_tensor as pinned; print(pinned.__file__)' \ + >"${NATIVE_IMPORT_LOG}" + # Set only this child environment. ROCm paths select the native HIP + # stack, while PYTHONPATH and TORCH_EXTENSIONS_DIR select the reviewed + # Q4 source and prebuilt extension cache without changing the login + # shell or normal service environment. + ROCM_HOME=/opt/rocm-10.0 ROCM_PATH=/opt/rocm-10.0 HIP_PATH=/opt/rocm-10.0 \ + PYTHONPATH=python TORCH_EXTENSIONS_DIR="${EXTENSION_CACHE}" \ + setsid nohup "${ROOT_DIR}/.venv/bin/python" -m freetoken.cli serve \ + --model-path "${MODEL_PATH}" \ + --served-model-name "${SERVED_MODEL}" \ + --host 127.0.0.1 --port "${PORT}" \ + --max-running-requests 4 \ + --attention-backend triton --moe-backend offload --nvfp4-backend triton \ + --expert-load serial --moe-cache-auto --memory-ratio "${MEMORY_RATIO}" \ + --max-seq-len-override 8192 --kv-reserve-tokens 8192 \ + --cuda-graph-max-bs "${CUDA_GRAPH_MAX_BS}" --disable-pynccl --disable-moe-prefill-overlap \ + >"${LOG_FILE}" 2>&1 & + echo "$!" >"${PID_FILE}" + ;; + stop) + # Prefer the recorded server PID, but verify it before a signal. This + # keeps a malformed artifact from targeting an unrelated user process. + [[ -f "${PID_FILE}" ]] || { echo "missing server PID file: ${PID_FILE}" >&2; exit 2; } + recorded_pid="$(<"${PID_FILE}")" + # A backend-startup failure can already have exited the frontend before + # the controller's EXIT trap runs. Treat that absent recorded process as + # successful cleanup rather than blocking normal-service recovery. + kill -0 "${recorded_pid}" 2>/dev/null || exit 0 + stop_qualified_group "${recorded_pid}" + ;; + *) + echo "unknown action: ${ACTION}; expected start or stop" >&2 + exit 2 + ;; +esac diff --git a/scripts/gmk-evo-x2/qwen36_exact_gguf_contract.py b/scripts/gmk-evo-x2/qwen36_exact_gguf_contract.py new file mode 100644 index 0000000000..e90cd38a7a --- /dev/null +++ b/scripts/gmk-evo-x2/qwen36_exact_gguf_contract.py @@ -0,0 +1,99 @@ +"""Run an explicit local-model contract check for the Qwen3.6 mixed GGUF loader.""" + +# Import argument parsing so operators must name the private model artifact explicitly. +import argparse + +# Import Torch for CPU/meta model construction without allocating production model memory. +import torch + +# Import tensor-parallel setup because model construction reads the active TP identity. +from freetoken.distributed.info import set_tp_info +# Import rotary-device setup because meta construction still requires an explicit safe device. +from freetoken.layers.rotary import set_rope_device +# Import the GGUF shim builder so the private artifact drives the exact model configuration. +from freetoken.models.gguf.config import build_gguf_shim +# Import the Qwen parser so the check uses the same architecture contract as runtime loading. +from freetoken.models.qwen3_5_moe.config import parse_gguf_config +# Import the production iterator so validation covers the real GGUF tensor mapping. +from freetoken.models.qwen3_5_moe.gguf import iter_gguf_weights +# Import model registration so the configured architecture resolves through the public runtime registry. +from freetoken.models.register import get_model_class +# Import the dtype context because meta construction must match runtime bfloat16 expectations. +from freetoken.utils.torch_utils import torch_dtype + + +def require(condition: bool, detail: object) -> None: + """Raise an optimization-safe contract failure with the supplied diagnostic.""" + # Reject false conditions explicitly because Python optimization can remove ordinary assert statements. + if not condition: + # Preserve the mismatch detail so operators can identify the exact tensor or tokenizer contract. + raise RuntimeError(f"Qwen GGUF contract failed: {detail}") + + +def validate(model_path: str, *, check_tokenizer: bool) -> None: + """Validate exact non-MoE tensor shape/dtype coverage and optional tokenizer round-trip.""" + # Configure one CPU rank because the local check is intentionally isolated from distributed launch state. + set_tp_info(0, 1) + # Pin rotary metadata to CPU so validation cannot accidentally initialize a GPU runtime. + set_rope_device(torch.device("cpu")) + # Parse the exact private artifact through the production GGUF configuration path. + config = parse_gguf_config(build_gguf_shim(model_path)) + # Build only metadata tensors so the contract check remains bounded on systems without model-sized RAM. + with torch.device("meta"), torch_dtype(torch.bfloat16): + # Resolve and instantiate the same registered model class selected during production serving. + model = get_model_class(config.architectures[0], config) + # Copy expected state keys so every observed GGUF tensor can remove exactly one contract entry. + expected = model.state_dict() + # Iterate non-expert tensors because the exact expert payload is validated by its separate streamed path. + for name, value in iter_gguf_weights( + model_path, + device="cpu", + include_moe_experts=False, + include_non_moe=True, + ): + # Remove the matching destination so unknown or duplicate names fail at their point of use. + target = expected.pop(name) + # Require shape equality because a shape-compatible load is the minimum safe tensor contract. + require(target.shape == value.shape, (name, target.shape, value.shape)) + # Require dtype equality because implicit conversion would invalidate the exact-file contract. + require(target.dtype == value.dtype, (name, target.dtype, value.dtype)) + # Require complete destination coverage so silently omitted tensors cannot pass the manual validator. + require(not expected, sorted(expected)) + # Emit a stable marker for private automation that records this bounded validation result. + print("EXACT_GGUF_STATE_CONTRACT_OK") + # Skip tokenizer work unless requested because tokenizer loading adds private-artifact I/O. + if not check_tokenizer: + # Return after the required tensor contract because no optional tokenizer gate was selected. + return + # Import lazily so tensor-only validation does not load tokenizer dependencies unnecessarily. + from freetoken.models.gguf.tokenizer import load_gguf_tokenizer + # Load the tokenizer from the same exact artifact to prevent cross-checkpoint substitution. + tokenizer = load_gguf_tokenizer(model_path) + # Use a stable visible string because the check is round-trip integrity, not model quality. + text = "Hello, model." + # Require exact decode equality so tokenizer normalization drift is visible to the operator. + require(tokenizer.decode(tokenizer.encode(text, add_special_tokens=False)) == text, text) + # Emit a separate marker so automation can distinguish tensor-only and tokenizer-complete runs. + print("TOKENIZER_ROUND_TRIP_OK") + + +def main(argv: list[str] | None = None) -> int: + """Parse explicit private inputs and run the local contract validator.""" + # Create the CLI parser inside main so importing this module never consumes pytest or caller arguments. + parser = argparse.ArgumentParser(description=__doc__) + # Require the local model path because the repository intentionally contains no private checkpoint. + parser.add_argument("model", help="Local Qwen35 GGUF file to validate on CPU/meta") + # Make tokenizer validation opt-in because it is slower and independent from tensor shape coverage. + parser.add_argument("--tokenizer", action="store_true", help="Also verify a tokenizer text round-trip") + # Parse only the caller-supplied arguments so tests and wrappers can invoke main deterministically. + args = parser.parse_args(argv) + # Run the complete selected validation before reporting success. + validate(args.model, check_tokenizer=args.tokenizer) + # Return zero only after every selected contract gate completes. + return 0 + + +# Execute the CLI only when invoked as a script so imports remain side-effect free. +if __name__ == "__main__": + # Propagate the explicit status code to shells and automation. + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/run_gemma4_gguf_text_control.sh b/scripts/gmk-evo-x2/run_gemma4_gguf_text_control.sh new file mode 100755 index 0000000000..bc5c37015c --- /dev/null +++ b/scripts/gmk-evo-x2/run_gemma4_gguf_text_control.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# Launch Gemma4 Q4 GGUF in an isolated GMKtek EVO-X2 control slot and restore Qwen. + +set -euo pipefail + +readonly CHECKOUT="${1:?usage: run_gemma4_gguf_text_control.sh ISOLATED_CHECKOUT}" +readonly MODE="${2:-text}" +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +# Bind recovery to the maintained Qwen source tree. The historical harness +# checkout was intentionally retired, so referring to it would let a Gemma +# control finish with the protected API still unavailable. +readonly PRODUCTION_DIR="${ROOT_DIR}/source-qwen-recovery-d6ee8cef479c" +readonly MODEL_PATH="${ROOT_DIR}/models/Gemma-4-26B-A4B-it-qat-q4_0-gguf/gemma-4-26B_q4_0-it.gguf" +readonly TEST_PORT="1923" +readonly PRODUCTION_PORT="1919" +readonly ARTIFACT_DIR="${ROOT_DIR}/artifacts/gemma4-gguf-${MODE}-$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "${ARTIFACT_DIR}" + +port_pid() { ss -ltnp "( sport = :$1 )" | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | head -1; } +production_ready() { + # A TCP listener and a 200 response can both exist while FreeToken is still + # loading its expert groups. Inspect the authoritative health status so a + # temporary candidate never begins while Qwen is only partially recovered. + timeout 5 curl -fsS "http://127.0.0.1:${PRODUCTION_PORT}/health" | grep -q '"status":"ok"' +} +restore_production() { + local test_pid + test_pid="$(port_pid "${TEST_PORT}")" + if [[ -n "${test_pid}" ]]; then + # Do not race the Qwen recovery process against the temporary Gemma + # process still releasing its ROCm context. A bare kill followed by an + # immediate recovery launch intermittently produced an empty Qwen log + # and a dead child on GMKtek EVO-X2. + kill "${test_pid}" || true + for _ in {1..30}; do + kill -0 "${test_pid}" 2>/dev/null || break + sleep 1 + done + # The port owner can exit before HIP finishes tearing down its GPU + # context. Give ROCm a bounded grace period before Qwen tries to claim + # the device, avoiding a child that dies before it can write server.log. + sleep 10 + fi + if ! production_ready; then + # The recovery script is intentionally external and protects the source + # checkout. Verify its observable health result rather than treating a + # background PID or an artifact-directory print as successful recovery. + local recovered=0 + # Launch exactly once. Qwen takes several minutes to load its three + # serial NVFP4 expert groups on GMKtek EVO-X2. Retrying the launcher while + # its listener already exists only produces a misleading refusal and + # wastes the short recovery window. + bash "${PRODUCTION_DIR}/scripts/gmk-evo-x2/start_qwen_recovery_server.sh" \ + | tee -a "${ARTIFACT_DIR}/recovery.log" || true + # The protected model normally needs roughly six to eight minutes from + # a cold recovery. Wait a bounded eight minutes for the authoritative + # ready status, rather than mistaking a temporary loading response for + # success or declaring a healthy in-progress recovery a failure. + for _ in {1..480}; do + production_ready && { + recovered=1 + break + } + sleep 1 + done + [[ "${recovered}" == "1" ]] || echo "WARNING: Qwen recovery did not become reachable" >&2 + fi +} +trap restore_production EXIT + +case "${MODE}" in + text|vision) ;; + *) echo "mode must be text or vision, got ${MODE}" >&2; exit 2 ;; +esac + +# Keep the conservative production-shaped setting as the default. The +# explicit opt-in is used only by an isolated candidate run while investigating +# Gemma prefill and first-token latency. Keeping the option in an array avoids +# shell edits that can accidentally remove the command's log redirection or +# background-process marker. +moe_prefill_args=(--disable-moe-prefill-overlap) +if [[ "${FREETOKEN_GEMMA4_PREFILL_OVERLAP:-0}" == "1" ]]; then + moe_prefill_args=() +fi + +# The default preserves the qualified memory budget. Candidate runs may raise +# this value to test whether keeping more Gemma expert material resident in the +# unified GPU-visible memory improves prefill without changing model weights or +# the request protocol. +readonly GEMMA_MEMORY_RATIO="${FREETOKEN_GEMMA4_MEMORY_RATIO:-0.35}" +readonly GEMMA_MAX_RUNNING_REQUESTS="${FREETOKEN_GEMMA4_MAX_RUNNING_REQUESTS:-4}" + +# Refuse to evict the protected service during its multi-minute NVFP4 recovery. +production_ready + +production_pid="$(port_pid "${PRODUCTION_PORT}")" +[[ -z "${production_pid}" ]] || kill "${production_pid}" +for _ in {1..60}; do ss -ltn "( sport = :${PRODUCTION_PORT} )" | grep -q "${PRODUCTION_PORT}" || break; sleep 1; done + +# The preceding time-share and recovery tests can leave cold pages in the host +# swap file even when enough RAM is currently free. Qwen has released its +# memory before this point, so cycling the already-configured swap file is a +# bounded way to give the isolated Gemma candidate a clean measurement start. +# This does not resize swap or change the host's vm.swappiness policy. +sudo swapoff -a +sudo swapon -a +swapon --show --bytes >"${ARTIFACT_DIR}/swap-after-qwen-release.txt" + +cd "${CHECKOUT}" +vision_env=() +if [[ "${MODE}" == "vision" ]]; then + # This explicit opt-in causes the isolated Gemma candidate to allocate and + # load its sibling 1.2 GiB mmproj vision tower. Text mode preserves the + # normal production memory budget. + vision_env=(FREETOKEN_LOAD_VISION=1) + # The embedding fingerprint is a temporary parity aid. Preserve its explicit + # caller opt-in so ordinary vision controls never synchronize the device to + # compute debug statistics or expand the normally concise server log. + if [[ "${FREETOKEN_GEMMA4_VISION_DEBUG:-}" == "1" ]]; then + vision_env+=(FREETOKEN_GEMMA4_VISION_DEBUG=1) + fi +fi +# ``env`` is required here: an expanded Bash array is not parsed as assignment +# words, so placing ``${vision_env[@]}`` before ``nohup`` directly would try to +# execute the literal ``FREETOKEN_LOAD_VISION=1`` string as a program. +env ROCM_HOME=/opt/rocm-10.0 ROCM_PATH=/opt/rocm-10.0 HIP_PATH=/opt/rocm-10.0 \ +FREETOKEN_ROCM_PREFILL_WARMUP="${FREETOKEN_ROCM_PREFILL_WARMUP:-1}" \ +PYTHONPATH=python TORCH_EXTENSIONS_DIR="${ROOT_DIR}/cache/torch_extensions" \ +"${vision_env[@]}" nohup "${ROOT_DIR}/.venv/bin/python" -m freetoken.cli serve \ + --model-path "${MODEL_PATH}" --served-model-name gemma4-26b-q4-amd \ + --host 127.0.0.1 --port "${TEST_PORT}" --attention-backend triton \ + --moe-backend offload --expert-load serial --moe-cache-auto --memory-ratio "${GEMMA_MEMORY_RATIO}" \ + --max-running-requests "${GEMMA_MAX_RUNNING_REQUESTS}" \ + --max-seq-len-override 8192 --kv-reserve-tokens 2048 --cuda-graph-max-bs 0 \ + --disable-pynccl "${moe_prefill_args[@]}" >"${ARTIFACT_DIR}/server.log" 2>&1 & +candidate_pid=$! +for _ in {1..480}; do + grep -q 'API server is ready to serve' "${ARTIFACT_DIR}/server.log" && break + kill -0 "${candidate_pid}" 2>/dev/null || exit 1 + sleep 1 +done +grep -q 'API server is ready to serve' "${ARTIFACT_DIR}/server.log" +curl -fsS "http://127.0.0.1:${TEST_PORT}/health" >"${ARTIFACT_DIR}/health.json" +# Capture the environment as observed by the actual candidate process, rather +# than assuming a wrapper-level export survived ``nohup`` and multiprocessing. +# This artifact is written only for the parity diagnostic and contains solely +# the named boolean flag, never the server's complete environment. +if [[ "${FREETOKEN_GEMMA4_VISION_DEBUG:-}" == "1" ]]; then + tr '\0' '\n' <"/proc/${candidate_pid}/environ" | \ + grep '^FREETOKEN_GEMMA4_VISION_DEBUG=' >"${ARTIFACT_DIR}/vision-debug-env.txt" || true +fi +PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/verify_gemma4_gguf_text.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model gemma4-26b-q4-amd \ + --gguf "${MODEL_PATH}" --artifact "${ARTIFACT_DIR}/quality.json" \ + >"${ARTIFACT_DIR}/quality.log" 2>&1 + +if [[ "${FREETOKEN_GEMMA4_MATRIX:-}" == "1" ]]; then + # The short arithmetic gate above remains mandatory. This opt-in matrix + # runs only after quality passes and records fixed-length warmup, prefill, + # decode, TTFT, and token-gap evidence in the same immutable artifact. + PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/benchmark_gemma4_gguf_text_matrix.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model gemma4-26b-q4-amd \ + --gguf "${MODEL_PATH}" --samples "${FREETOKEN_GEMMA4_MATRIX_SAMPLES:-5}" \ + --max-tokens "${FREETOKEN_GEMMA4_MATRIX_TOKENS:-128}" \ + --prompt-repeat "${FREETOKEN_GEMMA4_PROMPT_REPEAT:-1}" \ + --artifact "${ARTIFACT_DIR}/text-matrix.json" \ + >"${ARTIFACT_DIR}/text-matrix.log" 2>&1 +fi + +if [[ "${FREETOKEN_GEMMA4_CONCURRENCY:-}" == "1" ]]; then + PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/benchmark_gemma4_concurrency.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model gemma4-26b-q4-amd \ + --clients "${FREETOKEN_GEMMA4_CLIENTS:-4}" --rounds "${FREETOKEN_GEMMA4_ROUNDS:-3}" \ + --max-tokens "${FREETOKEN_GEMMA4_MATRIX_TOKENS:-128}" --artifact "${ARTIFACT_DIR}/concurrency.json" \ + --prompt-repeat "${FREETOKEN_GEMMA4_PROMPT_REPEAT:-1}" \ + >"${ARTIFACT_DIR}/concurrency.log" 2>&1 +fi + +if [[ "${FREETOKEN_GEMMA4_LONG_CONTEXT:-}" == "1" ]]; then + PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/benchmark_gemma4_long_context.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model gemma4-26b-q4-amd \ + --artifact "${ARTIFACT_DIR}/long-context.json" >"${ARTIFACT_DIR}/long-context.log" 2>&1 +fi + +if [[ "${FREETOKEN_GEMMA4_ENDURANCE:-}" == "1" ]]; then + PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/benchmark_gemma4_endurance.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model gemma4-26b-q4-amd \ + --sessions "${FREETOKEN_GEMMA4_SESSIONS:-30}" --interval "${FREETOKEN_GEMMA4_INTERVAL:-1}" \ + --artifact "${ARTIFACT_DIR}/endurance.json" >"${ARTIFACT_DIR}/endurance.log" 2>&1 +fi + +if [[ "${MODE}" == "vision" ]]; then + # Keep the candidate alive through the actual OpenAI image_url contract + # control. The verifier writes a self-contained response/usage artifact; + # only after it succeeds does the EXIT trap reclaim port 1923 and restore + # the protected Qwen server. + image_verify_args=() + if [[ "${FREETOKEN_GEMMA4_EXTENDED:-}" == "1" ]]; then + # The core three-fixture gate stays fast enough for every normal + # candidate. This explicit option adds color and spatial-direction + # regression controls after the core pipeline has already passed. + image_verify_args+=(--extended) + fi + if [[ -n "${FREETOKEN_GEMMA4_IMAGE_REPETITIONS:-}" ]]; then + # The verifier validates this as a positive integer. Keeping the value + # in the environment lets an operator request a repeatability campaign + # without changing the normal short candidate-control behavior. + image_verify_args+=(--repetitions "${FREETOKEN_GEMMA4_IMAGE_REPETITIONS}") + fi + PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/verify_gemma4_gguf_image.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model gemma4-26b-q4-amd \ + --stream "${image_verify_args[@]}" --artifact "${ARTIFACT_DIR}/image-quality.json" \ + >"${ARTIFACT_DIR}/image-quality.log" 2>&1 + # The long-response fixture supplies an output-length quality gate, which + # makes its stream timing suitable for a visual decode-TPS measurement. + PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/verify_gemma4_gguf_visual_tps.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model gemma4-26b-q4-amd \ + --artifact "${ARTIFACT_DIR}/visual-tps.json" \ + >"${ARTIFACT_DIR}/visual-tps.log" 2>&1 +fi diff --git a/scripts/gmk-evo-x2/run_gemma4_llamacpp_vision_control.sh b/scripts/gmk-evo-x2/run_gemma4_llamacpp_vision_control.sh new file mode 100755 index 0000000000..d4428bdb3b --- /dev/null +++ b/scripts/gmk-evo-x2/run_gemma4_llamacpp_vision_control.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Run Gemma4 Q4 plus its vision projector through the ROCm 10 llama.cpp control. +# +# This is the matched comparison companion to run_gemma4_gguf_text_control.sh: +# it uses the identical text GGUF, sibling mmproj file, loopback isolation, text +# question, and OpenAI data-URL image fixtures. It never modifies llama-swap or +# the protected Qwen source checkout. + +set -euo pipefail + +readonly CHECKOUT="${1:?usage: run_gemma4_llamacpp_vision_control.sh ISOLATED_CHECKOUT}" +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +# Use the maintained recovery checkout so a matched llama.cpp control cannot +# finish with the protected Qwen API unavailable because of a retired path. +readonly PRODUCTION_DIR="${ROOT_DIR}/source-qwen-recovery-d6ee8cef479c" +readonly LLAMA_SERVER="${ROOT_DIR}/llama.cpp-rocm10-b10141/build-rocm10-clang/bin/llama-server" +readonly MODEL_PATH="${ROOT_DIR}/models/Gemma-4-26B-A4B-it-qat-q4_0-gguf/gemma-4-26B_q4_0-it.gguf" +readonly MMPROJ_PATH="${ROOT_DIR}/models/Gemma-4-26B-A4B-it-qat-q4_0-gguf/gemma-4-26B-it-mmproj.gguf" +readonly TEST_PORT="1924" +readonly PRODUCTION_PORT="1919" +readonly MODEL_NAME="gemma4-26b-q4-llamacpp-rocm10" +readonly ARTIFACT_DIR="${ROOT_DIR}/artifacts/gemma4-llamacpp-vision-$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "${ARTIFACT_DIR}" + +port_pid() { ss -ltnp "( sport = :$1 )" | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | head -1; } +production_ready() { + timeout 5 curl -fsS "http://127.0.0.1:${PRODUCTION_PORT}/health" | grep -q '"status":"ok"' +} +restore_production() { + local test_pid recovered + test_pid="$(port_pid "${TEST_PORT}")" + if [[ -n "${test_pid}" ]]; then + kill "${test_pid}" || true + for _ in {1..30}; do kill -0 "${test_pid}" 2>/dev/null || break; sleep 1; done + # HIP teardown outlives the listener. Do not race the next ROCm process. + sleep 10 + fi + if ! production_ready; then + recovered=0 + # Start only once. The serial NVFP4 Qwen load on GMKtek EVO-X2 lasts minutes; + # retrying its launcher after the listener exists merely reports a + # refusal and shortens the useful ready-status wait. + bash "${PRODUCTION_DIR}/scripts/gmk-evo-x2/start_qwen_recovery_server.sh" \ + | tee -a "${ARTIFACT_DIR}/recovery.log" || true + # Keep the benchmark process alive until Qwen is actually serving, up + # to the known cold-start envelope, not merely until health answers. + for _ in {1..480}; do + production_ready && { + recovered=1 + break + } + sleep 1 + done + [[ "${recovered}" == "1" ]] || echo "WARNING: Qwen recovery did not become reachable" >&2 + fi +} +trap restore_production EXIT + +[[ -x "${LLAMA_SERVER}" ]] || { echo "missing llama-server: ${LLAMA_SERVER}" >&2; exit 2; } +[[ -f "${MODEL_PATH}" && -f "${MMPROJ_PATH}" ]] || { echo "missing Gemma GGUF or mmproj" >&2; exit 2; } +production_ready + +production_pid="$(port_pid "${PRODUCTION_PORT}")" +[[ -z "${production_pid}" ]] || kill "${production_pid}" +for _ in {1..60}; do ss -ltn "( sport = :${PRODUCTION_PORT} )" | grep -q "${PRODUCTION_PORT}" || break; sleep 1; done + +# Release stale pages only after the protected FreeToken process has exited. +# This preserves the host's swap-file size and swappiness policy while giving +# the standalone projector control a clean shared-memory baseline. +sudo swapoff -a +sudo swapon -a +swapon --show --bytes >"${ARTIFACT_DIR}/swap-after-qwen-release.txt" + +# Use the same ROCm 10 libraries, full text-model and projector offload, Q8 KV, +# Flash Attention, one slot, and 8,192-token context as the existing Qwen +# llama.cpp controls. The projector is explicit so no download or auto-selection +# alters the comparison. +export LD_LIBRARY_PATH="/opt/rocm-10.0/llvm/lib:/opt/rocm-10.0/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" +"${LLAMA_SERVER}" -m "${MODEL_PATH}" -mm "${MMPROJ_PATH}" --mmproj-offload \ + --alias "${MODEL_NAME}" -ngl all -c 8192 -np 1 -b 2048 -ub 512 \ + -ctk q8_0 -ctv q8_0 -fa on --jinja --no-context-shift --no-warmup \ + --reasoning off --reasoning-budget 0 \ + --host 127.0.0.1 --port "${TEST_PORT}" >"${ARTIFACT_DIR}/server.log" 2>&1 & +candidate_pid=$! +for _ in {1..240}; do + timeout 5 curl -fsS "http://127.0.0.1:${TEST_PORT}/health" >"${ARTIFACT_DIR}/health.json" && break + kill -0 "${candidate_pid}" 2>/dev/null || { tail -120 "${ARTIFACT_DIR}/server.log" >&2; exit 1; } + sleep 1 +done +test -s "${ARTIFACT_DIR}/health.json" + +cd "${CHECKOUT}" +PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/verify_gemma4_gguf_text.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model "${MODEL_NAME}" \ + --gguf "${MODEL_PATH}" --artifact "${ARTIFACT_DIR}/quality.json" \ + >"${ARTIFACT_DIR}/quality.log" 2>&1 + +if [[ "${FREETOKEN_GEMMA4_MATRIX:-}" == "1" ]]; then + # Reuse the identical fixed-length matrix used by the native FreeToken + # control. The wrapper remains opt-in so the normal vision quality gate + # does not silently become a longer benchmark. + PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/benchmark_gemma4_gguf_text_matrix.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model "${MODEL_NAME}" \ + --gguf "${MODEL_PATH}" --samples "${FREETOKEN_GEMMA4_MATRIX_SAMPLES:-5}" \ + --max-tokens "${FREETOKEN_GEMMA4_MATRIX_TOKENS:-128}" \ + --prompt-repeat "${FREETOKEN_GEMMA4_PROMPT_REPEAT:-1}" \ + --artifact "${ARTIFACT_DIR}/text-matrix.json" \ + >"${ARTIFACT_DIR}/text-matrix.log" 2>&1 +fi + +if [[ "${FREETOKEN_GEMMA4_CONCURRENCY:-}" == "1" ]]; then + PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/benchmark_gemma4_concurrency.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model "${MODEL_NAME}" \ + --clients "${FREETOKEN_GEMMA4_CLIENTS:-4}" --rounds "${FREETOKEN_GEMMA4_ROUNDS:-3}" \ + --max-tokens "${FREETOKEN_GEMMA4_MATRIX_TOKENS:-128}" --artifact "${ARTIFACT_DIR}/concurrency.json" \ + --prompt-repeat "${FREETOKEN_GEMMA4_PROMPT_REPEAT:-1}" \ + >"${ARTIFACT_DIR}/concurrency.log" 2>&1 +fi + +if [[ "${FREETOKEN_GEMMA4_LONG_CONTEXT:-}" == "1" ]]; then + PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/benchmark_gemma4_long_context.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model "${MODEL_NAME}" \ + --artifact "${ARTIFACT_DIR}/long-context.json" >"${ARTIFACT_DIR}/long-context.log" 2>&1 +fi +image_verify_args=() +if [[ "${FREETOKEN_GEMMA4_EXTENDED:-}" == "1" ]]; then + # Keep the normal llama.cpp reference quick, but permit the identical + # expanded fixture set when checking color and spatial parity with + # FreeToken after a multimodal implementation change. + image_verify_args+=(--extended) +fi +PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/verify_gemma4_gguf_image.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model "${MODEL_NAME}" \ + --max-tokens 128 --stream "${image_verify_args[@]}" --artifact "${ARTIFACT_DIR}/image-quality.json" \ + >"${ARTIFACT_DIR}/image-quality.log" 2>&1 +# Use the identical deterministic fixture and visible-output quality gate as +# FreeToken. This keeps visual decode timing comparable despite llama.cpp's +# optional reasoning channel. Gemma4 through llama.cpp may emit a substantial +# reasoning trace before visible content, so 1,024 tokens establishes whether +# the runtime can complete the user-visible response at all. +PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/verify_gemma4_gguf_visual_tps.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model "${MODEL_NAME}" \ + --max-tokens 1024 --artifact "${ARTIFACT_DIR}/visual-tps.json" \ + >"${ARTIFACT_DIR}/visual-tps.log" 2>&1 diff --git a/scripts/gmk-evo-x2/run_qwen_dpm_policy_benchmark.sh b/scripts/gmk-evo-x2/run_qwen_dpm_policy_benchmark.sh new file mode 100755 index 0000000000..d02889a556 --- /dev/null +++ b/scripts/gmk-evo-x2/run_qwen_dpm_policy_benchmark.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Run the isolated GMKtek EVO-X2 Qwen scheduler workload with a temporary GPU DPM policy. +# +# This wrapper exists because the normal scheduler harness deliberately refuses an +# already-existing artifact directory, whereas policy telemetry must be written +# before the harness begins. It therefore creates one parent evidence directory +# and reserves a new, non-existent `benchmark` child for the harness itself. +# +# The script changes only GPU DPM policy for the duration of its own process. +# Its EXIT trap restores the requested prior policy even if the benchmark fails. +# It neither starts nor stops FreeToken, touches llama-swap, nor contacts a host +# other than GMKtek EVO-X2's local API endpoint through the delegated harness. + +set -euo pipefail + +# Require the desired temporary policy explicitly so accidental invocation cannot +# silently change the GPU to an unintended policy level. +readonly TEMPORARY_POLICY="${1:?usage: run_qwen_dpm_policy_benchmark.sh POLICY [ARTIFACT_ROOT]}" + +# Store preflight and restoration telemetry in a unique parent directory. The +# second argument permits a caller to choose an immutable evidence location. +readonly ARTIFACT_ROOT="${2:-${HOME}/freetoken-amd/artifacts/qwen-dpm-${TEMPORARY_POLICY}-$(date -u +%Y%m%dT%H%M%SZ)}" + +# Keep the benchmark child absent. run_qwen_scheduler_baseline.sh delegates to +# a Python harness that creates this directory atomically to prevent artifact +# collisions and preserve evidence integrity. +readonly BENCHMARK_DIR="${ARTIFACT_ROOT}/benchmark" +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +readonly HARNESS="${ROOT_DIR}/source-qwen-harness-d6ee8ce/scripts/gmk-evo-x2/run_qwen_scheduler_baseline.sh" +readonly POLICY_LOG="${ARTIFACT_ROOT}/dpm-policy.txt" + +# Fail before a policy change if an operator supplied a reused artifact root. +if [[ -e "${ARTIFACT_ROOT}" ]]; then + echo "error: artifact root already exists: ${ARTIFACT_ROOT}" >&2 + exit 2 +fi + +mkdir -p "${ARTIFACT_ROOT}" + +# Restore the safe default policy and append post-run telemetry. Each command +# is best-effort so a benchmark failure cannot conceal the restoration attempt. +restore_policy() { + sudo rocm-smi --setperflevel auto || true + rocm-smi --showperflevel | tee -a "${POLICY_LOG}" || true +} +trap restore_policy EXIT + +# Apply and record the requested policy before the warm benchmark begins. +sudo rocm-smi --setperflevel "${TEMPORARY_POLICY}" +rocm-smi --showperflevel | tee "${POLICY_LOG}" + +# Pass the guaranteed-absent child path to the existing fixed scheduler workload. +bash "${HARNESS}" "${BENCHMARK_DIR}" diff --git a/scripts/gmk-evo-x2/run_qwen_gguf_endurance_battery.sh b/scripts/gmk-evo-x2/run_qwen_gguf_endurance_battery.sh new file mode 100755 index 0000000000..e4403946fe --- /dev/null +++ b/scripts/gmk-evo-x2/run_qwen_gguf_endurance_battery.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Run an isolated, process-scoped Qwen GGUF endurance battery on GMKtek EVO-X2. +# +# Linux reports swap for every desktop and monitoring process. A system-wide +# zero-swap requirement can therefore reject a healthy model server because an +# unrelated service such as netdata or Xwayland has one swapped page. This +# battery keeps that whole-host number as telemetry, but enforces zero swapped +# pages only for the verified FreeToken Q4 server process group and its +# multiprocessing children. + +set -euo pipefail + +# Require a fresh caller-owned artifact directory for every endurance run. +readonly ARTIFACT_ROOT="${1:?usage: run_qwen_gguf_endurance_battery.sh ARTIFACT_ROOT [SESSION_COUNT] [INTERVAL_SECONDS]}" +# Default to a one-hour cadence while permitting short, explicitly labelled +# diagnostic runs that use the same request and validation contract. +readonly SESSION_COUNT="${2:-60}" +# Sleep after a completed session so normal time-based drift is visible instead +# of compressing every request into a short throughput-only batch. +readonly INTERVAL_SECONDS="${3:-60}" + +# Keep all fixed GMKtek EVO-X2 paths explicit for reproducibility and host isolation. +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +# Allow an isolated candidate worktree to reuse the exact endurance contract. +# The caller must choose a path under the dedicated Qwen source root, so this +# override cannot accidentally execute arbitrary code or touch port 1919. +readonly SOURCE_DIR="${FREETOKEN_Q4_SOURCE_DIR:-${ROOT_DIR}/source-qwen-gguf-5c7f0fd}" +readonly VENV_PYTHON="${ROOT_DIR}/.venv/bin/python" +readonly RUNNER="${SOURCE_DIR}/benchmarks/gmk_evo_x2/run_multiturn_state_suite.py" +readonly SUITE="${SOURCE_DIR}/benchmarks/gmk_evo_x2/multiturn_state_suite.json" +readonly MODEL="qwen36-35b-a3b-q4km-gguf-amd" +readonly PORT="1922" +readonly EXPECTED_HOST="${FREETOKEN_EXPECTED_HOST:?Set FREETOKEN_EXPECTED_HOST to the approved test hostname}" + +# Reject malformed numeric input before opening a socket or creating artifacts. +case "${SESSION_COUNT}" in ''|*[!0-9]*) echo "session count must be a positive integer" >&2; exit 2;; esac +case "${INTERVAL_SECONDS}" in ''|*[!0-9]*) echo "interval must be a non-negative integer" >&2; exit 2;; esac +(( SESSION_COUNT > 0 )) || { echo "session count must be positive" >&2; exit 2; } +[[ ! -e "${ARTIFACT_ROOT}" ]] || { echo "artifact root already exists: ${ARTIFACT_ROOT}" >&2; exit 2; } +[[ "${SOURCE_DIR}" == "${ROOT_DIR}/source-qwen-"* ]] || { + echo "source directory must be an isolated Qwen checkout under ${ROOT_DIR}" >&2 + exit 2 +} +[[ -x "${VENV_PYTHON}" && -f "${RUNNER}" && -f "${SUITE}" ]] || { + echo "missing Qwen endurance dependency" >&2 + exit 2 +} + +# Resolve the current HTTP listener instead of trusting a stale PID file. +listener_pid() { + ss -ltnp "( sport = :${PORT} )" | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | head -1 +} + +# Verify that the port owner is the isolated Q4 test runner before reading its +# process tree. This prevents a port collision from turning into an unrelated +# process inspection or false passing endurance result. +qualified_server_pid() { + local pid command + pid="$(listener_pid)" + [[ "${pid}" =~ ^[0-9]+$ ]] || return 1 + [[ -r "/proc/${pid}/cmdline" ]] || return 1 + command="$(tr '\0' ' ' < "/proc/${pid}/cmdline")" + [[ "${command}" == *"freetoken.cli serve"* ]] || return 1 + [[ "${command}" == *"qwen36-35b-a3b-q4km-gguf-amd"* ]] || return 1 + printf '%s\n' "${pid}" +} + +# Sum VmSwap across the server's dedicated process group. The qualified +# launcher creates a group whose ID equals the HTTP server PID. Requiring that +# invariant detects manually started or partially recovered process trees. +runner_swap_kib() { + local server_pid pgid pid seen=0 total=0 swapped + server_pid="$(qualified_server_pid)" || return 1 + pgid="$(ps -o pgid= -p "${server_pid}" | tr -d ' ')" + [[ "${pgid}" == "${server_pid}" ]] || return 1 + while read -r pid; do + [[ -r "/proc/${pid}/status" ]] || continue + swapped="$(awk '/^VmSwap:/{print $2}' "/proc/${pid}/status")" + total=$((total + ${swapped:-0})) + seen=$((seen + 1)) + done < <(ps -eo pid=,pgid= | awk -v group="${pgid}" '$2 == group {print $1}') + (( seen > 0 )) || return 1 + printf '%s\n' "${total}" +} + +# Record whole-host and process-scoped memory facts separately. Whole-host +# swap remains useful for diagnosing host contention, but only the runner value +# is a pass or fail condition for this model-service qualification. +record_memory() { + local destination="$1" + local runner_swap + runner_swap="$(runner_swap_kib)" || { + echo "cannot resolve qualified Q4 process group" >&2 + return 1 + } + { + printf 'captured_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'runner_swap_kib=%s\n' "${runner_swap}" + printf 'whole_host_swap_kib=%s\n' "$(awk '/SwapTotal/{t=$2}/SwapFree/{f=$2} END{print t-f}' /proc/meminfo)" + awk '/^(MemAvailable|SwapCached|SwapTotal|SwapFree):/{print}' /proc/meminfo + free -k + rocm-smi --showtemp --showperflevel --showmeminfo vram 2>&1 || true + } >"${destination}" + [[ "${runner_swap}" == "0" ]] +} + +mkdir -p "${ARTIFACT_ROOT}/sessions" +record_memory "${ARTIFACT_ROOT}/preflight.txt" || { + echo "refusing endurance run: qualified Q4 process has swapped pages" >&2 + exit 2 +} + +# Use the exact deterministic three-turn suite for every timed session. Each +# per-session JSON contains the visible output and tail timing, while this +# wrapper adds process-scoped swap and GPU telemetry at the session boundary. +export PYTHONPATH="${SOURCE_DIR}/python" +for session in $(seq -w 1 "${SESSION_COUNT}"); do + started_epoch="$(date +%s)" + "${VENV_PYTHON}" "${RUNNER}" \ + --base-url "http://127.0.0.1:${PORT}/v1" \ + --model "${MODEL}" \ + --artifact "${ARTIFACT_ROOT}/sessions/session-${session}.json" \ + --suite "${SUITE}" \ + --expected-host "${EXPECTED_HOST}" \ + --max-tokens 64 >"${ARTIFACT_ROOT}/sessions/session-${session}.log" 2>&1 + record_memory "${ARTIFACT_ROOT}/sessions/session-${session}-telemetry.txt" || { + echo "runner swap gate failed after session ${session}" >&2 + exit 2 + } + elapsed=$(( $(date +%s) - started_epoch )) + # `seq -w` produces labels such as 08. Force decimal interpretation so + # Bash does not treat that label as an invalid octal literal in arithmetic. + if (( 10#${session} < SESSION_COUNT && elapsed < INTERVAL_SECONDS )); then + sleep $((INTERVAL_SECONDS - elapsed)) + fi +done + +# Retain a final sample after the last conversation for recovery verification. +record_memory "${ARTIFACT_ROOT}/postflight.txt" diff --git a/scripts/gmk-evo-x2/run_qwen_gguf_raw_control.sh b/scripts/gmk-evo-x2/run_qwen_gguf_raw_control.sh new file mode 100755 index 0000000000..db6f7a321e --- /dev/null +++ b/scripts/gmk-evo-x2/run_qwen_gguf_raw_control.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Run one isolated Qwen GGUF raw-prompt quality control on GMKtek EVO-X2. +# +# This script deliberately takes the production API offline only while an +# isolated checkout owns the Strix Halo GPU. Its EXIT trap always stops that +# temporary server and invokes the production recovery script before returning. +# The control sends a caller-rendered prompt through /v1/completions, allowing +# direct comparison with llama.cpp without a server-specific chat template. + +set -euo pipefail + +# The isolated checkout is required so this procedure can never modify the +# production source tree while proving a candidate change. +readonly CHECKOUT="${1:?usage: run_qwen_gguf_raw_control.sh ISOLATED_CHECKOUT [DECODE_TOKENS]}" +# A 512-token budget is normally sufficient to finish the fixed AIME answer; +# callers may supply another positive limit when investigating longer outputs. +readonly DECODE_TOKENS="${2:-512}" +# GMKtek EVO-X2's persistent project root keeps models, artifacts, and production +# recovery tooling outside the disposable candidate checkout. +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +readonly PRODUCTION_DIR="${ROOT_DIR}/source-qwen-harness-d6ee8ce" +readonly MODEL_PATH="${ROOT_DIR}/models/controls/qwen36-35b-a3b-unsloth-a483e9e6/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf" +readonly TOKENIZER_PATH="${ROOT_DIR}/models/Qwen3.6-35B-A3B-NVFP4" +readonly TEST_PORT="1922" +readonly PRODUCTION_PORT="1919" +readonly SERVED_MODEL="qwen36-35b-a3b-q4km-gguf-amd" +readonly ARTIFACT_DIR="${ROOT_DIR}/artifacts/qwen-gguf-raw-$(date -u +%Y%m%dT%H%M%SZ)" + +# Artifact immutability makes a repeated timestamp collision an explicit error. +mkdir -p "${ARTIFACT_DIR}" + +port_pid() { + # Resolve the actual listener PID rather than relying on a stale pidfile. + ss -ltnp "( sport = :$1 )" | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | head -1 +} + +restore_production() { + # Stop only the temporary listener, if it reached startup. + local test_pid + test_pid="$(port_pid "${TEST_PORT}")" + [[ -z "${test_pid}" ]] || kill "${test_pid}" || true + # Avoid a duplicate recovery when the production endpoint survived a setup + # failure. The recovery helper owns the production command and its logs. + if ! timeout 5 curl -fsS "http://127.0.0.1:${PRODUCTION_PORT}/health" >/dev/null; then + bash "${PRODUCTION_DIR}/scripts/gmk-evo-x2/start_qwen_recovery_server.sh" \ + | tee "${ARTIFACT_DIR}/recovery.log" + fi +} +trap restore_production EXIT + +# Release the one GPU from the persistent service before launching the candidate. +production_pid="$(port_pid "${PRODUCTION_PORT}")" +if [[ -n "${production_pid}" ]]; then + kill "${production_pid}" +fi +for _ in {1..60}; do + ss -ltn "( sport = :${PRODUCTION_PORT} )" | grep -q "${PRODUCTION_PORT}" || break + sleep 1 +done + +# Start a test-only server with the same AMD execution configuration used by the +# prior Q4 controls. Output is retained verbatim with the evidence artifact. +cd "${CHECKOUT}" +ROCM_HOME=/opt/rocm-10.0 ROCM_PATH=/opt/rocm-10.0 HIP_PATH=/opt/rocm-10.0 \ +PYTHONPATH=python TORCH_EXTENSIONS_DIR="${ROOT_DIR}/cache/torch_extensions" \ +nohup "${ROOT_DIR}/.venv/bin/python" -m freetoken.cli serve \ + --model-path "${MODEL_PATH}" \ + --served-model-name "${SERVED_MODEL}" \ + --host 127.0.0.1 --port "${TEST_PORT}" \ + --attention-backend triton --moe-backend offload --nvfp4-backend triton \ + --expert-load serial --moe-cache-auto --memory-ratio 0.35 \ + --max-seq-len-override 8192 --kv-reserve-tokens 2048 \ + --cuda-graph-max-bs 0 --disable-pynccl --disable-moe-prefill-overlap \ + >"${ARTIFACT_DIR}/server.log" 2>&1 & +candidate_pid=$! +for _ in {1..480}; do + grep -q 'API server is ready to serve' "${ARTIFACT_DIR}/server.log" && break + kill -0 "${candidate_pid}" 2>/dev/null || exit 1 + sleep 1 +done +grep -q 'API server is ready to serve' "${ARTIFACT_DIR}/server.log" + +# Persist the request body, final text, exact prompt hash, server usage, and +# first-token/decode timings in one self-contained JSON control artifact. +PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" \ + scripts/gmk-evo-x2/verify_qwen_raw_prompt_quality.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model "${SERVED_MODEL}" \ + --tokenizer "${TOKENIZER_PATH}" --decode "${DECODE_TOKENS}" \ + --artifact "${ARTIFACT_DIR}/raw-quality.json" \ + >"${ARTIFACT_DIR}/raw-quality.log" 2>&1 diff --git a/scripts/gmk-evo-x2/run_qwen_gguf_timeshare_endurance.sh b/scripts/gmk-evo-x2/run_qwen_gguf_timeshare_endurance.sh new file mode 100755 index 0000000000..ac9346f5cb --- /dev/null +++ b/scripts/gmk-evo-x2/run_qwen_gguf_timeshare_endurance.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Run a long isolated Q4 endurance battery and restore GMKtek EVO-X2's NVFP4 service. +# +# This controller owns one deliberate GPU time-share window. It does not touch +# llama-swap or any LAN endpoint. It stops the verified dedicated loopback +# recovery session, starts the isolated Q4 test session, runs the existing +# process-scoped battery, and restores the normal service even when the battery +# fails or the controller receives a termination signal. + +set -euo pipefail + +# Require a caller-owned immutable root, then permit the full 24-hour default +# while also allowing a shorter explicitly labelled diagnostic duration. +readonly ARTIFACT_ROOT="${1:?usage: run_qwen_gguf_timeshare_endurance.sh ARTIFACT_ROOT [SESSION_COUNT] [INTERVAL_SECONDS]}" +readonly SESSION_COUNT="${2:-1440}" +readonly INTERVAL_SECONDS="${3:-60}" + +# Keep every host-specific path explicit so an invocation cannot silently +# operate on another machine's service or an arbitrary source checkout. +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +readonly Q4_SOURCE_DIR="${FREETOKEN_Q4_SOURCE_DIR:?set FREETOKEN_Q4_SOURCE_DIR to an isolated Q4 worktree}" +readonly RECOVERY_SOURCE_DIR="${FREETOKEN_RECOVERY_SOURCE_DIR:?set FREETOKEN_RECOVERY_SOURCE_DIR to the recovery-launcher worktree}" +readonly Q4_LAUNCHER="${Q4_SOURCE_DIR}/scripts/gmk-evo-x2/launch_qwen_gguf_qualified.sh" +readonly Q4_BATTERY="${Q4_SOURCE_DIR}/scripts/gmk-evo-x2/run_qwen_gguf_endurance_battery.sh" +readonly RECOVERY_STOPPER="${RECOVERY_SOURCE_DIR}/scripts/gmk-evo-x2/stop_qwen_recovery_server.sh" +readonly RECOVERY_STARTER="${RECOVERY_SOURCE_DIR}/scripts/gmk-evo-x2/start_qwen_recovery_server.sh" +readonly Q4_ARTIFACT_DIR="${ARTIFACT_ROOT}/q4-server" +readonly BATTERY_ARTIFACT_DIR="${ARTIFACT_ROOT}/battery" +readonly RECOVERY_ARTIFACT="${ARTIFACT_ROOT}/recovery-health.json" + +# Reject unsafe input before stopping the protected model service or creating +# any artifact. The source guards mirror the candidate launcher safeguards. +case "${SESSION_COUNT}" in ''|*[!0-9]*) echo "session count must be positive" >&2; exit 2;; esac +case "${INTERVAL_SECONDS}" in ''|*[!0-9]*) echo "interval must be non-negative" >&2; exit 2;; esac +(( SESSION_COUNT > 0 )) || { echo "session count must be positive" >&2; exit 2; } +[[ ! -e "${ARTIFACT_ROOT}" ]] || { echo "artifact root already exists: ${ARTIFACT_ROOT}" >&2; exit 2; } +[[ "${Q4_SOURCE_DIR}" == "${ROOT_DIR}/source-qwen-"* ]] || { echo "Q4 source must be under ${ROOT_DIR}" >&2; exit 2; } +[[ "${RECOVERY_SOURCE_DIR}" == "${ROOT_DIR}/source-qwen-"* ]] || { echo "recovery source must be under ${ROOT_DIR}" >&2; exit 2; } +[[ -f "${Q4_LAUNCHER}" && -f "${Q4_BATTERY}" && -f "${RECOVERY_STOPPER}" && -f "${RECOVERY_STARTER}" ]] || { + echo "missing time-share dependency" >&2 + exit 2 +} + +# Poll the documented health state instead of treating a bound port or model +# listing as proof that a cold server has loaded all expert banks. +wait_for_serving() { + local port="$1" destination="$2" status maintenance missing_listener=0 + for _ in $(seq 1 900); do + # A launch may need a few seconds to bind the port, but a missing + # listener for a sustained interval means the candidate exited and the + # controller must enter recovery instead of waiting the full timeout. + if ss -ltn "( sport = :${port} )" | grep -q ":${port}"; then + missing_listener=0 + else + missing_listener=$((missing_listener + 1)) + if (( missing_listener >= 30 )); then + echo "server on port ${port} exited before readiness" >&2 + return 1 + fi + fi + curl -fsS --max-time 5 "http://127.0.0.1:${port}/health" >"${destination}" 2>/dev/null || true + status="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("status", ""))' "${destination}" 2>/dev/null || true)" + maintenance="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("maintenance", ""))' "${destination}" 2>/dev/null || true)" + [[ "${status}" == "ok" && "${maintenance}" == "serving" ]] && return 0 + sleep 1 + done + echo "server on port ${port} did not reach serving state" >&2 + return 1 +} + +# Restore in every exit path. The Q4 launcher verifies its own exact process +# group before signalling it, and the recovery launcher creates the dedicated +# group needed by future time-share windows. +restore_normal_service() { + local status=0 + if [[ -f "${Q4_ARTIFACT_DIR}/server.pid" ]]; then + FREETOKEN_Q4_SOURCE_DIR="${Q4_SOURCE_DIR}" bash "${Q4_LAUNCHER}" stop "${Q4_ARTIFACT_DIR}" || status=1 + fi + if ! curl -fsS --max-time 5 http://127.0.0.1:1919/health >"${RECOVERY_ARTIFACT}" 2>/dev/null; then + bash "${RECOVERY_STARTER}" >"${ARTIFACT_ROOT}/recovery-start.log" 2>&1 || status=1 + fi + wait_for_serving 1919 "${RECOVERY_ARTIFACT}" || status=1 + return "${status}" +} + +mkdir -p "${ARTIFACT_ROOT}" +printf 'started_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"${ARTIFACT_ROOT}/controller.txt" +printf 'session_count=%s\ninterval_seconds=%s\n' "${SESSION_COUNT}" "${INTERVAL_SECONDS}" >>"${ARTIFACT_ROOT}/controller.txt" +trap 'restore_normal_service' EXIT INT TERM + +# The stopper refuses an unmanaged legacy tree. That fail-closed behavior +# prevents this controller from guessing at child ownership on a shared host. +bash "${RECOVERY_STOPPER}" +FREETOKEN_Q4_SOURCE_DIR="${Q4_SOURCE_DIR}" bash "${Q4_LAUNCHER}" start "${Q4_ARTIFACT_DIR}" 0.25 +wait_for_serving 1922 "${ARTIFACT_ROOT}/q4-health.json" +FREETOKEN_Q4_SOURCE_DIR="${Q4_SOURCE_DIR}" bash "${Q4_BATTERY}" "${BATTERY_ARTIFACT_DIR}" "${SESSION_COUNT}" "${INTERVAL_SECONDS}" +"${ROOT_DIR}/.venv/bin/python" "${Q4_SOURCE_DIR}/benchmarks/gmk_evo_x2/summarize_qwen_gguf_endurance.py" \ + "${BATTERY_ARTIFACT_DIR}" --expected-sessions "${SESSION_COUNT}" >"${ARTIFACT_ROOT}/summary.json" +printf 'completed_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >>"${ARTIFACT_ROOT}/controller.txt" diff --git a/scripts/gmk-evo-x2/run_qwen_llamacpp_raw_control.sh b/scripts/gmk-evo-x2/run_qwen_llamacpp_raw_control.sh new file mode 100755 index 0000000000..747e25242b --- /dev/null +++ b/scripts/gmk-evo-x2/run_qwen_llamacpp_raw_control.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Run a caller-rendered Qwen GGUF raw-prompt quality control against ROCm llama.cpp. +# This uses the same model file, prompt renderer, decoding parameters, and evidence +# schema as run_qwen_gguf_raw_control.sh, then restores FreeToken on exit. + +set -euo pipefail + +readonly DECODE_TOKENS="${1:-1024}" +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +readonly PRODUCTION_DIR="${ROOT_DIR}/source-qwen-harness-d6ee8ce" +readonly LLAMA_SERVER="${ROOT_DIR}/llama.cpp-rocm10-b10141/build-rocm10-clang/bin/llama-server" +readonly MODEL_PATH="${ROOT_DIR}/models/controls/qwen36-35b-a3b-unsloth-a483e9e6/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf" +readonly TOKENIZER_PATH="${ROOT_DIR}/models/Qwen3.6-35B-A3B-NVFP4" +readonly TEST_PORT="1921" +readonly PRODUCTION_PORT="1919" +readonly SERVED_MODEL="qwen36-35b-a3b-q4km-llama-raw" +readonly HARNESS_DIR="${ROOT_DIR}/validation-qwen-gguf-d1dd473" +readonly ARTIFACT_DIR="${ROOT_DIR}/artifacts/qwen-llama-raw-$(date -u +%Y%m%dT%H%M%SZ)" +mkdir -p "${ARTIFACT_DIR}" + +port_pid() { ss -ltnp "( sport = :$1 )" | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | head -1; } +restore_production() { + local test_pid + test_pid="$(port_pid "${TEST_PORT}")" + [[ -z "${test_pid}" ]] || kill "${test_pid}" || true + if ! timeout 5 curl -fsS "http://127.0.0.1:${PRODUCTION_PORT}/health" >/dev/null; then + bash "${PRODUCTION_DIR}/scripts/gmk-evo-x2/start_qwen_recovery_server.sh" | tee "${ARTIFACT_DIR}/recovery.log" + fi +} +trap restore_production EXIT + +production_pid="$(port_pid "${PRODUCTION_PORT}")" +[[ -z "${production_pid}" ]] || kill "${production_pid}" +for _ in {1..60}; do ss -ltn "( sport = :${PRODUCTION_PORT} )" | grep -q "${PRODUCTION_PORT}" || break; sleep 1; done + +export LD_LIBRARY_PATH="/opt/rocm-10.0/llvm/lib:/opt/rocm-10.0/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" +nohup "${LLAMA_SERVER}" -m "${MODEL_PATH}" --alias "${SERVED_MODEL}" -ngl all -c 8192 -np 1 \ + -b 2048 -ub 512 -ctk q8_0 -ctv q8_0 -fa on --jinja --reasoning-format deepseek \ + --no-context-shift --no-warmup --host 127.0.0.1 --port "${TEST_PORT}" \ + >"${ARTIFACT_DIR}/server.log" 2>&1 & +candidate_pid=$! +for _ in {1..180}; do + timeout 5 curl -fsS "http://127.0.0.1:${TEST_PORT}/health" >"${ARTIFACT_DIR}/health.json" && break + kill -0 "${candidate_pid}" 2>/dev/null || exit 1 + sleep 1 +done +test -s "${ARTIFACT_DIR}/health.json" + +cd "${HARNESS_DIR}" +PYTHONPATH=python "${ROOT_DIR}/.venv/bin/python" scripts/gmk-evo-x2/verify_qwen_raw_prompt_quality.py \ + --base-url "http://127.0.0.1:${TEST_PORT}" --model "${SERVED_MODEL}" \ + --tokenizer "${TOKENIZER_PATH}" --decode "${DECODE_TOKENS}" \ + --artifact "${ARTIFACT_DIR}/raw-quality.json" >"${ARTIFACT_DIR}/raw-quality.log" 2>&1 diff --git a/scripts/gmk-evo-x2/run_qwen_llamacpp_rocm_control.sh b/scripts/gmk-evo-x2/run_qwen_llamacpp_rocm_control.sh new file mode 100755 index 0000000000..e2259a1400 --- /dev/null +++ b/scripts/gmk-evo-x2/run_qwen_llamacpp_rocm_control.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# Run the isolated ROCm 10 llama.cpp Qwen3.6-35B-A3B control on GMKtek EVO-X2. +# +# This script intentionally starts a short-lived loopback-only llama.cpp server +# on port 1921. It never contacts llama-swap, modifies its configuration, stops +# the FreeToken service on port 1919, or uses another LAN host. The server is +# terminated by the EXIT trap after evidence capture, including on a failure. + +set -euo pipefail + +# Keep the precise source revision, model revision, local model path, and API +# identity visible in the command itself so the comparison can be reproduced +# without guessing which llama.cpp build or Qwen quantization was selected. +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +readonly SOURCE_DIR="${ROOT_DIR}/source-qwen-harness-d6ee8ce" +readonly LLAMA_SERVER="${ROOT_DIR}/llama.cpp-rocm10-b10141/build-rocm10-clang/bin/llama-server" +readonly MODEL_DIR="${ROOT_DIR}/models/controls/qwen36-35b-a3b-unsloth-a483e9e6" +readonly MODEL_FILE="${MODEL_DIR}/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf" +readonly TOKENIZER_DIR="${ROOT_DIR}/models/Qwen3.6-35B-A3B-NVFP4" +readonly MODEL_NAME="qwen3.6-35b-a3b-q4km-llamacpp-rocm10" +readonly BASE_URL="http://127.0.0.1:1921/v1" +readonly ARTIFACT_ROOT="${1:-${ROOT_DIR}/artifacts/qwen35b-llamacpp-rocm10-$(date -u +%Y%m%dT%H%M%SZ)}" +readonly BENCHMARK_DIR="${ARTIFACT_ROOT}/benchmark" +readonly SERVER_LOG="${ARTIFACT_ROOT}/llama-server.log" +readonly SERVER_PID_FILE="${ARTIFACT_ROOT}/llama-server.pid" + +# Refuse ambiguous or partial input before allocating GPU memory. The matching +# FreeToken tokenizer counts generated text consistently across both endpoints. +if [[ ! -x "${LLAMA_SERVER}" ]]; then + echo "error: ROCm llama-server is missing or not executable: ${LLAMA_SERVER}" >&2 + exit 2 +fi +if [[ ! -f "${MODEL_FILE}" ]]; then + echo "error: matching Qwen GGUF is missing: ${MODEL_FILE}" >&2 + exit 2 +fi +if [[ ! -d "${TOKENIZER_DIR}" ]]; then + echo "error: FreeToken Qwen tokenizer directory is missing: ${TOKENIZER_DIR}" >&2 + exit 2 +fi +if [[ -e "${ARTIFACT_ROOT}" ]]; then + echo "error: artifact root already exists: ${ARTIFACT_ROOT}" >&2 + exit 2 +fi + +mkdir -p "${ARTIFACT_ROOT}" + +# ROCm 10's llama.cpp build dynamically links LLVM's libclang runtime. Extend +# only this script's process environment so global shell and service settings +# remain unchanged. Keep any pre-existing library path entries available too. +export LD_LIBRARY_PATH="/opt/rocm-10.0/llvm/lib:/opt/rocm-10.0/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" + +# Stop only the temporary child recorded by this script. The guard prevents a +# malformed PID file from targeting another process, and wait reaps the child +# before leaving its raw server log and benchmark evidence on disk. +cleanup_server() { + if [[ -f "${SERVER_PID_FILE}" ]]; then + local server_pid + server_pid="$(<"${SERVER_PID_FILE}")" + if [[ "${server_pid}" =~ ^[0-9]+$ ]] && kill -0 "${server_pid}" 2>/dev/null; then + kill "${server_pid}" 2>/dev/null || true + wait "${server_pid}" 2>/dev/null || true + fi + fi +} +trap cleanup_server EXIT + +# Start the exact ROCm 10 b10141 control on an otherwise unused loopback port. +# One slot, 8,192 context tokens, full GPU offload, Flash Attention, and Q8 KV +# cache retain the previously documented GMKtek EVO-X2 ROCm control conventions. +"${LLAMA_SERVER}" \ + -m "${MODEL_FILE}" \ + --alias "${MODEL_NAME}" \ + -ngl all \ + -c 8192 \ + -np 1 \ + -b 2048 \ + -ub 512 \ + -ctk q8_0 \ + -ctv q8_0 \ + -fa on \ + --jinja \ + --reasoning-format deepseek \ + --no-context-shift \ + --no-warmup \ + --metrics \ + --slots \ + --host 127.0.0.1 \ + --port 1921 >"${SERVER_LOG}" 2>&1 & +echo "$!" >"${SERVER_PID_FILE}" + +# Wait for a definite local health response, reporting the preserved server log +# if initialization fails rather than silently benchmarking a different server. +for _ in $(seq 1 180); do + # HTTP 200 is not sufficient: FreeToken and llama.cpp can expose a health + # endpoint while weights are still loading. Require the explicit ready + # state before sending benchmark traffic, otherwise the first request can + # receive a transient 503 and invalidate the whole comparison. + if curl -fsS "${BASE_URL%/v1}/health" >"${ARTIFACT_ROOT}/health-ready.json" \ + && grep -q '"status"[[:space:]]*:[[:space:]]*"ok"' "${ARTIFACT_ROOT}/health-ready.json"; then + break + fi + if ! kill -0 "$(<"${SERVER_PID_FILE}")" 2>/dev/null; then + echo "error: temporary llama.cpp server exited during initialization" >&2 + tail -n 120 "${SERVER_LOG}" >&2 || true + exit 1 + fi + sleep 1 +done +if [[ ! -s "${ARTIFACT_ROOT}/health-ready.json" ]]; then + echo "error: temporary llama.cpp server was not healthy within 180 seconds" >&2 + exit 1 +fi + +# Delegate the unchanged fixed workload to the existing harness while overriding +# only endpoint identity and tokenizer location for this temporary control. +GMK_EVO_X2_QWEN_BASE_URL="${BASE_URL}" \ +GMK_EVO_X2_QWEN_MODEL_NAME="${MODEL_NAME}" \ +GMK_EVO_X2_QWEN_TOKENIZER_DIR="${TOKENIZER_DIR}" \ + bash "${SOURCE_DIR}/scripts/gmk-evo-x2/run_qwen_scheduler_baseline.sh" "${BENCHMARK_DIR}" + +if [[ "${GMK_EVO_X2_QWEN_QUALITY_SUITE:-}" == "1" ]]; then + # The optional suite uses only deterministic visible-output controls. Keep + # it opt-in so the normal throughput control remains unchanged, while a + # paired quality campaign can run against this exact temporary ROCm server. + PYTHONPATH="${SOURCE_DIR}/python" "${ROOT_DIR}/.venv/bin/python" \ + "${SOURCE_DIR}/benchmarks/gmk_evo_x2/run_quality_suite.py" \ + --base-url "${BASE_URL}" \ + --model "${MODEL_NAME}" \ + --expected-host "${FREETOKEN_EXPECTED_HOST:?Set FREETOKEN_EXPECTED_HOST to the approved test hostname}" \ + --max-tokens 64 \ + --artifact "${ARTIFACT_ROOT}/quality.json" \ + >"${ARTIFACT_ROOT}/quality.log" 2>&1 +fi + +# Capture final endpoint health before the EXIT trap terminates the control. +curl -fsS "${BASE_URL%/v1}/health" >"${ARTIFACT_ROOT}/health-before-cleanup.json" diff --git a/scripts/gmk-evo-x2/run_qwen_llamacpp_rocm_timeshare_control.sh b/scripts/gmk-evo-x2/run_qwen_llamacpp_rocm_timeshare_control.sh new file mode 100755 index 0000000000..1f9cfdb819 --- /dev/null +++ b/scripts/gmk-evo-x2/run_qwen_llamacpp_rocm_timeshare_control.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# Run the GMKtek EVO-X2 ROCm llama.cpp Qwen control after temporarily releasing the +# isolated FreeToken benchmark server, then recover and validate FreeToken. +# +# A 64 GB Strix Halo host cannot keep the current FreeToken NVFP4 Qwen service +# and the fully offloaded 35B Q4_K_M llama.cpp control resident at the same +# time. This wrapper measures the two servers in time-share mode. It never +# touches llama-swap, systemd, or a service outside loopback port 1919. + +set -euo pipefail + +# Keep the fixed GMKtek EVO-X2 paths explicit to prevent comparison with another +# llama.cpp build or benchmark harness revision. +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +readonly SOURCE_DIR="${ROOT_DIR}/source-qwen-harness-d6ee8ce" +readonly FREETOKEN_HEALTH_URL="http://127.0.0.1:1919/health" +readonly CONTROL_SCRIPT="${SOURCE_DIR}/scripts/gmk-evo-x2/run_qwen_llamacpp_rocm_control.sh" +readonly RECOVERY_SCRIPT="${SOURCE_DIR}/scripts/gmk-evo-x2/start_qwen_recovery_server.sh" +readonly ARTIFACT_ROOT="${1:-${ROOT_DIR}/artifacts/qwen35b-llamacpp-rocm10-timeshare-$(date -u +%Y%m%dT%H%M%SZ)}" +readonly CONTROL_ARTIFACT="${ARTIFACT_ROOT}/llamacpp-control" +readonly BEFORE_HEALTH_FILE="${ARTIFACT_ROOT}/freetoken-health-before.json" +readonly AFTER_HEALTH_FILE="${ARTIFACT_ROOT}/freetoken-health-after.json" +readonly SWAP_BEFORE_FILE="${ARTIFACT_ROOT}/swap-before.txt" +readonly SWAP_AFTER_RELEASE_FILE="${ARTIFACT_ROOT}/swap-after-release.txt" +readonly SWAP_AFTER_FILE="${ARTIFACT_ROOT}/swap-after.txt" + +# Avoid overwriting evidence from an earlier run and reject partial setup before +# stopping the live benchmark server. +if [[ -e "${ARTIFACT_ROOT}" ]]; then + echo "error: artifact root already exists: ${ARTIFACT_ROOT}" >&2 + exit 2 +fi +test -x "${CONTROL_SCRIPT}" +test -x "${RECOVERY_SCRIPT}" +mkdir -p "${ARTIFACT_ROOT}" + +# Locate only a process that both owns the dedicated test port and identifies +# itself as the FreeToken server. This prevents targeting an unrelated process. +find_freetoken_pid() { + local pid command + pid="$(ss -ltnp 'sport = :1919' | sed -n 's/.*pid=\([0-9][0-9]*\).*/\1/p' | head -n 1)" + if [[ ! "${pid}" =~ ^[0-9]+$ ]]; then + return 1 + fi + command="$(tr '\0' ' ' <"/proc/${pid}/cmdline" 2>/dev/null || true)" + [[ "${command}" == *"freetoken.cli serve"* ]] || return 1 + printf '%s\n' "${pid}" +} + +# Wait for HTTP health instead of accepting a listener while FreeToken loads +# native modules and model state after recovery. +wait_for_freetoken_health() { + local attempt health_payload + for attempt in $(seq 1 720); do + # A loading server returns HTTP 200 before its MoE expert banks and KV + # cache are usable. Save every latest reply for diagnostics, but accept + # recovery only when the API explicitly reports the serving state. + health_payload="$(curl -fsS "${FREETOKEN_HEALTH_URL}" 2>/dev/null || true)" + printf '%s\n' "${health_payload}" >"${AFTER_HEALTH_FILE}" + if [[ "${health_payload}" == *'"status":"ok"'* ]]; then + return 0 + fi + sleep 1 + done + return 1 +} + +# Preserve the configured swap file while clearing pages faulted by the prior +# failed coexistence allocation. This does not alter vm.swappiness. +reset_swap_pages() { + sudo swapoff -a + sudo swapon -a +} + +# Capture a healthy start state, then stop only the identified FreeToken child. +curl -fsS "${FREETOKEN_HEALTH_URL}" >"${BEFORE_HEALTH_FILE}" +swapon --show --bytes >"${SWAP_BEFORE_FILE}" +freetoken_pid="$(find_freetoken_pid)" || { + echo "error: no verified FreeToken server owns loopback port 1919" >&2 + exit 1 +} +printf '%s\n' "${freetoken_pid}" >"${ARTIFACT_ROOT}/freetoken-server-pid.txt" +kill "${freetoken_pid}" +for _ in $(seq 1 180); do + if ! kill -0 "${freetoken_pid}" 2>/dev/null; then + break + fi + sleep 1 +done +if kill -0 "${freetoken_pid}" 2>/dev/null; then + echo "error: FreeToken server did not stop after SIGTERM" >&2 + exit 1 +fi + +# Standalone llama.cpp must not inherit swapped pages from the coexistence test. +reset_swap_pages +swapon --show --bytes >"${SWAP_AFTER_RELEASE_FILE}" + +# Run the unchanged ROCm llama.cpp control, preserving its status while always +# restoring FreeToken before the wrapper returns. +set +e +bash "${CONTROL_SCRIPT}" "${CONTROL_ARTIFACT}" +control_status=$? +set -e +printf '%s\n' "${control_status}" >"${ARTIFACT_ROOT}/llamacpp-control-exit-code.txt" + +# The recovery script prints its own dated artifact directory. Preserve it so +# the time-shared control links to native-cache and startup evidence. +bash "${RECOVERY_SCRIPT}" | tee "${ARTIFACT_ROOT}/freetoken-recovery-artifact.txt" +if ! wait_for_freetoken_health; then + echo "error: FreeToken did not become healthy after time-share control" >&2 + exit 1 +fi +swapon --show --bytes >"${SWAP_AFTER_FILE}" + +# A benchmark failure is returned only after recovered FreeToken health passes. +exit "${control_status}" diff --git a/scripts/gmk-evo-x2/run_qwen_multiturn_battery.sh b/scripts/gmk-evo-x2/run_qwen_multiturn_battery.sh new file mode 100755 index 0000000000..7c4a340752 --- /dev/null +++ b/scripts/gmk-evo-x2/run_qwen_multiturn_battery.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Run a fixed number of isolated Qwen multi-turn state-retention sessions. +# +# Each session reuses the versioned three-turn suite and writes its own immutable +# JSON artifact. The wrapper never starts, stops, or rebuilds Qwen. It requires +# a healthy, swap-free GMKtek EVO-X2 server before the first request and writes an +# aggregate summary only after every requested session has completed. + +set -euo pipefail + +readonly ARTIFACT_ROOT="${1:?usage: run_qwen_multiturn_battery.sh ARTIFACT_ROOT [SESSION_COUNT]}" +readonly SESSION_COUNT="${2:-30}" +# Default to the strict clean-memory gate. A caller may pass a higher, +# explicitly recorded ceiling for a diagnostic characterization run. +readonly MAX_SWAP_KIB="${GMK_EVO_X2_BATTERY_MAX_SWAP_KIB:-64}" +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +readonly SOURCE_DIR="${ROOT_DIR}/source-qwen-harness-d6ee8ce" +readonly VENV_PYTHON="${ROOT_DIR}/.venv/bin/python" +readonly RUNNER="${SOURCE_DIR}/benchmarks/gmk_evo_x2/run_multiturn_state_suite.py" +readonly SUITE="${SOURCE_DIR}/benchmarks/gmk_evo_x2/multiturn_state_suite.json" +readonly MODEL="qwen3.6-35b-a3b-nvfp4-amd" +readonly EXPECTED_HOST="${FREETOKEN_EXPECTED_HOST:?Set FREETOKEN_EXPECTED_HOST to the approved test hostname}" + +case "${SESSION_COUNT}" in + ''|*[!0-9]*) echo "session count must be a positive integer" >&2; exit 2 ;; +esac +if (( SESSION_COUNT < 1 )); then + echo "session count must be positive" >&2 + exit 2 +fi +case "${MAX_SWAP_KIB}" in + ''|*[!0-9]*) echo "maximum swap must be a non-negative integer" >&2; exit 2 ;; +esac +if [[ -e "${ARTIFACT_ROOT}" ]]; then + echo "refusing to overwrite artifact root: ${ARTIFACT_ROOT}" >&2 + exit 2 +fi +test -x "${VENV_PYTHON}" +test -f "${RUNNER}" +test -f "${SUITE}" + +# Do not start an endurance-style workload from an already degraded memory +# state. Swap is a failure signal for this campaign, not a performance cache. +# Ubuntu can immediately fault one bookkeeping page after a clean swap reset, +# so permit at most 64 KiB. Any larger value is treated as real pressure. +swap_used_kb() { + local total free + total="$(awk '/SwapTotal/ {print $2}' /proc/meminfo)" + free="$(awk '/SwapFree/ {print $2}' /proc/meminfo)" + echo $((total - free)) +} +assert_clean_swap() { + local used + used="$(swap_used_kb)" + if (( used > MAX_SWAP_KIB )); then + echo "refusing multi-turn battery with swap in use: ${used} KiB exceeds ${MAX_SWAP_KIB} KiB" >&2 + exit 2 + fi +} +assert_clean_swap +curl -fsS "http://127.0.0.1:1919/health" | grep -q '"status":"ok"' +# The health request can wake a lazily swapped worker page. Check again before +# the first test request so a seemingly clean preflight cannot mask that state. +assert_clean_swap + +mkdir -p "${ARTIFACT_ROOT}/sessions" +export PYTHONPATH="${SOURCE_DIR}/python" + +for session in $(seq -w 1 "${SESSION_COUNT}"); do + "${VENV_PYTHON}" "${RUNNER}" \ + --base-url "http://127.0.0.1:1919/v1" \ + --model "${MODEL}" \ + --artifact "${ARTIFACT_ROOT}/sessions/session-${session}.json" \ + --suite "${SUITE}" \ + --expected-host "${EXPECTED_HOST}" \ + --max-tokens 64 \ + >"${ARTIFACT_ROOT}/sessions/session-${session}.log" 2>&1 + # Detect sustained memory deterioration at a session boundary while still + # preserving all completed raw artifacts for later diagnosis. + assert_clean_swap +done + +# The summary retains raw per-session files and records tail values across all +# sessions, which makes a single late response visible instead of averaged out. +"${VENV_PYTHON}" - "${ARTIFACT_ROOT}" "${SESSION_COUNT}" "${MAX_SWAP_KIB}" <<'PY' +import json +import statistics +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +expected = int(sys.argv[2]) +max_swap_kib = int(sys.argv[3]) +records = [json.loads(path.read_text(encoding="utf-8")) for path in sorted((root / "sessions").glob("session-*.json"))] +ttft = [item["tail_metrics"]["max_ttft_seconds"] for item in records if item["tail_metrics"]["max_ttft_seconds"] is not None] +gaps = [item["tail_metrics"]["max_token_gap_seconds"] for item in records if item["tail_metrics"]["max_token_gap_seconds"] is not None] +def observed(values, percentile): + if not values: + return None + values = sorted(values) + return values[max(0, int(len(values) * percentile + 0.999999999) - 1)] +summary = { + "schema_version": 1, + "requested_sessions": expected, + "maximum_swap_kib": max_swap_kib, + "completed_sessions": len(records), + "passed_sessions": sum(item["status"] == "passed" for item in records), + "max_turn_ttft_seconds": { + "mean": statistics.mean(ttft) if ttft else None, + "p95": observed(ttft, 0.95), + "p99": observed(ttft, 0.99), + "max": max(ttft) if ttft else None, + }, + "max_token_gap_seconds": { + "mean": statistics.mean(gaps) if gaps else None, + "p95": observed(gaps, 0.95), + "p99": observed(gaps, 0.99), + "max": max(gaps) if gaps else None, + }, + "status": "passed" if len(records) == expected and all(item["status"] == "passed" for item in records) else "failed", +} +(root / "summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8") +print(json.dumps(summary, sort_keys=True)) +PY diff --git a/scripts/gmk-evo-x2/run_qwen_q4_rocprof_trace.sh b/scripts/gmk-evo-x2/run_qwen_q4_rocprof_trace.sh new file mode 100755 index 0000000000..8cefdd2abd --- /dev/null +++ b/scripts/gmk-evo-x2/run_qwen_q4_rocprof_trace.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# Capture a safe, warm-cache ROCprof trace for the isolated Qwen3.6 Q4 control. +# +# This controller deliberately treats profiling as a diagnostic time-share job, +# not as a TPS benchmark. It first checks the normal NVFP4 API, prewarms the +# exact Q4 HIP cache without the profiler, traces one bounded request, requests +# graceful profiler finalization, verifies a SQLite trace database, and only +# then restarts the normal service. It never exposes the candidate beyond its +# loopback-only test port and never contacts llama-swap. + +# Exit for programming errors, unset values, and failed pipeline stages. +set -euo pipefail + +# Require a unique caller-owned artifact directory for immutable provenance. +readonly ARTIFACT_DIR="${1:?usage: run_qwen_q4_rocprof_trace.sh ARTIFACT_DIR [SOURCE_DIR]}" +# Permit an explicit reviewed Qwen checkout while keeping the qualified source +# as the default for ordinary diagnostic traces. +readonly SOURCE_DIR="${2:-${HOME}/freetoken-amd/source-qwen-bench-metrics-f1baf13}" +# Keep all fixed host paths together so they are easy to audit before use. +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +readonly NORMAL_SOURCE_DIR="${ROOT_DIR}/source-qwen-c06-fc3346f" +readonly MODEL_PATH="${ROOT_DIR}/models/controls/qwen36-35b-a3b-unsloth-a483e9e6/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf" +readonly VENV_PYTHON="${ROOT_DIR}/.venv/bin/python" +readonly Q4_MODEL_NAME="qwen36-35b-a3b-q4km-gguf-amd" +readonly Q4_PORT="1922" +readonly NORMAL_PORT="1919" +# Keep reusable generated HIP code outside every source checkout and under the +# managed cache root accepted by the qualified Q4 launcher. +readonly EXTENSION_CACHE="${ROOT_DIR}/cache/rocprof-q4-warm-cache" +readonly Q4_LAUNCHER="${SOURCE_DIR}/scripts/gmk-evo-x2/launch_qwen_gguf_qualified.sh" +readonly NORMAL_STARTER="${NORMAL_SOURCE_DIR}/scripts/gmk-evo-x2/start_qwen_recovery_server.sh" +readonly NORMAL_STOPPER="${NORMAL_SOURCE_DIR}/scripts/gmk-evo-x2/stop_qwen_recovery_server.sh" +readonly PROFILER_WRAPPER="${SOURCE_DIR}/scripts/gmk-evo-x2-rocprof-wheel-sdk.sh" +readonly INSPECTOR="${SOURCE_DIR}/scripts/gmk-evo-x2/inspect_rocprof_db.py" +readonly PROFILE_DIR="${ARTIFACT_DIR}/profile" +readonly PREWARM_DIR="${ARTIFACT_DIR}/prewarm" +readonly PROFILE_PID_FILE="${PROFILE_DIR}/profile.pid" +readonly PROFILE_LOG="${PROFILE_DIR}/server.log" + +# Track lifecycle ownership so the EXIT trap restores only services this script +# actually stopped or started. +normal_was_stopped=0 +profile_pid="" + +# Return success only when the expected loopback API has completed a models call. +wait_for_models() { + local port="$1" + local attempts="$2" + local destination="$3" + for _ in $(seq 1 "${attempts}"); do + if curl -fsS --max-time 8 "http://127.0.0.1:${port}/v1/models" >"${destination}"; then + return 0 + fi + sleep 2 + done + return 1 +} + +# Wait for the scheduler's explicit ready record, not merely the frontend +# listener. Uvicorn answers `/v1/models` before the model scheduler has built +# its expert banks, and requests during that interval correctly return HTTP 503. +wait_for_scheduler_ready() { + local log_file="$1" + local attempts="$2" + for _ in $(seq 1 "${attempts}"); do + if [[ -f "${log_file}" ]] && rg -Fq 'API server is ready to serve' "${log_file}"; then + return 0 + fi + sleep 2 + done + return 1 +} + +# Wait for a test listener to disappear before reusing the GPU or port. +wait_for_port_clear() { + for _ in $(seq 1 45); do + if ! ss -ltn "( sport = :${Q4_PORT} )" | grep -q LISTEN; then + return 0 + fi + sleep 2 + done + return 1 +} + +# Confirm the recorded profiler process is the exact isolated Q4 server before +# sending it an interrupt. This guard makes the shared host safe to operate. +is_profile_process() { + local pid="$1" + local command + [[ "${pid}" =~ ^[0-9]+$ ]] || return 1 + [[ -r "/proc/${pid}/cmdline" ]] || return 1 + command="$(tr '\0' ' ' < "/proc/${pid}/cmdline")" + [[ "${command}" == *"freetoken.cli serve"* ]] && + [[ "${command}" == *"${MODEL_PATH}"* ]] && + [[ "${command}" == *"--port ${Q4_PORT}"* ]] +} + +# Request graceful rocprof finalization first. The bounded wait protects the +# normal service from an indefinitely stuck trace while preserving enough time +# for rocprofv3 to write its SQLite database. +finalize_profile() { + [[ -n "${profile_pid}" ]] || return 0 + kill -0 "${profile_pid}" 2>/dev/null || return 0 + is_profile_process "${profile_pid}" || { + echo "refusing to stop an unrecognized profiler process: ${profile_pid}" >&2 + return 1 + } + local pgid + pgid="$(ps -o pgid= -p "${profile_pid}" | tr -d ' ')" + [[ "${pgid}" == "${profile_pid}" ]] || { + echo "profiler process lacks its dedicated group: ${profile_pid}/${pgid}" >&2 + return 1 + } + kill -INT -- "-${pgid}" || true + for _ in $(seq 1 90); do + kill -0 "${profile_pid}" 2>/dev/null || return 0 + sleep 1 + done + echo "profiler did not finalize within 90 seconds" >&2 + return 1 +} + +# Always restore the normal API when this controller owns its time-share slot. +restore_normal_service() { + local status="$?" + set +e + finalize_profile + if [[ "${normal_was_stopped}" == "1" ]]; then + bash "${NORMAL_STARTER}" "${ARTIFACT_DIR}/normal-recovery" || true + wait_for_models "${NORMAL_PORT}" 240 "${ARTIFACT_DIR}/normal-health-after.json" || true + fi + exit "${status}" +} + +# Install recovery before stopping the normal service so interrupts do not leave +# the GMKtek EVO-X2 without its normal local OpenAI-compatible endpoint. +trap restore_normal_service EXIT INT TERM + +# Fail closed when a caller supplies an unexpected source tree or missing tools. +[[ "${SOURCE_DIR}" == "${ROOT_DIR}/source-qwen-"* ]] || { echo "invalid source directory" >&2; exit 2; } +[[ -f "${MODEL_PATH}" && -x "${VENV_PYTHON}" && -x "${Q4_LAUNCHER}" ]] || { echo "missing Q4 prerequisites" >&2; exit 2; } +[[ -x "${NORMAL_STARTER}" && -x "${NORMAL_STOPPER}" && -x "${PROFILER_WRAPPER}" ]] || { echo "missing lifecycle helper" >&2; exit 2; } +[[ -f "${INSPECTOR}" ]] || { echo "missing ROCprof inspector" >&2; exit 2; } +[[ ! -e "${ARTIFACT_DIR}" ]] || { echo "artifact directory already exists: ${ARTIFACT_DIR}" >&2; exit 2; } +mkdir -p "${PREWARM_DIR}" "${PROFILE_DIR}" "${EXTENSION_CACHE}" + +# Preserve proof that the protected API was healthy before reclaiming the GPU. +wait_for_models "${NORMAL_PORT}" 1 "${ARTIFACT_DIR}/normal-health-before.json" || { + echo "normal API is not healthy; refusing profiler time-share" >&2 + exit 2 +} + +# Reserve the GPU through the existing tested stopper, then prewarm the exact +# Q4 server and extension cache without ROCprof overhead. +bash "${NORMAL_STOPPER}" +normal_was_stopped=1 +FREETOKEN_Q4_SOURCE_DIR="${SOURCE_DIR}" FREETOKEN_Q4_EXTENSION_CACHE_DIR="${EXTENSION_CACHE}" \ + bash "${Q4_LAUNCHER}" start "${PREWARM_DIR}" 0.25 0 +wait_for_models "${Q4_PORT}" 120 "${PREWARM_DIR}/models.json" +wait_for_scheduler_ready "${PREWARM_DIR}/server.log" 120 + +# Force one short deterministic request so startup and kernel compilation occur +# before tracing. The response is retained as evidence, not scored for TPS. +curl -fsS --max-time 90 -H 'Content-Type: application/json' \ + -d "{\"model\":\"${Q4_MODEL_NAME}\",\"messages\":[{\"role\":\"user\",\"content\":\"Reply with exactly: warm cache confirmed\"}],\"temperature\":0,\"max_tokens\":16,\"stream\":false}" \ + "http://127.0.0.1:${Q4_PORT}/v1/chat/completions" >"${PREWARM_DIR}/response.json" +FREETOKEN_Q4_SOURCE_DIR="${SOURCE_DIR}" FREETOKEN_Q4_EXTENSION_CACHE_DIR="${EXTENSION_CACHE}" \ + bash "${Q4_LAUNCHER}" stop "${PREWARM_DIR}" 0.25 0 +wait_for_port_clear + +# Start the identical Q4 command inside a dedicated session. The profile has a +# short delayed collection window that excludes most initialization, while the +# post-ready request below remains inside the 75-second capture interval. +cd "${SOURCE_DIR}" +ROCM_HOME=/opt/rocm-10.0 ROCM_PATH=/opt/rocm-10.0 HIP_PATH=/opt/rocm-10.0 \ +PYTHONPATH=python TORCH_EXTENSIONS_DIR="${EXTENSION_CACHE}" \ +setsid nohup bash "${PROFILER_WRAPPER}" -d "${PROFILE_DIR}/rocprof" -f rocpd \ + --runtime-trace --kernel-trace --memory-copy-trace --collection-period 45:75:1 \ + --process-sync true -- "${VENV_PYTHON}" -m freetoken.cli serve \ + --model-path "${MODEL_PATH}" --served-model-name "${Q4_MODEL_NAME}" \ + --host 127.0.0.1 --port "${Q4_PORT}" --max-running-requests 4 \ + --attention-backend triton --moe-backend offload --nvfp4-backend triton \ + --expert-load serial --moe-cache-auto --memory-ratio 0.25 \ + --max-seq-len-override 8192 --kv-reserve-tokens 8192 --cuda-graph-max-bs 0 \ + --disable-pynccl --disable-moe-prefill-overlap >"${PROFILE_LOG}" 2>&1 & +profile_pid="$!" +printf '%s\n' "${profile_pid}" >"${PROFILE_PID_FILE}" +wait_for_models "${Q4_PORT}" 120 "${PROFILE_DIR}/models.json" +wait_for_scheduler_ready "${PROFILE_LOG}" 120 + +# Generate one bounded greedy request. Its fixed shape gives the trace a clear +# prefill and decode region while avoiding a variable reasoning-stream workload. +curl -fsS --max-time 120 -H 'Content-Type: application/json' \ + -d "{\"model\":\"${Q4_MODEL_NAME}\",\"messages\":[{\"role\":\"user\",\"content\":\"Write exactly 300 numbered lines. Every line must contain the words cache and expert.\"}],\"temperature\":0,\"top_p\":1,\"max_tokens\":900,\"stream\":false}" \ + "http://127.0.0.1:${Q4_PORT}/v1/chat/completions" >"${PROFILE_DIR}/workload-response.json" + +# Give asynchronous ROCprof writers a short post-request interval before the +# graceful interrupt, then require one finalized SQLite database as the gate. +sleep 5 +finalize_profile +profile_pid="" +database="$(find "${PROFILE_DIR}/rocprof" -type f -name '*_results.db' -print -quit)" +[[ -n "${database}" ]] || { echo "ROCprof database was not finalized" >&2; exit 3; } +"${VENV_PYTHON}" "${INSPECTOR}" --tail-seconds 30 "${database}" >"${PROFILE_DIR}/trace-summary.txt" +printf 'database=%s\n' "${database}" >"${PROFILE_DIR}/trace-database.txt" diff --git a/scripts/gmk-evo-x2/run_qwen_scheduler_baseline.sh b/scripts/gmk-evo-x2/run_qwen_scheduler_baseline.sh new file mode 100755 index 0000000000..80f916acb5 --- /dev/null +++ b/scripts/gmk-evo-x2/run_qwen_scheduler_baseline.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Measure warm Qwen decode throughput against the isolated GMKtek EVO-X2 FreeToken API. +# +# The workload is deliberately a fixed 48-times scheduler paragraph. It preserves +# the former 733-token-class GMKtek EVO-X2 baseline shape while remaining separate from +# the unrecovered upstream paper workload. This script neither starts nor stops a +# server and never contacts llama-swap or any non-GMKtek EVO-X2 endpoint. + +set -euo pipefail + +# Accept a caller-supplied artifact root so each run has immutable evidence. +readonly ARTIFACT_DIR="${1:?usage: run_qwen_scheduler_baseline.sh ARTIFACT_DIR}" +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +readonly SOURCE_DIR="${ROOT_DIR}/source-qwen-harness-d6ee8ce" +# Keep benchmark code independent from the source checkout serving the normal +# API. A deployed server checkout can intentionally stay frozen while a newer +# isolated checkout supplies the reviewed benchmark harness. This override +# changes neither the target URL nor the model selected below. +readonly BENCHMARK_SOURCE_DIR="${GMK_EVO_X2_QWEN_BENCHMARK_SOURCE_DIR:-${SOURCE_DIR}}" +readonly VENV_PYTHON="${ROOT_DIR}/.venv/bin/python" +# Preserve the original FreeToken service as the default while permitting an +# explicitly named, isolated local control to reuse this exact workload. The +# optional overrides are intentionally not exported globally, so ordinary +# service runs remain bound to port 1919 and the validated FreeToken model. +readonly MODEL_DIR="${GMK_EVO_X2_QWEN_TOKENIZER_DIR:-${ROOT_DIR}/models/Qwen3.6-35B-A3B-NVFP4}" +readonly MODEL_NAME="${GMK_EVO_X2_QWEN_MODEL_NAME:-qwen3.6-35b-a3b-nvfp4-amd}" +readonly BASE_URL="${GMK_EVO_X2_QWEN_BASE_URL:-http://127.0.0.1:1919/v1}" +readonly EXPECTED_HOST="${FREETOKEN_EXPECTED_HOST:?Set FREETOKEN_EXPECTED_HOST to the approved test hostname}" +readonly BASE_PROMPT="The scheduler manages incoming inference requests by prioritizing, batching, and assigning them to available compute resources to optimize throughput and latency. " + +# Form the fixed input without shell interpolation at call time. The harness +# records its SHA-256 and checkpoint token count, so any future wording change +# becomes visible in the result artifact rather than silently changing TPS. +PROMPT="" +for _ in $(seq 1 48); do + PROMPT+="${BASE_PROMPT}" +done + +# Refuse a stale deployment explicitly instead of failing later with Python's +# unhelpful file-not-found message. This makes source provenance visible in the +# artifact-producing command and prevents an accidental benchmark substitution. +test -f "${BENCHMARK_SOURCE_DIR}/benchmarks/gmk_evo_x2/run_api_benchmark.py" +export PYTHONPATH="${BENCHMARK_SOURCE_DIR}/python" +cd "${BENCHMARK_SOURCE_DIR}" + +# Forced-length greedy decoding yields a comparable stream interval. Qwen's +# reasoning stream is explicitly disabled because this measures final-token +# decoding, not variable-length internal reasoning. A warmup is retained but +# saved separately by the harness before the three scored samples. +"${VENV_PYTHON}" benchmarks/gmk_evo_x2/run_api_benchmark.py \ + --model "${MODEL_NAME}" \ + --tokenizer "${MODEL_DIR}" \ + --base-url "${BASE_URL}" \ + --expected-host "${EXPECTED_HOST}" \ + --artifact-dir "${ARTIFACT_DIR}" \ + --samples 3 \ + --warmup \ + --mode throughput \ + --expected-text "" \ + --max-tokens 256 \ + --prompt "${PROMPT}" \ + --reasoning-effort none diff --git a/scripts/gmk-evo-x2/start_qwen_recovery_server.sh b/scripts/gmk-evo-x2/start_qwen_recovery_server.sh new file mode 100755 index 0000000000..d83f27573b --- /dev/null +++ b/scripts/gmk-evo-x2/start_qwen_recovery_server.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# Start the isolated FreeToken Qwen NVFP4 recovery server on GMKtek EVO-X2. +# +# This script never touches systemd, llama-swap, or the masked production +# llama.cpp service on port 18302. It launches one loopback-only FreeToken +# process on port 1919 and writes all output into a uniquely timestamped +# artifact directory so post-reboot results remain reproducible. + +set -euo pipefail + +# Keep every recovery run separate from previous logs and benchmark artifacts. +readonly RUN_ID="qwen-reboot-recovery-$(date -u +%Y%m%dT%H%M%SZ)" +readonly ROOT_DIR="${FREETOKEN_ROOT_DIR:-${HOME}/freetoken-amd}" +readonly SOURCE_DIR="${ROOT_DIR}/source-qwen-harness-d6ee8ce" +readonly VENV_PYTHON="${ROOT_DIR}/.venv/bin/python" +readonly MODEL_DIR="${ROOT_DIR}/models/Qwen3.6-35B-A3B-NVFP4" +# Pair a strict native cache with the exact source revision that built it. +# Unlike a generic Torch extension directory, the FreeToken cache identifies +# individual helper-kernel ABI names, so loading objects built from another +# source revision could silently defeat the no-JIT guarantee. +readonly SOURCE_REVISION="$(git -C "${SOURCE_DIR}" rev-parse --short=12 HEAD)" +readonly ROCM_KERNEL_CACHE_DIR="${FREETOKEN_ROCM_KERNEL_CACHE_DIR:-${ROOT_DIR}/cache/kernel-cache-rocm-gfx1151-${SOURCE_REVISION}}" +readonly MEMORY_RATIO="${FREETOKEN_MEMORY_RATIO:-0.35}" +# The previous 2,048-token reserve made the advertised 8,192-token sequence +# limit unreachable because --moe-cache-auto allocated the remaining budget to +# experts. GMKtek EVO-X2 validation proved an 8,192-token reserve keeps zero swap, +# preserves short-decode TPS, and enables a real 6,856-token cold-prefill test. +# Permit a small, explicit set of recovery overrides for isolated experiments. +readonly KV_RESERVE_TOKENS="${FREETOKEN_KV_RESERVE_TOKENS:-8192}" +readonly CUDA_GRAPH_MAX_BS="${FREETOKEN_CUDA_GRAPH_MAX_BS:-0}" +readonly FP8_GEMV_BLOCK_N="${FREETOKEN_FP8_GEMV_BLOCK_N:-16}" +readonly FP8_GEMV_NUM_WARPS="${FREETOKEN_FP8_GEMV_NUM_WARPS:-1}" +readonly FP8_GEMV_SCALE_ACTIVATION="${FREETOKEN_FP8_GEMV_SCALE_ACTIVATION:-0}" +readonly MOE_COLLECT_STATS="${FREETOKEN_MOE_COLLECT_STATS:-0}" +readonly FUSED_COPY_BLOCKS_PER_BANK="${FREETOKEN_FUSED_COPY_BLOCKS_PER_BANK:-8}" +readonly ARTIFACT_DIR="${ROOT_DIR}/artifacts/${RUN_ID}" +readonly LOG_FILE="${ARTIFACT_DIR}/server.log" +readonly PID_FILE="${ARTIFACT_DIR}/server.pid" +readonly NATIVE_BUILD_LOG="${ARTIFACT_DIR}/native-extension-build.log" +readonly NATIVE_IMPORT_LOG="${ARTIFACT_DIR}/native-extension-import.txt" + +# Refuse to launch if another process already owns the dedicated test port. +if ss -ltn "sport = :1919" | grep -q LISTEN; then + echo "refusing to start: loopback benchmark port 1919 is already listening" >&2 + exit 1 +fi + +# Validate all immutable runtime inputs before starting a background process. +test -d "${SOURCE_DIR}" +test -x "${VENV_PYTHON}" +test -d "${MODEL_DIR}" +test -d "${ROCM_KERNEL_CACHE_DIR}" +case "${MEMORY_RATIO}" in + 0.[0-9][0-9]) ;; + *) echo "invalid FREETOKEN_MEMORY_RATIO: ${MEMORY_RATIO}" >&2; exit 2 ;; +esac +case "${KV_RESERVE_TOKENS}" in + 2048|4096|8192) ;; + *) echo "invalid FREETOKEN_KV_RESERVE_TOKENS: ${KV_RESERVE_TOKENS}" >&2; exit 2 ;; +esac +case "${CUDA_GRAPH_MAX_BS}" in + 0|1|2|4|8) ;; + *) echo "invalid FREETOKEN_CUDA_GRAPH_MAX_BS: ${CUDA_GRAPH_MAX_BS}" >&2; exit 2 ;; +esac +case "${FP8_GEMV_BLOCK_N}" in + 16|32) ;; + *) echo "invalid FREETOKEN_FP8_GEMV_BLOCK_N: ${FP8_GEMV_BLOCK_N}" >&2; exit 2 ;; +esac +case "${FP8_GEMV_NUM_WARPS}" in + 1|2|4) ;; + *) echo "invalid FREETOKEN_FP8_GEMV_NUM_WARPS: ${FP8_GEMV_NUM_WARPS}" >&2; exit 2 ;; +esac +case "${FP8_GEMV_SCALE_ACTIVATION}" in + 0|1) ;; + *) echo "invalid FREETOKEN_FP8_GEMV_SCALE_ACTIVATION: ${FP8_GEMV_SCALE_ACTIVATION}" >&2; exit 2 ;; +esac +case "${MOE_COLLECT_STATS}" in + 0|1) ;; + *) echo "invalid FREETOKEN_MOE_COLLECT_STATS: ${MOE_COLLECT_STATS}" >&2; exit 2 ;; +esac +case "${FUSED_COPY_BLOCKS_PER_BANK}" in + 8|64) ;; + *) echo "invalid FREETOKEN_FUSED_COPY_BLOCKS_PER_BANK: ${FUSED_COPY_BLOCKS_PER_BANK}" >&2; exit 2 ;; +esac +mkdir -p "${ARTIFACT_DIR}" + +# These variables select the native ROCm toolchain and retain the existing HIP +# extension cache. Reusing the cache prevents a JIT build from contaminating the +# warm API benchmark that follows server readiness. +export PYTHONPATH="${SOURCE_DIR}/python" +export TORCH_EXTENSIONS_DIR="${ROOT_DIR}/cache/torch_extensions" +export ROCM_PATH="/opt/rocm-10.0" +export HIP_PATH="/opt/rocm-10.0" +export ROCM_HOME="/opt/rocm-10.0" +# The completed gfx1151 cache contains every valid C++/HIP helper in the +# FreeToken catalog. Make the server resolve objects only from that cache and +# fail explicitly if a source edit introduces a missing specialization. Triton +# keeps its own persistent code cache; this flag governs FreeToken's C++/HIP +# helper JIT rather than disabling native Triton execution. +export FREETOKEN_KERNEL_CACHE_DIR="${ROCM_KERNEL_CACHE_DIR}" +export FREETOKEN_DISABLE_JIT=1 +# Pass the explicitly recorded FP8 output-row tile to the isolated process. +# The code permits only 16 (validated baseline) and 32 (a deterministic, +# quality-gated gfx1151 candidate), so an accidental shell value cannot create +# an untracked Triton specialization. +export FREETOKEN_FP8_GEMV_BLOCK_N="${FP8_GEMV_BLOCK_N}" +# Keep every additional kernel specialization explicit in the artifact's +# launch environment. This makes a subsequent quality failure attributable to +# one bounded variable rather than an implicit, inherited shell setting. +export FREETOKEN_FP8_GEMV_NUM_WARPS="${FP8_GEMV_NUM_WARPS}" +export FREETOKEN_FP8_GEMV_SCALE_ACTIVATION="${FP8_GEMV_SCALE_ACTIVATION}" +# Both fused-copy grid widths are precompiled into the strict gfx1151 cache. +# The default eight blocks is the established service baseline; sixty-four is +# an isolated, quality-gated copy-path candidate. +export FREETOKEN_FUSED_COPY_BLOCKS_PER_BANK="${FUSED_COPY_BLOCKS_PER_BANK}" + +# Cache counters are opt-in because their atomic updates are diagnostic work. +# The default leaves the verified performance service unchanged, while an +# isolated launch can enable a single post-workload read-only snapshot. +EXTRA_ARGS=() +if [[ "${MOE_COLLECT_STATS}" == "1" ]]; then + EXTRA_ARGS+=(--moe-collect-stats) +fi + +# FreeToken's MoE offload path requires the in-tree pinned-tensor extension. +# A clean git worktree does not contain generated shared objects, so verify the +# import first and build the two native modules in that worktree only when it +# is absent. The build log is an artifact because the extension's compiler, +# ROCm headers, and link result are part of a reproducible HIP validation. +if ! "${VENV_PYTHON}" -c 'import freetoken.kernel._pinned_tensor' >/dev/null 2>&1; then + ( + cd "${SOURCE_DIR}" + "${VENV_PYTHON}" setup.py build_ext --inplace + ) >"${NATIVE_BUILD_LOG}" 2>&1 +fi +"${VENV_PYTHON}" -c \ + 'import freetoken.kernel._pinned_tensor as pinned; print(pinned.__file__)' \ + >"${NATIVE_IMPORT_LOG}" + +# The fixed policy is the validated GMKtek EVO-X2 Qwen configuration. The default +# 0.35 memory budget and 8,192-token KV reserve make the advertised context +# limit real while --moe-cache-auto retains as many MoE experts as safely fit. +# A constrained environment override supports isolated cache-capacity controls +# without editing the server command. Serial expert loading is the ROCm-correct route and prefill +# overlap stays disabled for the validated safe baseline. Graph capture defaults +# to zero because ROCm correctness takes priority; the bounded override enables +# an isolated batch-size experiment without changing the baseline command. The +# FP8 row-tile override changes neither split-K partitioning nor reduction order +# and is only used with a separately saved deterministic quality result. Wave +# count and activation scaling are likewise disabled defaults and require their +# own raw-output plus model-level quality evidence before any promotion. +# `setsid` gives this complete multiprocessing server a dedicated process +# group. A later controlled stop can therefore release the frontend, scheduler, +# tokenizer, and tracker together instead of leaving a GPU-owning child behind. +setsid nohup "${VENV_PYTHON}" -m freetoken.cli serve \ + --model-path "${MODEL_DIR}" \ + --served-model-name qwen3.6-35b-a3b-nvfp4-amd \ + --host 127.0.0.1 \ + --port 1919 \ + --attention-backend triton \ + --moe-backend offload \ + --nvfp4-backend triton \ + --expert-load serial \ + --moe-cache-auto \ + --memory-ratio "${MEMORY_RATIO}" \ + --max-seq-len-override 8192 \ + --kv-reserve-tokens "${KV_RESERVE_TOKENS}" \ + --cuda-graph-max-bs "${CUDA_GRAPH_MAX_BS}" \ + --disable-pynccl \ + --disable-moe-prefill-overlap \ + "${EXTRA_ARGS[@]}" \ + >"${LOG_FILE}" 2>&1 < /dev/null & + +# Persist the child PID for diagnostics and explicit shutdown after the run. +echo "$!" >"${PID_FILE}" +printf '%s\n' "${ARTIFACT_DIR}" diff --git a/scripts/gmk-evo-x2/stop_qwen_recovery_server.sh b/scripts/gmk-evo-x2/stop_qwen_recovery_server.sh new file mode 100755 index 0000000000..6db7b82373 --- /dev/null +++ b/scripts/gmk-evo-x2/stop_qwen_recovery_server.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Stop only the GMKtek EVO-X2 loopback NVFP4 recovery server as one process group. +# +# The FreeToken frontend creates scheduler and tokenizer child processes. A +# parent-only signal can leave one of those children holding GPU memory or the +# internal distributed port. This helper verifies the listener's exact model +# and port before signalling its dedicated session, so it cannot target an +# unrelated service on the shared machine. + +set -euo pipefail + +# Keep the protected service identity explicit rather than inferring it from a +# PID file that might be stale after a reboot or failed experimental run. +readonly PORT="1919" +readonly MODEL_PATH="${HOME}/freetoken-amd/models/Qwen3.6-35B-A3B-NVFP4" + +# Resolve the actual TCP listener because it is the authoritative owner of the +# endpoint that this helper is permitted to stop. +listener_pid() { + ss -ltnp "( sport = :${PORT} )" | sed -n 's/.*pid=\([0-9]*\).*/\1/p' | head -1 +} + +# Require the intended FreeToken command and model path before a process group +# signal. This prevents an accidental port reuse from becoming a destructive +# signal to another local application. +is_recovery_server() { + local pid="$1" command + [[ "${pid}" =~ ^[0-9]+$ ]] || return 1 + [[ -r "/proc/${pid}/cmdline" ]] || return 1 + command="$(tr '\0' ' ' < "/proc/${pid}/cmdline")" + [[ "${command}" == *"freetoken.cli serve"* ]] || return 1 + [[ "${command}" == *"${MODEL_PATH}"* ]] || return 1 + [[ "${command}" == *"--port ${PORT}"* ]] +} + +# A successful start uses setsid, making the server PID its own process-group +# ID. Refuse legacy non-isolated launches rather than guessing which children +# belong to the server. The caller can keep the service running and inspect it. +stop_server() { + local pid="$1" pgid + is_recovery_server "${pid}" || { + echo "refusing to stop an unrecognized port ${PORT} listener: ${pid}" >&2 + return 1 + } + pgid="$(ps -o pgid= -p "${pid}" | tr -d ' ')" + [[ "${pgid}" == "${pid}" ]] || { + echo "refusing legacy non-isolated recovery server ${pid}; restart it with the current launcher first" >&2 + return 1 + } + kill -TERM -- "-${pgid}" || true + for _ in $(seq 1 90); do + kill -0 "${pid}" 2>/dev/null || break + sleep 1 + done + # Escalate only the already-verified dedicated process group if a stuck HIP + # operation prevented graceful Python shutdown. + kill -0 "${pid}" 2>/dev/null && kill -KILL -- "-${pgid}" || true +} + +pid="$(listener_pid)" +[[ -n "${pid}" ]] || { echo "no recovery server is listening on port ${PORT}"; exit 0; } +stop_server "${pid}" + +# Confirm the port was released before the caller starts an isolated candidate. +for _ in $(seq 1 15); do + [[ -z "$(listener_pid)" ]] && exit 0 + sleep 1 +done +echo "recovery server listener remained on port ${PORT}" >&2 +exit 1 diff --git a/scripts/gmk-evo-x2/verify_gemma4_gguf_image.py b/scripts/gmk-evo-x2/verify_gemma4_gguf_image.py new file mode 100644 index 0000000000..5270f96e94 --- /dev/null +++ b/scripts/gmk-evo-x2/verify_gemma4_gguf_image.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Run one deterministic OpenAI image control against an isolated Gemma4 server. + +The generated solid-red PNG removes network and copyrighted-image variables. It +still exercises every production-relevant multimodal boundary: OpenAI content +parts, data-URL decoding, Gemma4 resize/patching, shaped ZMQ tensors, the ROCm +vision tower, projector, image-slot scatter, and response formatting. +""" + +from __future__ import annotations + +import argparse +import base64 +import io +import json +import time +import urllib.error +import urllib.request +from pathlib import Path + +from PIL import Image + + +def _png_data_url(image: Image.Image) -> str: + """Encode one generated RGB fixture as an OpenAI-compatible data URL.""" + buf = io.BytesIO() + image.save(buf, format="PNG") + return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode("ascii") + + +def _post_json(url: str, payload: dict) -> dict: + """Send one bounded JSON request and decode the server's JSON response.""" + body = json.dumps(payload).encode("utf-8") + request = urllib.request.Request(url, data=body, headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(request, timeout=180) as response: # nosec B310: caller controls local base URL + return json.loads(response.read().decode("utf-8")) + + +def _post_json_stream(url: str, payload: dict) -> tuple[dict, dict]: + """Stream one OpenAI response and retain timing plus the final message. + + The returned metrics deliberately distinguish the server-reported completion + token count from the number of network chunks. A chunk is not necessarily a + token, so TPS is calculated only when final OpenAI usage is available. + """ + payload = {**payload, "stream": True, "stream_options": {"include_usage": True}} + body = json.dumps(payload).encode("utf-8") + request = urllib.request.Request(url, data=body, headers={"Content-Type": "application/json"}) + started = time.perf_counter() + first_chunk: float | None = None + last_chunk: float | None = None + content: list[str] = [] + reasoning: list[str] = [] + usage: dict = {} + try: + with urllib.request.urlopen(request, timeout=300) as response: # nosec B310: caller controls local base URL + for raw in response: + line = raw.decode("utf-8").strip() + if not line.startswith("data: "): + continue + data = line[6:] + if data == "[DONE]": + break + event = json.loads(data) + usage = event.get("usage") or usage + for choice in event.get("choices", []): + delta = choice.get("delta", {}) + piece = delta.get("content") or "" + thought = delta.get("reasoning_content") or "" + if piece or thought: + now = time.perf_counter() + first_chunk = first_chunk if first_chunk is not None else now + last_chunk = now + content.append(piece) + reasoning.append(thought) + except urllib.error.HTTPError as exc: + # The server's JSON body explains whether the image wire shape, template, + # tokenizer, or vision model rejected the request. Preserve it in the + # raised error instead of leaving only an uninformative HTTP status. + detail = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"image request HTTP {exc.code}: {detail}") from exc + elapsed = time.perf_counter() - started + completion = usage.get("completion_tokens") + generated_window = (last_chunk - first_chunk) if first_chunk is not None and last_chunk is not None else 0.0 + metrics = { + "wall_s": elapsed, + "ttft_ms": (first_chunk - started) * 1000 if first_chunk is not None else None, + "stream_window_s": generated_window, + "completion_tokens": completion, + "completion_tok_s": completion / generated_window if completion and generated_window else None, + } + return { + "choices": [{"message": {"role": "assistant", "content": "".join(content), "reasoning_content": "".join(reasoning)}}], + "usage": usage, + }, metrics + + +def _two_color_image( + size: tuple[int, int], first: tuple[int, int, int], second: tuple[int, int, int], *, horizontal: bool +) -> Image.Image: + """Build one sharp two-color spatial fixture without external image assets. + + ``horizontal=True`` means the first color occupies the left half and the + second color occupies the right half. ``False`` places the first color on + top. Keeping this construction local and procedural makes the exact input + bytes reproducible while exercising real image decoding and preprocessing. + """ + width, height = size + image = Image.new("RGB", size, second) + if horizontal: + for x in range(width // 2): + for y in range(height): + image.putpixel((x, y), first) + else: + for x in range(width): + for y in range(height // 2): + image.putpixel((x, y), first) + return image + + +def main() -> int: + """Run color and spatial fixtures, validate exact answers, and save evidence.""" + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--artifact", type=Path, required=True) + parser.add_argument( + "--max-tokens", type=int, default=16, + help="per-case generation cap; llama.cpp needs a larger cap when it emits thought first", + ) + parser.add_argument("--stream", action="store_true", help="capture stream timing and final usage") + parser.add_argument( + "--extended", + action="store_true", + help="add deterministic blue, yellow, right-half, and top-half controls after core parity passes", + ) + parser.add_argument( + "--repetitions", + type=int, + default=1, + help="run the complete selected fixture set this many times against the same ready server", + ) + args = parser.parse_args() + if args.repetitions < 1: + parser.error("--repetitions must be at least one") + + split = _two_color_image((96, 48), (255, 0, 0), (0, 0, 255), horizontal=True) + cases = [ + ("solid_red", Image.new("RGB", (16, 16), (255, 0, 0)), + "What is the dominant color in the image? Reply with one lowercase word.", "red"), + ("solid_green", Image.new("RGB", (16, 16), (0, 255, 0)), + "What is the dominant color in the image? Reply with one lowercase word.", "green"), + ("red_left_blue_right", split, + "What color is the left half of the image? Reply with one lowercase word.", "red"), + ] + if args.extended: + # These controls deliberately ask about the opposite half and a vertical + # layout. They catch a pipeline that recognizes colors but reverses or + # otherwise loses spatial coordinates after patch pooling. + cases.extend([ + ("solid_blue", Image.new("RGB", (16, 16), (0, 0, 255)), + "What is the dominant color in the image? Reply with one lowercase word.", "blue"), + ("solid_yellow", Image.new("RGB", (16, 16), (255, 255, 0)), + "What is the dominant color in the image? Reply with one lowercase word.", "yellow"), + ("red_left_blue_right_right_half", split, + "What color is the right half of the image? Reply with one lowercase word.", "blue"), + ("blue_top_yellow_bottom_top_half", + _two_color_image((48, 96), (0, 0, 255), (255, 255, 0), horizontal=False), + "What color is the top half of the image? Reply with one lowercase word.", "blue"), + ]) + records = [] + for repetition in range(1, args.repetitions + 1): + for name, image, prompt, expected in cases: + request = { + "model": args.model, + "messages": [{"role": "user", "content": [ + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": _png_data_url(image)}}, + ]}], + "temperature": 0, + "max_tokens": args.max_tokens, + } + started = time.perf_counter() + metrics = None + if args.stream: + response, metrics = _post_json_stream(args.base_url.rstrip("/") + "/v1/chat/completions", request) + else: + response = _post_json(args.base_url.rstrip("/") + "/v1/chat/completions", request) + text = response["choices"][0]["message"]["content"].strip().lower() + records.append({ + "control": name, + "repetition": repetition, + "prompt": prompt, + "expected": expected, + "actual": text, + "elapsed_s": time.perf_counter() - started, + "stream_metrics": metrics, + "usage": response.get("usage"), + "response": response, + }) + record = { + "schema_version": 3, + "fixture_set": "extended" if args.extended else "core", + "repetitions": args.repetitions, + "passed": all(item["actual"] == item["expected"] for item in records), + "cases": records, + } + args.artifact.parent.mkdir(parents=True, exist_ok=True) + args.artifact.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + if not record["passed"]: + failures = [f"{item['control']}: expected {item['expected']!r}, got {item['actual']!r}" for item in records if item["actual"] != item["expected"]] + raise SystemExit("Gemma4 image control failed: " + "; ".join(failures)) + print(json.dumps(record, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/verify_gemma4_gguf_text.py b/scripts/gmk-evo-x2/verify_gemma4_gguf_text.py new file mode 100644 index 0000000000..12b22ae86d --- /dev/null +++ b/scripts/gmk-evo-x2/verify_gemma4_gguf_text.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Capture a reproducible text-only Gemma4 GGUF quality and decode control.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import time +import urllib.request +from pathlib import Path + +from freetoken.utils.hf import load_tokenizer + + +def main() -> int: + """Render one fixed arithmetic question, stream it once, and retain all evidence.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--gguf", required=True) + parser.add_argument("--artifact", required=True, type=Path) + parser.add_argument("--decode", type=int, default=256) + args = parser.parse_args() + + question = "What is 17 times 19? Reply with only the decimal number." + expected = "323" + tokenizer = load_tokenizer(args.gguf) + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": question}], tokenize=False, add_generation_prompt=True + ) + assert isinstance(prompt, str) + body = { + "model": args.model, + "prompt": prompt, + "max_tokens": args.decode, + "temperature": 0.0, + "top_p": 1.0, + "top_k": -1, + "add_special_tokens": False, + "stream": True, + "stream_options": {"include_usage": True}, + } + req = urllib.request.Request( + args.base_url.rstrip("/") + "/v1/completions", + data=json.dumps(body).encode(), headers={"Content-Type": "application/json"}, + ) + started = time.perf_counter() + stamps: list[float] = [] + chunks: list[str] = [] + usage: dict = {} + with urllib.request.urlopen(req, timeout=300) as response: + for raw in response: + line = raw.decode().strip() + if not line.startswith("data: "): + continue + payload = line[6:] + if payload == "[DONE]": + break + event = json.loads(payload) + usage = event.get("usage") or usage + for choice in event.get("choices", []): + if text := choice.get("text"): + chunks.append(text) + stamps.append(time.perf_counter()) + text = "".join(chunks) + steps = max(len(stamps) - 1, 0) + duration = stamps[-1] - stamps[0] if steps else 0.0 + record = { + "schema_version": 1, + "control": "Gemma4 GGUF caller-rendered raw prompt", + "question": question, + "expected_answer": expected, + "prompt": prompt, + "prompt_sha256": hashlib.sha256(prompt.encode()).hexdigest(), + "prompt_token_count_local": len(tokenizer.encode(prompt, add_special_tokens=False)), + "usage": usage, + "text": text, + "answer_present": expected in text, + "metrics": { + "events": len(stamps), "decode_steps": steps, + "decode_tok_s": steps / duration if duration else 0.0, + "ttft_ms": (stamps[0] - started) * 1000 if stamps else 0.0, + }, + } + args.artifact.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n") + print(json.dumps(record, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/verify_gemma4_gguf_visual_tps.py b/scripts/gmk-evo-x2/verify_gemma4_gguf_visual_tps.py new file mode 100644 index 0000000000..d26f9f4815 --- /dev/null +++ b/scripts/gmk-evo-x2/verify_gemma4_gguf_visual_tps.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Measure a quality-gated long visible Gemma4 image response over OpenAI SSE.""" + +from __future__ import annotations + +import argparse +import base64 +import io +import json +import time +import urllib.request +from pathlib import Path + +from PIL import Image + + +def data_url(image: Image.Image) -> str: + """Encode the deterministic fixture without network access.""" + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode("ascii") + + +def main() -> int: + """Request a constrained visual description and preserve quality and TPS evidence.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--artifact", type=Path, required=True) + parser.add_argument("--max-tokens", type=int, default=256) + args = parser.parse_args() + + image = Image.new("RGB", (96, 48), (0, 0, 255)) + for x in range(48): + for y in range(48): + image.putpixel((x, y), (255, 0, 0)) + prompt = ( + "Describe this image in 45 to 65 words. State the colors, their left-to-right " + "arrangement, and the image shape. Do not use headings, bullet points, or reasoning." + ) + payload = {"model": args.model, "messages": [{"role": "user", "content": [ + {"type": "text", "text": prompt}, {"type": "image_url", "image_url": {"url": data_url(image)}}, + ]}], "temperature": 0, "max_tokens": args.max_tokens, "stream": True, + "stream_options": {"include_usage": True}} + request = urllib.request.Request(args.base_url.rstrip("/") + "/v1/chat/completions", + data=json.dumps(payload).encode(), headers={"Content-Type": "application/json"}) + started = time.perf_counter(); stamps: list[float] = []; content: list[str] = []; reasoning: list[str] = []; usage: dict = {} + with urllib.request.urlopen(request, timeout=300) as response: # nosec B310: local caller URL + for raw in response: + line = raw.decode().strip() + if not line.startswith("data: "): continue + event_data = line[6:] + if event_data == "[DONE]": break + event = json.loads(event_data); usage = event.get("usage") or usage + for choice in event.get("choices", []): + delta = choice.get("delta", {}); text = delta.get("content") or ""; thought = delta.get("reasoning_content") or "" + if text or thought: stamps.append(time.perf_counter()); content.append(text); reasoning.append(thought) + visible = "".join(content).strip(); normalized = visible.lower(); words = visible.split() + duration = stamps[-1] - stamps[0] if len(stamps) > 1 else 0.0 + completion = usage.get("completion_tokens") + record = {"schema_version": 1, "prompt": prompt, "visible": visible, "reasoning": "".join(reasoning), "usage": usage, + "quality": {"word_count": len(words), "has_red": "red" in normalized, "has_blue": "blue" in normalized, + "has_left": "left" in normalized, "has_right": "right" in normalized, + "visible_words_45_to_65": 45 <= len(words) <= 65}, + "metrics": {"events": len(stamps), "ttft_ms": (stamps[0]-started)*1000 if stamps else None, + "stream_window_s": duration, "completion_tokens": completion, + "completion_tok_s": completion/duration if completion and duration else None, + "wall_s": time.perf_counter()-started}} + record["passed"] = all(record["quality"].values()) + args.artifact.parent.mkdir(parents=True, exist_ok=True); args.artifact.write_text(json.dumps(record, indent=2)+"\n") + print(json.dumps(record, indent=2)) + if not record["passed"]: raise SystemExit("visual TPS quality gate failed") + return 0 + + +if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/verify_qwen_aime_quality.py b/scripts/gmk-evo-x2/verify_qwen_aime_quality.py new file mode 100644 index 0000000000..3a8dc7a3cd --- /dev/null +++ b/scripts/gmk-evo-x2/verify_qwen_aime_quality.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Verify GMKtek EVO-X2 Qwen output stability with the historical AIME-25 workload. + +The benchmark uses the same question, greedy sampling, thinking-enabled template, +and forced 128-token decode that exposed the rejected HIP router candidate. It +targets an already-running loopback server and never starts, stops, or changes it. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +import urllib.request +from pathlib import Path +from types import SimpleNamespace + +# Permit the helper to run from any working directory. The benchmark module is +# intentionally kept at the repository root rather than installed into the +# runtime wheel, so add that root before importing it. This keeps the quality +# gate reproducible on GMKtek EVO-X2 without relying on a caller to append `.` to +# PYTHONPATH by hand. +SOURCE_ROOT = Path(__file__).resolve().parents[2] +if str(SOURCE_ROOT) not in sys.path: + sys.path.insert(0, str(SOURCE_ROOT)) + +from benchmarks.bench_decode_moe import load_problem, resolve_sampling, stream_generate + + +# Preserve the original paper-inspired Qwen contract as the default. Other +# qualified source/model combinations have deliberately different output +# fingerprints, so callers must select those contracts explicitly instead of +# silently treating a mismatch as a model-quality failure. +DEFAULT_REFERENCE_SHA1 = "0acef4eab6f4" +SHA1_PREFIX_RE = re.compile(r"^[0-9a-f]{12}$") + + +def parse_args() -> argparse.Namespace: + """Read explicit server, checkpoint, and artifact inputs for one quality gate.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:1919") + parser.add_argument("--model", required=True) + parser.add_argument("--artifact", required=True, type=Path) + parser.add_argument("--aime", default=None) + parser.add_argument("--problem", type=int, default=0) + parser.add_argument("--decode", type=int, default=128) + parser.add_argument( + "--expected-sha1", + default=DEFAULT_REFERENCE_SHA1, + help=( + "12-character lowercase SHA1 prefix for the selected quality " + "contract; defaults to the historical paper-inspired Qwen gate" + ), + ) + return parser.parse_args() + + +def main() -> int: + """Warm the live server, score one deterministic stream, and persist raw evidence.""" + + args = parse_args() + if not SHA1_PREFIX_RE.fullmatch(args.expected_sha1): + raise SystemExit("--expected-sha1 must be exactly 12 lowercase hexadecimal characters") + problem, answer = load_problem(args.aime, args.problem) + sampling, sampling_source = resolve_sampling(args.model, greedy=True) + with urllib.request.urlopen(args.base_url.rstrip("/") + "/v1/models", timeout=10) as response: + model_id = json.load(response)["data"][0]["id"] + stream_args = SimpleNamespace(decode=args.decode) + # Send one complete request first so the measured request observes a populated + # expert cache instead of one-time load and scheduling work. + stream_generate(args.base_url, model_id, problem, sampling, stream_args) + # Capture the quality-gated request itself. stream_generate records a + # monotonic timestamp for every non-empty streamed text event. + result = stream_generate(args.base_url, model_id, problem, sampling, stream_args) + text = result["text"] + output_sha1 = hashlib.sha1(text.encode("utf-8")).hexdigest()[:12] + # The first event includes warm prompt processing. The intervals after it + # describe steady-state decode, which makes this directly comparable to the + # historical AIME benchmark and avoids reporting prompt work as token rate. + stamps = result["stamps"] + completion_tokens = result["usage"]["completion_tokens"] + decode_steps = max(completion_tokens - 1, 0) + decode_seconds = stamps[-1] - stamps[0] if len(stamps) >= 2 else 0.0 + gaps_ms = sorted((later - earlier) * 1e3 for earlier, later in zip(stamps, stamps[1:])) + metrics = { + "decode_steps": decode_steps, + "decode_seconds": decode_seconds, + "decode_tok_s": decode_steps / decode_seconds if decode_seconds > 0 else 0.0, + "ms_per_token": decode_seconds * 1e3 / decode_steps if decode_steps > 0 else 0.0, + "event_ms_p50": gaps_ms[len(gaps_ms) // 2] if gaps_ms else 0.0, + "event_ms_p99": gaps_ms[min(len(gaps_ms) - 1, int(len(gaps_ms) * 0.99))] if gaps_ms else 0.0, + "ttft_ms": (stamps[0] - result["t0"]) * 1e3 if stamps else 0.0, + "events": len(stamps), + } + artifact = { + "schema_version": 1, + "model_id": model_id, + "problem": args.problem, + "expected_answer": answer, + "sampling": sampling, + "sampling_source": sampling_source, + "prompt_tokens": result["usage"]["prompt_tokens"], + "completion_tokens": completion_tokens, + "metrics": metrics, + "expected_output_sha1": args.expected_sha1, + "output_sha1": output_sha1, + "status": "passed" if output_sha1 == args.expected_sha1 else "failed", + "text": text, + } + args.artifact.parent.mkdir(parents=True, exist_ok=True) + args.artifact.write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(artifact, indent=2, sort_keys=True)) + return 0 if artifact["status"] == "passed" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/verify_qwen_raw_prompt_quality.py b/scripts/gmk-evo-x2/verify_qwen_raw_prompt_quality.py new file mode 100644 index 0000000000..4150cf8d1b --- /dev/null +++ b/scripts/gmk-evo-x2/verify_qwen_raw_prompt_quality.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Capture a Qwen quality stream with one caller-rendered prompt. + +This GMKtek EVO-X2 control intentionally avoids ``/v1/chat/completions``. Different +servers can legitimately ship different Jinja renderers for the same GGUF, which +makes chat-token counts and output text incomparable even when their model +execution is correct. The script renders the request once with an explicit +Hugging Face tokenizer, sends that exact string to ``/v1/completions``, and +preserves the prompt, prompt hash, server usage, timings, and emitted text. + +Run it once against each isolated server. Equal ``prompt_sha256`` values are a +hard precondition for comparing output text or decode timing between those runs. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import time +import urllib.request +from pathlib import Path + +from transformers import AutoTokenizer + +# Keep the AIME question loader shared with the existing chat quality gate so +# this raw-prompt control changes only prompt transport, not the math workload. +SOURCE_ROOT = Path(__file__).resolve().parents[2] +if str(SOURCE_ROOT) not in sys.path: + sys.path.insert(0, str(SOURCE_ROOT)) + +from benchmarks.bench_decode_moe import load_problem + + +def parse_args() -> argparse.Namespace: + """Read every external input explicitly so the recorded artifact is reproducible.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", required=True, help="Server origin without /v1.") + parser.add_argument("--model", required=True, help="OpenAI model identifier sent to the server.") + parser.add_argument("--tokenizer", required=True, help="Local HF tokenizer directory used once for rendering.") + parser.add_argument("--artifact", required=True, type=Path, help="New JSON evidence path.") + parser.add_argument("--aime", default=None, help="Optional local AIME JSONL source.") + parser.add_argument("--problem", default=0, type=int, help="Zero-based AIME problem index.") + parser.add_argument("--decode", default=128, type=int, help="Maximum generated tokens.") + return parser.parse_args() + + +def render_prompt(tokenizer, problem: str) -> str: + """Render one thinking-enabled user message into the sole server input string.""" + prompt = tokenizer.apply_chat_template( + [{"role": "user", "content": problem}], + tokenize=False, + add_generation_prompt=True, + enable_thinking=True, + ) + if not isinstance(prompt, str): + raise TypeError("Qwen tokenizer returned a non-string chat prompt") + return prompt + + +def stream_completion(base_url: str, model: str, prompt: str, decode: int) -> tuple[str, dict, list[float], float]: + """Send one greedy raw completion and retain every client-visible text timestamp.""" + body = { + "model": model, + "prompt": prompt, + "max_tokens": decode, + "temperature": 0.0, + "top_p": 1.0, + "top_k": -1, + # The prompt came from apply_chat_template and already includes Qwen's + # assistant and thinking markers. Keeping it literal makes this a + # token-for-token control against llama.cpp's raw completion endpoint. + "add_special_tokens": False, + "stream": True, + "stream_options": {"include_usage": True}, + } + request = urllib.request.Request( + base_url.rstrip("/") + "/v1/completions", + data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json"}, + ) + started = time.perf_counter() + stamps: list[float] = [] + pieces: list[str] = [] + usage: dict = {} + with urllib.request.urlopen(request, timeout=300) as response: + for raw_line in response: + line = raw_line.decode("utf-8").strip() + if not line.startswith("data: "): + continue + payload = line[6:] + if payload == "[DONE]": + break + event = json.loads(payload) + if event.get("usage"): + usage = event["usage"] + for choice in event.get("choices", []): + text = choice.get("text") + if text: + stamps.append(time.perf_counter()) + pieces.append(text) + return "".join(pieces), usage, stamps, started + + +def main() -> int: + """Render, stream, calculate client timing, and write one self-contained artifact.""" + args = parse_args() + problem, answer = load_problem(args.aime, args.problem) + tokenizer = AutoTokenizer.from_pretrained(args.tokenizer, trust_remote_code=True) + prompt = render_prompt(tokenizer, problem) + text, usage, stamps, started = stream_completion(args.base_url, args.model, prompt, args.decode) + decode_steps = max(len(stamps) - 1, 0) + decode_seconds = stamps[-1] - stamps[0] if decode_steps else 0.0 + artifact = { + "schema_version": 1, + "control": "caller-rendered raw prompt via /v1/completions", + "base_url": args.base_url, + "model": args.model, + "tokenizer": args.tokenizer, + "problem": args.problem, + "expected_answer": answer, + "prompt": prompt, + "prompt_sha256": hashlib.sha256(prompt.encode("utf-8")).hexdigest(), + "prompt_token_count_local": len(tokenizer.encode(prompt, add_special_tokens=False)), + "usage": usage, + "output_sha1": hashlib.sha1(text.encode("utf-8")).hexdigest()[:12], + "text": text, + "metrics": { + "events": len(stamps), + "decode_steps": decode_steps, + "decode_seconds": decode_seconds, + "decode_tok_s": decode_steps / decode_seconds if decode_seconds else 0.0, + "ttft_ms": (stamps[0] - started) * 1e3 if stamps else 0.0, + }, + } + args.artifact.parent.mkdir(parents=True, exist_ok=True) + args.artifact.write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(artifact, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gmk-evo-x2/verify_rocm_kernel_cache.py b/scripts/gmk-evo-x2/verify_rocm_kernel_cache.py new file mode 100644 index 0000000000..5be99e72f6 --- /dev/null +++ b/scripts/gmk-evo-x2/verify_rocm_kernel_cache.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Prove that a GMKtek EVO-X2 FreeToken C++ and HIP cache resolves without JIT. + +The cache builder records successful compilation, but a file count alone cannot +prove that every shared object is loadable by the current Python, TVM FFI, ROCm +and FreeToken combination. This verifier sets the same strict environment used +by the isolated Qwen launcher, then asks every explicit AOT specification to +load itself. A missing object or ABI mismatch fails immediately because runtime +compilation remains disabled throughout the check. + +This utility never starts a server, loads a model checkpoint, mutates a cache, +or contacts any non-GMKtek EVO-X2 endpoint. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + """Read the exact read-only cache directory to validate.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cache-dir", required=True, type=Path) + return parser.parse_args() + + +def main() -> int: + """Load every catalog module under strict no-JIT rules and emit metadata.""" + + args = parse_args() + cache_dir = args.cache_dir.resolve() + if not cache_dir.is_dir(): + raise SystemExit(f"cache directory does not exist: {cache_dir}") + + # Set these before importing the FreeToken builders because each spec calls + # load_jit or load_aot internally. The loader first checks this directory + # and raises if an exact shared object is absent. + os.environ["FREETOKEN_KERNEL_CACHE_DIR"] = str(cache_dir) + os.environ["FREETOKEN_DISABLE_JIT"] = "1" + + import torch + + from freetoken.kernel.aot import default_kernel_specs + + if torch.version.hip is None: + raise SystemExit("expected a HIP-backed PyTorch runtime") + + specs = default_kernel_specs() + loaded_names: list[str] = [] + # build() returns immediately from the prebuilt cache path. The otherwise + # required build directory is never created because strict no-JIT makes a + # cache miss an exception before tvm_ffi receives a compile request. + for spec in specs: + spec.build(cache_dir / "verification-build-never-used" / spec.name) + loaded_names.append(spec.name) + + print( + json.dumps( + { + "cache_dir": str(cache_dir), + "device": torch.cuda.get_device_name(), + "hip": torch.version.hip, + "loaded_modules": len(loaded_names), + "status": "passed", + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/publish-wheels.sh b/scripts/publish-wheels.sh index e8a6ac41bf..600767aae7 100755 --- a/scripts/publish-wheels.sh +++ b/scripts/publish-wheels.sh @@ -12,6 +12,12 @@ # install error and a retry) is the safer failure. Requires `gh` authenticated # with write access to the target repo. # +# After the upload, writes `engine-.json` to the release: the pair's URLs, +# sha256 and sizes under a fixed asset name, so a Desktop resolves the pair with one +# static download (no api.github.com, no per-IP rate limit) and scans the asset list +# only when the manifest is missing. One file per platform -- the linux nightly and a +# hand-run windows publish never touch each other's manifest. +# # Environment: # FREETOKEN_WEB_REPO target repo (default: FlashML-org/FreeToken-Web) # FREETOKEN_WEB_TAG release tag (default: beta) @@ -90,6 +96,52 @@ for w in "${wheels[@]}"; do gh release upload "$TAG" "$w" -R "$REPO" done +# The manifest is written LAST so it never names a wheel that is not there yet. The +# Desktop compares asset basenames, so the URL is spelled the way GitHub's +# browser_download_url spells it: `+` percent-encoded. +asset_url() { printf 'https://github.com/%s/releases/download/%s/%s' "$REPO" "$TAG" "${1//+/%2B}"; } +wheel_json() { + local w="$1" name size sha + name="${w##*/}" + size="$(wc -c <"$w" | tr -d ' ')" + sha="$(sha256sum "$w" | cut -d' ' -f1)" + printf '{"name": "%s", "url": "%s", "sha256": "%s", "size": %s}' "$name" "$(asset_url "$name")" "$sha" "$size" +} +manifest_dir="$(mktemp -d)" +trap 'rm -rf "$manifest_dir"' EXIT +while IFS= read -r p; do + rt=""; kc="" + for w in "${wheels[@]}"; do + case "${w##*/}" in + freetoken-*"$p"*.whl) rt="$w" ;; + freetoken_kernel_cache-*"$p"*.whl) kc="$w" ;; + esac + done + rt_name="${rt##*/}" + # freetoken----.whl + version="$(cut -d- -f2 <<<"$rt_name")" + python_tag="$(cut -d- -f3 <<<"$rt_name")" + commit="$(grep -oE '\+g[0-9a-f]{7,}' <<<"$rt_name" | head -1 | sed 's/^+g//' || true)" + cuda="$(grep -oE '\+cu[0-9]+' <<<"${kc##*/}" | head -1 | sed 's/^+//' || true)" + manifest="$manifest_dir/engine-$p.json" + cat >"$manifest" <&2 + usage >&2 + exit 2 + ;; + esac +done + +if [[ -z "${SOURCE_DIR}" || -z "${ARTIFACT_DIR}" ]]; then + printf 'error: --source-dir and --artifact-dir are required\n' >&2 + usage >&2 + exit 2 +fi +if ! git -C "${SOURCE_DIR}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + printf 'error: source directory is not a Git checkout: %s\n' "${SOURCE_DIR}" >&2 + exit 2 +fi +if [[ -e "${ARTIFACT_DIR}" ]]; then + printf 'error: artifact directory already exists: %s\n' "${ARTIFACT_DIR}" >&2 + exit 3 +fi +if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then + printf 'error: Python executable was not found: %s\n' "${PYTHON_BIN}" >&2 + exit 4 +fi + +GPU_PROBE="$(${PYTHON_BIN} - "${EXPECTED_GFX}" <<'PY' +import json +import sys + +expected = sys.argv[1].lower() +try: + import torch +except Exception as error: + raise SystemExit(f"PyTorch import failed: {error!r}") + +if not torch.version.hip: + raise SystemExit("PyTorch does not report a HIP runtime") +if not torch.cuda.is_available(): + raise SystemExit("PyTorch HIP device is unavailable") + +properties = torch.cuda.get_device_properties(0) +architecture = getattr(properties, "gcnArchName", "") +if expected and architecture.lower() != expected: + raise SystemExit(f"GPU architecture {architecture!r} does not match required {expected!r}") + +print(json.dumps({ + "torch_version": torch.__version__, + "hip_version": torch.version.hip, + "triton_version": __import__("triton").__version__, + "device_name": torch.cuda.get_device_name(0), + "gcn_architecture": architecture, +}, sort_keys=True)) +PY +)" + +mkdir -p "${ARTIFACT_DIR}" +trap 'printf "error: manifest capture failed; incomplete artifact retained at %s\\n" "${ARTIFACT_DIR}" >&2' ERR + +if [[ "${INCLUDE_HOSTNAME}" -eq 1 ]]; then + PUBLIC_HOSTNAME="$(hostname -s)" +else + PUBLIC_HOSTNAME="redacted" +fi + +{ + printf 'captured_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + printf 'hostname=%s\n' "${PUBLIC_HOSTNAME}" + # Do not use the all-fields uname form: its second field is the local host + # name and would defeat the redacted default. These explicit fields preserve + # the useful operating-system, kernel, and architecture facts without + # identifying the machine that produced a public reproducibility bundle. + printf 'kernel_system=%s\n' "$(uname -s)" + printf 'kernel_release=%s\n' "$(uname -r)" + printf 'kernel_version=%s\n' "$(uname -v)" + printf 'machine_architecture=%s\n' "$(uname -m)" + test -r /etc/os-release && cat /etc/os-release + command -v lscpu >/dev/null 2>&1 && lscpu || true +} >"${ARTIFACT_DIR}/system.txt" + +{ + # Record the immutable source revision because reviewers need exact code provenance. + printf 'commit=%s +' "$(git -C "${SOURCE_DIR}" rev-parse HEAD)" + # Record the branch label because it identifies the tested line without exposing local paths. + printf 'branch=%s +' "$(git -C "${SOURCE_DIR}" branch --show-current || true)" + # Capture porcelain once because only aggregate counts, never private filenames, belong in public evidence. + SOURCE_STATUS="$(git -C "${SOURCE_DIR}" status --porcelain=v1)" + # Report whether tracked content differs so a nominal commit is not mistaken for a clean checkout. + printf 'tracked_changes=%s +' "$(printf '%s +' "${SOURCE_STATUS}" | awk 'NF && substr($0, 1, 2) != "??" { count++ } END { print count + 0 }')" + # Report the untracked count without publishing names that may contain private model or artifact details. + printf 'untracked_entries=%s +' "$(printf '%s +' "${SOURCE_STATUS}" | awk 'substr($0, 1, 2) == "??" { count++ } END { print count + 0 }')" + # Report only whether an origin exists because a raw remote URL can embed credentials or private hosts. + if git -C "${SOURCE_DIR}" remote get-url origin >/dev/null 2>&1; then + # Emit a boolean marker so provenance tooling knows the checkout has an upstream without learning its URL. + printf 'origin_configured=yes +' + else + # Emit the negative marker so missing-upstream evidence is explicit rather than silently omitted. + printf 'origin_configured=no +' + fi +} >"${ARTIFACT_DIR}/source-state.txt" + +printf '%s\n' "${GPU_PROBE}" >"${ARTIFACT_DIR}/python-hip.json" + +{ + command -v rocminfo || true + rocminfo 2>&1 || true +} >"${ARTIFACT_DIR}/rocminfo.txt" + +{ + command -v rocm-smi || true + rocm-smi --showproductname --showtemp --showperflevel --showmeminfo vram 2>&1 || true +} >"${ARTIFACT_DIR}/rocm-smi.txt" + +{ + free -b 2>&1 || true + swapon --show --bytes 2>&1 || true + vmstat 1 3 2>&1 || true + df -B1 / "${SOURCE_DIR}" 2>&1 || true + lsblk -d -o NAME,MODEL,SIZE,TRAN,ROTA 2>&1 || true +} >"${ARTIFACT_DIR}/memory-and-storage.txt" + +"${PYTHON_BIN}" - "${ARTIFACT_DIR}/manifest.json" "${PUBLIC_HOSTNAME}" "${EXPECTED_GFX}" "${GPU_PROBE}" <<'PY' +import json +import sys +from pathlib import Path + +output_path = Path(sys.argv[1]) +hostname = sys.argv[2] +expected_gfx = sys.argv[3] or None +gpu = json.loads(sys.argv[4]) +manifest = { + "schema_version": 1, + "host": hostname, + "expected_gfx": expected_gfx, + "native_hip": gpu, + "files": [ + "system.txt", + "source-state.txt", + "python-hip.json", + "rocminfo.txt", + "rocm-smi.txt", + "memory-and-storage.txt", + "manifest.json", + "SHA256SUMS", + ], + "collection": "read-only; no service, cache, clock, memory, or swap mutation", + "privacy": "hostname is redacted by default; no shell environment, process list, serial number, or network address is collected", +} +output_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") +PY + +( + cd "${ARTIFACT_DIR}" + sha256sum system.txt source-state.txt python-hip.json rocminfo.txt rocm-smi.txt \ + memory-and-storage.txt manifest.json >SHA256SUMS +) + +trap - ERR +printf '%s\n' "${ARTIFACT_DIR}" diff --git a/setup.py b/setup.py index cfe41b7d83..1f0d906317 100644 --- a/setup.py +++ b/setup.py @@ -3,14 +3,27 @@ import importlib.util from pathlib import Path +import sys + from setuptools import setup -from torch.utils.cpp_extension import BuildExtension, CUDA_HOME, CppExtension +import torch +from torch.utils.cpp_extension import BuildExtension, CUDA_HOME, ROCM_HOME, CppExtension ROOT = Path(__file__).parent +# The active PyTorch build, rather than toolkit discovery, defines the extension +# ABI. A developer can have a CUDA toolkit installed while building a HIP +# PyTorch environment; requiring CUDA_HOME to be absent would then link host +# extensions against cudart even though the process uses libamdhip64. +IS_ROCM = torch.version.hip is not None +GPU_RUNTIME_MACROS = [("FREETOKEN_USE_ROCM", "1")] if IS_ROCM else [] def _check_toolchain() -> None: + if IS_ROCM: + # nvcc/CUDA-major checks below are meaningless on a ROCm torch build + # (torch.version.cuda is None there), so _toolchain.py's check is a no-op. + return path = ROOT / "python" / "freetoken" / "kernel" / "_toolchain.py" spec = importlib.util.spec_from_file_location("_freetoken_toolchain", path) module = importlib.util.module_from_spec(spec) @@ -18,20 +31,39 @@ def _check_toolchain() -> None: module.check_nvcc_matches_torch() -def _cuda_runtime_paths() -> tuple[list[str], list[str]]: +def _gpu_runtime_paths() -> tuple[list[str], list[str], list[str], list[str]]: + """Returns (include_dirs, library_dirs, libraries, extra_link_args).""" + if IS_ROCM: + rocm_home = Path(ROCM_HOME) + library_dirs = [d for d in (rocm_home / "lib64", rocm_home / "lib") if d.exists()] + # The pip-vendored rocm-sdk-core ships versioned sonames (libamdhip64.so.7) + # without the bare .so dev symlink `-lamdhip64` needs, so link the exact + # file. At runtime the dynamic linker dedupes on SONAME, so this resolves + # to whichever libamdhip64 torch itself already loaded into the process. + hip_lib = next( + (f for d in library_dirs for f in sorted(d.glob("libamdhip64.so*"))), None + ) + if hip_lib is None: + raise RuntimeError(f"libamdhip64.so* not found under {library_dirs}") + return ( + [str(rocm_home / "include")], + [str(d) for d in library_dirs], + [], + [f"-l:{hip_lib.name}"], + ) if CUDA_HOME is None: raise RuntimeError( - "CUDA_HOME is required to build freetoken.kernel._pinned_tensor " - "because it links against the CUDA runtime API." + "CUDA_HOME (or ROCM_HOME) is required to build freetoken.kernel._pinned_tensor " + "because it links against the CUDA/HIP runtime API." ) cuda_home = Path(CUDA_HOME) library_dirs = [str(cuda_home / "lib64")] if (cuda_home / "lib").exists(): library_dirs.append(str(cuda_home / "lib")) - return [str(cuda_home / "include")], library_dirs + return [str(cuda_home / "include")], library_dirs, ["cudart"], [] -cuda_include_dirs, cuda_library_dirs = _cuda_runtime_paths() +cuda_include_dirs, cuda_library_dirs, cuda_libraries, cuda_extra_link_args = _gpu_runtime_paths() _check_toolchain() @@ -44,8 +76,10 @@ def _cuda_runtime_paths() -> tuple[list[str], list[str]]: ], include_dirs=cuda_include_dirs, library_dirs=cuda_library_dirs, - libraries=["cudart"], + libraries=cuda_libraries, + extra_link_args=cuda_extra_link_args, extra_compile_args=["-O3", "-std=c++17"], + define_macros=GPU_RUNTIME_MACROS, ), # CPU-compute MoE executor for --moe-backend cpu. Links cudart for the # cudaLaunchHostFunc submit/sync graph nodes; the bf16 GEMV microkernels @@ -59,9 +93,21 @@ def _cuda_runtime_paths() -> tuple[list[str], list[str]]: ], include_dirs=cuda_include_dirs, library_dirs=cuda_library_dirs, - libraries=["cudart"], + libraries=cuda_libraries, + extra_link_args=cuda_extra_link_args, extra_compile_args=["-O3", "-std=c++17", "-pthread"], + define_macros=GPU_RUNTIME_MACROS, ), + # --ple-backend disk row store; Linux-only until the TableFile/BatchReader seams grow Windows bodies + *([ + CppExtension( + name="freetoken.kernel._ple_store", + sources=[ + "python/freetoken/kernel/csrc/ple_store/ple_store_ext.cpp", + ], + extra_compile_args=["-O3", "-std=c++17"], + ) + ] if sys.platform == "linux" else []), ], cmdclass={"build_ext": BuildExtension.with_options(use_ninja=True)}, ) diff --git a/tests/attention/test_dsa_kpool.py b/tests/attention/test_dsa_kpool.py new file mode 100644 index 0000000000..fb856e1757 --- /dev/null +++ b/tests/attention/test_dsa_kpool.py @@ -0,0 +1,369 @@ +"""Glm5NextDSABackend (kpool indexer) vs an eager reference. + +Exercises the full path -- raw K/gate stores, pool-completion compression, +pool-granular scoring, top-k + expansion + tail, gathered sparse MLA -- on a +single request at toy dims (Hi=16 index heads, Di=64, kpool=4, topk=32): + +* kv_len <= index_topk: the identity/dense path must equal full softmax MLA. +* kv_len > index_topk: prefill queries must match a subset-softmax reference + built from an independently computed pooled-score top-k (+ tail). +* decode: pooled entries appear exactly at pool-completion steps and match the + softmax(gate+APE) reference; decode outputs match the same subset reference. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + +H, LATENT = 2, 64 # MLA heads, kv_lora_rank (== latent width: NoPE has no kpe half) +HI, DI = 16, 64 # index heads (kernel needs >= 16), index head dim (pow2) +KPOOL, TOPK = 4, 32 +SM_SCALE = 0.125 +DEV = "cuda" + + +def _args(num_layers=1): + from freetoken.models.glm5_next.args import Glm5NextArgs + + return Glm5NextArgs( + hidden_size=32, num_heads=H, + q_lora_rank=16, kv_lora_rank=LATENT, qk_nope_head_dim=LATENT, + qk_rope_head_dim=0, v_head_dim=LATENT, mla_nope=True, norm_eps=1e-5, + max_position=4096, + index_n_heads=HI, index_head_dim=DI, index_topk=TOPK, + indexer_types=("full",) * num_layers, indexer_rope_interleave=True, + index_kpool=KPOOL, index_kpool_compress=True, + index_kpool_always_select_tail=True, + linear_num_heads=0, linear_head_dim=0, linear_conv_kernel_dim=4, + linear_lower_bound=-5.0, + layer_types=("deepseek_sparse_attention",) * num_layers, + mlp_layer_types=("dense",) * num_layers, + mhc=False, mhc_num_residual_streams=1, hc_eps=1e-6, + mhc_sinkhorn_iterations=0, mhc_tau=0.05, mhc_post_mult_value=2.0, + mhc_no_norm_weight=False, swiglu_limit=None, rope_theta=10000.0, + ) + + +@pytest.fixture() +def harness(monkeypatch): + from freetoken.attention.dsa_indexer_kpool import Glm5NextDSABackend + from freetoken.kvcache.dsa_pool import KpoolDSAKVCache + + pool = KpoolDSAKVCache( + latent_dim=LATENT, num_layers=1, num_pages=8, page_size=64, + dtype=torch.bfloat16, device=torch.device(DEV), + index_head_dim=DI, num_index_layers=1, + index_ratio=KPOOL, num_req_slots=4, + ) + page_table = torch.full((4, 512), -1, dtype=torch.int32, device=DEV) + page_table[0, :512] = torch.arange(512, dtype=torch.int32, device=DEV) + ctx = SimpleNamespace(kv_cache=pool, page_table=page_table) + monkeypatch.setattr("freetoken.attention.dsa.get_global_ctx", lambda: ctx) + + config = SimpleNamespace( + glm5_args=_args(), glm_dsa_args=None, num_qo_heads=H, + attn_sm_scale=SM_SCALE, num_layers=1, + ) + backend = Glm5NextDSABackend(config) + torch.manual_seed(0) + # The APE is a MODEL parameter, passed per call via DSAIndexerInputs. + ape = torch.randn(KPOOL, DI, device=DEV, dtype=torch.float32) * 0.3 + return backend, pool, ape + + +def _req(device_len, cached_len=0): + return SimpleNamespace( + table_idx=0, device_len=device_len, extend_len=device_len - cached_len, + cached_len=cached_len, linear_slot_idx=None, + ) + + +def _prefill_batch(t0, t1): + return SimpleNamespace( + phase="prefill", padded_reqs=[_req(t1, t0)], reqs=[_req(t1, t0)], + positions=torch.arange(t0, t1, device=DEV), + out_loc=torch.arange(t0, t1, device=DEV), + active_table_idx=None, + ) + + +def _decode_batch(pos): + return SimpleNamespace( + phase="decode", padded_reqs=[_req(pos + 1, pos)], reqs=[_req(pos + 1, pos)], + positions=torch.tensor([pos], device=DEV), + out_loc=torch.tensor([pos], device=DEV), + active_table_idx=torch.tensor([0], device=DEV), + ) + + +def _rand_seq(total, seed=1): + torch.manual_seed(seed) + mk = lambda *s: torch.randn(*s, device=DEV, dtype=torch.bfloat16) + return dict( + q_nope=mk(total, H, LATENT), c_kv=mk(total, LATENT), + qi=mk(total, HI, DI), ki=mk(total, DI), + wi=(torch.randn(total, HI, device=DEV).float() * 0.5), + gate=mk(total, DI), + ) + + +def _run(backend, batch, d, sl, ape): + from freetoken.attention.dsa import DSAIndexerInputs + + t = batch.positions.shape[0] + backend.prepare_metadata(batch) + return backend.mla_forward( + d["q_nope"][sl], d["q_nope"].new_empty(t, H, 0), + d["c_kv"][sl], d["c_kv"].new_empty(t, 0), + 0, batch, + indexer_inputs=DSAIndexerInputs( + q=d["qi"][sl], k=d["ki"][sl], w=d["wi"][sl], + gate=d["gate"][sl], ape=ape, + ), + ) + + +# ---- eager references ----------------------------------------------------------------- + + +def _ref_pooled(d, ape, n_pools): + """[n_pools, DI] softmax(gate+ape)-weighted pooled keys (fp32 -> bf16).""" + k = d["ki"][: n_pools * KPOOL].view(n_pools, KPOOL, DI).float() + g = d["gate"][: n_pools * KPOOL].view(n_pools, KPOOL, DI).float() + w = torch.softmax(g + ape, dim=1) + return (w * k).sum(1).to(torch.bfloat16) + + +def _ref_scores(d, ape, q_idx_t, w_t, n_pools): + """Pool scores for one query: sum_h w_h * relu(q_h . k_pool) * DI**-0.5.""" + kp = _ref_pooled(d, ape, n_pools).float() + s = torch.relu(q_idx_t.float() @ kp.T) # [HI, n_pools] + return ((w_t * DI**-0.5).unsqueeze(1) * s).sum(0) + + +def _ref_attend(d, q_t, positions): + """Full softmax MLA over latent rows at ``positions`` for one query [H, LATENT].""" + lat = d["c_kv"][positions].float() # [n, LATENT] + logits = q_t.float() @ lat.T * SM_SCALE # [H, n] + p = torch.softmax(logits, dim=-1) + return (p @ lat).to(torch.bfloat16) + + +def _ref_selected_positions(d, ape, q_idx_t, w_t, pos): + """Reference kpool selection for a query at ``pos``: top-k complete pools + expanded to tokens, plus the tail [n_pools*KPOOL, pos].""" + n_pools = (pos + 1) // KPOOL + sel_pools = min(TOPK // KPOOL, n_pools) + picked = torch.topk( + _ref_scores(d, ape, q_idx_t, w_t, n_pools), sel_pools + ).indices.tolist() + positions = [p * KPOOL + o for p in picked for o in range(KPOOL)] + positions += list(range(n_pools * KPOOL, pos + 1)) + return sorted(set(positions)) + + +def test_dense_path_matches_full_softmax(harness): + backend, pool, ape = harness + total = 20 # < TOPK -> identity/dense path + d = _rand_seq(total) + out = _run(backend, _prefill_batch(0, total), d, slice(0, total), ape) + for t in range(total): + ref = _ref_attend(d, d["q_nope"][t], list(range(t + 1))) + err = (out[t].float() - ref.float()).abs().max().item() + assert err < 2e-2, f"dense query {t}: err {err}" + + +def test_sparse_prefill_matches_reference(harness): + backend, pool, ape = harness + total = 60 # > TOPK -> kpool scoring + d = _rand_seq(total, seed=2) + out = _run(backend, _prefill_batch(0, total), d, slice(0, total), ape) + + # Pooled entries in the slab match the compression reference. + n_pools = total // KPOOL + # Shadow slab: pool p lives at token_slot // KPOOL == p (identity page table). + slab = pool.index_k_cache(0)[torch.arange(n_pools, device=DEV)] + ref_pool = _ref_pooled(d, ape, n_pools) + assert (slab.float() - ref_pool.float()).abs().max().item() < 2e-2 + + for t in (35, 47, 59): # queries past TOPK (sparse regime) + sel = _ref_selected_positions(d, ape, d["qi"][t], d["wi"][t], t) + ref = _ref_attend(d, d["q_nope"][t], sel) + err = (out[t].float() - ref.float()).abs().max().item() + assert err < 3e-2, f"sparse query {t}: err {err}" + + +def test_decode_completion_and_selection(harness): + backend, pool, ape = harness + total, extra = 60, 6 # decode positions 60..65; completion at 63 + d = _rand_seq(total + extra, seed=3) + _run(backend, _prefill_batch(0, total), d, slice(0, total), ape) + + for pos in range(total, total + extra): + out = _run(backend, _decode_batch(pos), d, slice(pos, pos + 1), ape) + sel = _ref_selected_positions(d, ape, d["qi"][pos], d["wi"][pos], pos) + ref = _ref_attend(d, d["q_nope"][pos], sel) + err = (out[0].float() - ref.float()).abs().max().item() + assert err < 3e-2, f"decode pos {pos}: err {err}" + + if pos % KPOOL == KPOOL - 1: # a pool completed this step + n_pools = (pos + 1) // KPOOL + row = pos // KPOOL # the pool's shadow row + got = pool.index_k_cache(0)[row] + want = _ref_pooled(d, ape, n_pools)[-1] + assert (got.float() - want.float()).abs().max().item() < 2e-2 + + +def test_sparse_batch_with_sub_pool_request(harness): + """A sparse prefill batch may carry a request shorter than one pool: it has nothing + to score and must come out as the dense (tail-only) attention over its own rows.""" + from freetoken.attention import dsa as dsa_mod + + backend, pool, ape = harness + dsa_mod.get_global_ctx().page_table[1, :64] = torch.arange( + 448, 512, dtype=torch.int32, device=DEV + ) + ta = 60 # > TOPK -> the batch takes the sparse path + for tb in (1, 2, 3): + d = _rand_seq(ta + tb, seed=6) + req_b = _req(tb) + req_b.table_idx = 1 + reqs = [_req(ta), req_b] + batch = SimpleNamespace( + phase="prefill", padded_reqs=reqs, reqs=reqs, + positions=torch.cat([torch.arange(ta), torch.arange(tb)]).to(DEV), + out_loc=torch.cat([torch.arange(ta), torch.arange(448, 448 + tb)]).to(DEV), + active_table_idx=None, + ) + out = _run(backend, batch, d, slice(0, ta + tb), ape) + for j in range(tb): + t = ta + j + ref = _ref_attend(d, d["q_nope"][t], list(range(ta, t + 1))) + err = (out[t].float() - ref.float()).abs().max().item() + assert err < 3e-2, f"sub-pool request len {tb}, query {j}: err {err}" + t = ta - 1 + sel = _ref_selected_positions(d, ape, d["qi"][t], d["wi"][t], t) + ref = _ref_attend(d, d["q_nope"][t], sel) + assert (out[t].float() - ref.float()).abs().max().item() < 3e-2 + + +def test_chunked_prefill_mid_pool_start(harness): + """A chunk may start MID-POOL (main's soft prefill_chunk_align keeps an + unaligned end when the budget cannot fill a page): the straddling pool's + older members come from the tail ring and the pooled slab + outputs must + match the single-shot run.""" + backend, pool, ape = harness + total, split = 60, 30 # split % KPOOL == 2 -> pool 7 straddles the chunks + d = _rand_seq(total, seed=4) + out1 = _run(backend, _prefill_batch(0, split), d, slice(0, split), ape) + out2 = _run(backend, _prefill_batch(split, total), d, slice(split, total), ape) + + n_pools = total // KPOOL + slab = pool.index_k_cache(0)[torch.arange(n_pools, device=DEV)] + ref_pool = _ref_pooled(d, ape, n_pools) + assert (slab.float() - ref_pool.float()).abs().max().item() < 2e-2 + + for t in (35, 47, 59): + sel = _ref_selected_positions(d, ape, d["qi"][t], d["wi"][t], t) + ref = _ref_attend(d, d["q_nope"][t], sel) + err = (out2[t - split].float() - ref.float()).abs().max().item() + assert err < 3e-2, f"mid-pool chunked query {t}: err {err}" + + +def test_interleaved_decode_requests_do_not_pollute_rings(harness): + """Two requests decoding in alternation: tail rings and shadow rows are keyed + by table_idx, so neither request's pools may absorb the other's raw K/gate.""" + from freetoken.attention import dsa as dsa_mod + + backend, pool, ape = harness + dsa_mod.get_global_ctx().page_table[1, :128] = torch.arange( + 256, 384, dtype=torch.int32, device=DEV + ) + total, extra = 60, 6 + streams = { # table_idx -> (data, physical row base) + 0: (_rand_seq(total + extra, seed=7), 0), + 1: (_rand_seq(total + extra, seed=8), 256), + } + + def _batch_for(table, phase, t0, t1): + base = streams[table][1] + req = _req(t1, t0) + req.table_idx = table + return SimpleNamespace( + phase=phase, padded_reqs=[req], reqs=[req], + positions=torch.arange(t0, t1, device=DEV), + out_loc=torch.arange(base + t0, base + t1, device=DEV), + active_table_idx=( + torch.tensor([table], device=DEV) if phase == "decode" else None + ), + ) + + for table in (0, 1): + d = streams[table][0] + _run(backend, _batch_for(table, "prefill", 0, total), d, slice(0, total), ape) + + for pos in range(total, total + extra): + for table in (0, 1): # alternate every step + d, base = streams[table] + out = _run( + backend, _batch_for(table, "decode", pos, pos + 1), d, + slice(pos, pos + 1), ape, + ) + sel = _ref_selected_positions(d, ape, d["qi"][pos], d["wi"][pos], pos) + ref = _ref_attend(d, d["q_nope"][pos], sel) + err = (out[0].float() - ref.float()).abs().max().item() + assert err < 3e-2, f"table {table} decode pos {pos}: err {err}" + + if pos % KPOOL == KPOOL - 1: + n_pools = (pos + 1) // KPOOL + row = (base + pos) // KPOOL + got = pool.index_k_cache(0)[row] + want = _ref_pooled(d, ape, n_pools)[-1] + err = (got.float() - want.float()).abs().max().item() + assert err < 2e-2, f"table {table} pool at pos {pos}: err {err}" + + +def test_padding_and_empty_batch_leave_shadow_rows_clean(): + """The compression kernel writes every row somewhere, but masked-off rows + (padding request == -1, or a pool that cannot close yet) must land only on + their designated scratch rows -- the shadow region and the rings stay clean. + An empty batch is a no-op.""" + from freetoken.kernel.triton.kpool_compress import kpool_compress_store + + shadow_n, n_req = 8, 2 + slab = torch.full((shadow_n + n_req, DI), 7.0, dtype=torch.bfloat16, device=DEV) + ring_k = torch.full((n_req * KPOOL, DI), 3.0, dtype=torch.bfloat16, device=DEV) + ring_g = ring_k.clone() + ape = torch.randn(KPOOL, DI, dtype=torch.float32, device=DEV) + k = torch.randn(2, DI, dtype=torch.bfloat16, device=DEV) + gate = torch.randn(2, DI, dtype=torch.bfloat16, device=DEV) + + kpool_compress_store( + k, gate, ring_k, ring_g, ape, + ring_slots=torch.tensor([0, 1], dtype=torch.int32, device=DEV), + token_to_req=torch.tensor([0, -1], dtype=torch.int32, device=DEV), + cu_seqlens=torch.tensor([0, 1, 1], dtype=torch.int32, device=DEV), + positions=torch.tensor([0, 0], device=DEV), # pos 0: no pool can close + slab=slab, cmp_rows=torch.tensor([8, 9], dtype=torch.int32, device=DEV), + ratio=KPOOL, + ) + assert torch.equal(slab[:shadow_n], torch.full_like(slab[:shadow_n], 7.0)) + assert torch.equal(ring_k, torch.full_like(ring_k, 3.0)) + assert torch.equal(ring_g, ring_k) + + before = slab.clone() + kpool_compress_store( + k[:0], gate[:0], ring_k, ring_g, ape, + ring_slots=torch.tensor([0], dtype=torch.int32, device=DEV), + token_to_req=torch.empty(0, dtype=torch.int32, device=DEV), + cu_seqlens=torch.tensor([0, 0], dtype=torch.int32, device=DEV), + positions=torch.empty(0, dtype=torch.int64, device=DEV), + slab=slab, cmp_rows=torch.empty(0, dtype=torch.int32, device=DEV), + ratio=KPOOL, + ) + assert torch.equal(slab, before) diff --git a/tests/benchmarks/test_gmk_evo_x2_benchmark.py b/tests/benchmarks/test_gmk_evo_x2_benchmark.py new file mode 100644 index 0000000000..8bf8133d10 --- /dev/null +++ b/tests/benchmarks/test_gmk_evo_x2_benchmark.py @@ -0,0 +1,339 @@ +"""Unit tests for the GMKtek EVO-X2 Qwen API benchmark safety primitives.""" + +from __future__ import annotations + +import unittest +import json +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from benchmarks.gmk_evo_x2.run_api_benchmark import ( + client_prefill_tps, + nearest_rank_percentile, + numeric_summary, + parse_args, + require_expected_host, + require_loopback_url, +) +from benchmarks.gmk_evo_x2.run_quality_suite import evaluate_check +from benchmarks.gmk_evo_x2.run_multiturn_state_suite import nearest_rank +from benchmarks.gmk_evo_x2.run_long_context_control import build_prompt +from benchmarks.gmk_evo_x2.run_concurrent_api_control import parse_args as parse_concurrent_args +from benchmarks.gmk_evo_x2.summarize_qwen_gguf_endurance import summarize + + +class RequireExpectedHostTests(unittest.TestCase): + """Exercise the host guard without requiring any third-party test package.""" + + def test_accepts_gmk_evo_x2_short_name(self) -> None: + """The harness accepts the exact GMKtek EVO-X2 host name used by the test policy.""" + + with patch("socket.gethostname", return_value="test-machine-1"): + self.assertEqual(require_expected_host("test-machine-1"), "test-machine-1") + + def test_rejects_other_hosts(self) -> None: + """The harness prevents accidental benchmark traffic to any other LAN machine.""" + + with patch("socket.gethostname", return_value="amd-host"): + with self.assertRaisesRegex(RuntimeError, "refusing benchmark"): + require_expected_host("test-machine-1") + + def test_loopback_url_guard_rejects_remote_targets(self) -> None: + """The benchmark refuses to send private prompts to a non-loopback API by mistake.""" + + # Require a documentation-only remote address to fail before any socket is opened. + with self.assertRaisesRegex(ValueError, "loopback"): + # Exercise the shared policy helper directly so the failure is independent from argparse output. + require_loopback_url("http://192.0.2.10:1919/v1") + + def test_throughput_mode_requires_two_requested_tokens(self) -> None: + """The TPS mode rejects a one-token interval before it can produce nonsense.""" + + with self.assertRaises(SystemExit): + parse_args( + [ + "--model", "qwen", + "--tokenizer", "tokenizer", + "--artifact-dir", "artifacts", + "--mode", "throughput", + "--max-tokens", "1", + ] + ) + + def test_quality_mode_defaults_to_no_reasoning(self) -> None: + """The canary requests final-answer text instead of an unbounded thought stream.""" + + args = parse_args( + [ + "--model", "qwen", + "--tokenizer", "tokenizer", + "--artifact-dir", "artifacts", + "--expected-host", "test-machine", + ] + ) + self.assertEqual(args.reasoning_effort, "none") + + +class TailMetricTests(unittest.TestCase): + """Keep percentile output stable and auditable for later tail studies.""" + + def test_nearest_rank_percentiles_select_observed_values(self) -> None: + """A four-event stream has no fictional interpolated p95 or p99 value.""" + + values = [0.01, 0.02, 0.03, 0.04] + self.assertEqual(nearest_rank_percentile(values, 0.50), 0.02) + self.assertEqual(nearest_rank_percentile(values, 0.95), 0.04) + self.assertEqual(nearest_rank_percentile(values, 0.99), 0.04) + + def test_empty_metric_summary_has_explicit_nulls(self) -> None: + """A one-token answer must not fabricate token-gap tail statistics.""" + + self.assertTrue(all(value is None for value in numeric_summary([]).values())) + + def test_client_prefill_rate_uses_prompt_tokens_and_first_text_time(self) -> None: + """The reported prefill rate has the documented client-visible boundary.""" + + self.assertEqual(client_prefill_tps(120, 0.5), 240.0) + self.assertIsNone(client_prefill_tps(None, 0.5)) + self.assertIsNone(client_prefill_tps(120, None)) + self.assertIsNone(client_prefill_tps(120, 0.0)) + + +class QualitySuiteCheckTests(unittest.TestCase): + """Verify fixture scoring without needing a server or model weights.""" + + def test_exact_check_accepts_only_visible_exact_text(self) -> None: + """Whitespace around an otherwise exact completion is acceptable.""" + + self.assertEqual(evaluate_check(" GMK_EVO_X2\n", {"kind": "exact", "value": "GMK_EVO_X2"}), (True, None)) + self.assertFalse(evaluate_check("GMK_EVO_X2!", {"kind": "exact", "value": "GMK_EVO_X2"})[0]) + + def test_json_fields_check_rejects_nonvisible_or_wrong_structure(self) -> None: + """The gate requires a valid visible JSON object with the requested fields.""" + + self.assertEqual( + evaluate_check('{"status":"ok","value":7}', {"kind": "json_fields", "fields": {"status": "ok", "value": 7}}), + (True, None), + ) + self.assertFalse(evaluate_check("not json", {"kind": "json_fields", "fields": {"status": "ok"}})[0]) + + +class MultiTurnTailMetricTests(unittest.TestCase): + """Keep short-suite tail aggregation tied to recorded rather than invented values.""" + + def test_nearest_rank_uses_the_observed_worst_value_for_p99(self) -> None: + """Three turn values make p99 the actual worst measured turn.""" + + self.assertEqual(nearest_rank([0.1, 0.2, 0.3], 0.99), 0.3) + + +class LongContextControlTests(unittest.TestCase): + """Keep the controlled long prompt deterministic and retrieval-focused.""" + + def test_prompt_starts_with_marker_and_ends_with_exact_instruction(self) -> None: + """The retrieval answer appears only in the protected prefix.""" + + prompt = build_prompt(2) + self.assertTrue(prompt.startswith("Protected marker: azure-17")) + self.assertEqual(prompt.count("azure-17"), 1) + self.assertTrue(prompt.endswith("Reply with only the protected marker and no other text.")) + + def test_prompt_rejects_zero_filler(self) -> None: + """A zero-context request cannot accidentally masquerade as a long test.""" + + with self.assertRaises(ValueError): + build_prompt(0) + + def test_prefix_nonce_precedes_the_long_filler(self) -> None: + """A changing early nonce prevents reuse of the long filler prefix.""" + + prompt = build_prompt(2, prefix_nonce="sample-1") + self.assertIn("Per-sample prefix nonce: sample-1", prompt) + self.assertLess(prompt.index("sample-1"), prompt.index("This is deterministic filler")) + + +class DpmPolicyWrapperTests(unittest.TestCase): + """Protect the policy wrapper's separate telemetry and harness paths.""" + + def test_dpm_wrapper_reserves_a_new_harness_child_directory(self) -> None: + """Policy logs use a parent while the immutable harness receives `benchmark`.""" + + repository_root = Path(__file__).resolve().parents[2] + wrapper = repository_root / "scripts" / "gmk-evo-x2" / "run_qwen_dpm_policy_benchmark.sh" + contents = wrapper.read_text(encoding="utf-8") + + self.assertIn('readonly BENCHMARK_DIR="${ARTIFACT_ROOT}/benchmark"', contents) + self.assertIn('bash "${HARNESS}" "${BENCHMARK_DIR}"', contents) + self.assertNotIn('mkdir -p "${BENCHMARK_DIR}"', contents) + + +class QwenRecoveryContextTests(unittest.TestCase): + """Protect the recovery server's validated long-context cache allocation.""" + + def test_recovery_reserves_the_advertised_8192_token_context(self) -> None: + """A restart must not silently shrink the usable cache back to 2,068 tokens.""" + + repository_root = Path(__file__).resolve().parents[2] + recovery = repository_root / "scripts" / "gmk-evo-x2" / "start_qwen_recovery_server.sh" + contents = recovery.read_text(encoding="utf-8") + + self.assertIn('readonly KV_RESERVE_TOKENS="${FREETOKEN_KV_RESERVE_TOKENS:-8192}"', contents) + self.assertIn('--kv-reserve-tokens "${KV_RESERVE_TOKENS}"', contents) + + def test_recovery_uses_a_dedicated_group_and_checked_stop_helper(self) -> None: + """Recovery must make later GPU handoff safe for isolated ROCm candidates.""" + + repository_root = Path(__file__).resolve().parents[2] + recovery = repository_root / "scripts" / "gmk-evo-x2" / "start_qwen_recovery_server.sh" + stopper = repository_root / "scripts" / "gmk-evo-x2" / "stop_qwen_recovery_server.sh" + + self.assertIn('setsid nohup "${VENV_PYTHON}" -m freetoken.cli serve', recovery.read_text(encoding="utf-8")) + contents = stopper.read_text(encoding="utf-8") + self.assertIn('readonly PORT="1919"', contents) + self.assertIn('readonly MODEL_PATH="${HOME}/freetoken-amd/models/Qwen3.6-35B-A3B-NVFP4"', contents) + self.assertIn('[[ "${pgid}" == "${pid}" ]]', contents) + self.assertIn('kill -TERM -- "-${pgid}"', contents) + + def test_timeshare_endurance_requires_explicit_sources_and_health_recovery(self) -> None: + """The extended Q4 battery must fail closed and restore the protected service.""" + + repository_root = Path(__file__).resolve().parents[2] + controller = repository_root / "scripts" / "gmk-evo-x2" / "run_qwen_gguf_timeshare_endurance.sh" + contents = controller.read_text(encoding="utf-8") + + self.assertIn('FREETOKEN_Q4_SOURCE_DIR:?set FREETOKEN_Q4_SOURCE_DIR', contents) + self.assertIn('FREETOKEN_RECOVERY_SOURCE_DIR:?set FREETOKEN_RECOVERY_SOURCE_DIR', contents) + self.assertIn('readonly SESSION_COUNT="${2:-1440}"', contents) + self.assertIn('[[ -f "${Q4_LAUNCHER}" && -f "${Q4_BATTERY}"', contents) + self.assertIn('missing_listener >= 30', contents) + self.assertIn('trap \'restore_normal_service\' EXIT INT TERM', contents) + self.assertIn('wait_for_serving 1919 "${RECOVERY_ARTIFACT}"', contents) + self.assertIn('wait_for_serving 1922 "${ARTIFACT_ROOT}/q4-health.json"', contents) + + def test_q4_cleanup_accepts_an_already_exited_failed_frontend(self) -> None: + """A failed candidate must not prevent the normal service from recovering.""" + + repository_root = Path(__file__).resolve().parents[2] + launcher = repository_root / "scripts" / "gmk-evo-x2" / "launch_qwen_gguf_qualified.sh" + contents = launcher.read_text(encoding="utf-8") + + self.assertIn('kill -0 "${recorded_pid}" 2>/dev/null || exit 0', contents) + + def test_q4_launcher_builds_its_native_extension_in_a_clean_worktree(self) -> None: + """A clean candidate must not fail at runtime due to a missing HIP extension.""" + + repository_root = Path(__file__).resolve().parents[2] + launcher = repository_root / "scripts" / "gmk-evo-x2" / "launch_qwen_gguf_qualified.sh" + contents = launcher.read_text(encoding="utf-8") + + self.assertIn('readonly NATIVE_BUILD_LOG="${ARTIFACT_DIR}/native-extension-build.log"', contents) + self.assertIn('setup.py build_ext --inplace', contents) + self.assertIn('import freetoken.kernel._pinned_tensor as pinned', contents) + + def test_multiturn_battery_requires_swap_free_preflight(self) -> None: + """Repeated state tests must not begin from a swapped memory condition.""" + + repository_root = Path(__file__).resolve().parents[2] + wrapper = repository_root / "scripts" / "gmk-evo-x2" / "run_qwen_multiturn_battery.sh" + contents = wrapper.read_text(encoding="utf-8") + + self.assertIn('readonly MAX_SWAP_KIB="${GMK_EVO_X2_BATTERY_MAX_SWAP_KIB:-64}"', contents) + self.assertIn('refusing multi-turn battery with swap in use: ${used} KiB exceeds ${MAX_SWAP_KIB} KiB', contents) + self.assertIn('if (( used > MAX_SWAP_KIB )); then', contents) + self.assertIn('assert_clean_swap\ncurl -fsS', contents) + self.assertIn('"requested_sessions": expected', contents) + + +class ConcurrentControlArgumentTests(unittest.TestCase): + """Reject nonsensical concurrent workloads before they can reach GMKtek EVO-X2.""" + + def test_concurrency_must_be_positive(self) -> None: + """Zero clients has no latency or throughput meaning.""" + + with self.assertRaises(SystemExit): + parse_concurrent_args(["--model", "qwen", "--tokenizer", "tokenizer", "--artifact", "artifact", "--concurrency", "0"]) + + +class QwenEnduranceSummaryTests(unittest.TestCase): + """Ensure retained endurance evidence cannot hide missing or swapped sessions.""" + + def _write_session(self, root: Path, number: int, runner_swap_kib: int = 0) -> None: + """Write the smallest valid passed session plus its explicit telemetry.""" + + sessions = root / "sessions" + sessions.mkdir(exist_ok=True) + payload = { + "status": "passed", + "results": [{"id": "remember", "status": "passed"}], + "tail_metrics": {"max_ttft_seconds": 0.4, "max_token_gap_seconds": 0.02}, + } + (sessions / f"session-{number:02d}.json").write_text(json.dumps(payload)) + (sessions / f"session-{number:02d}-telemetry.txt").write_text( + f"runner_swap_kib={runner_swap_kib}\nwhole_host_swap_kib=39088\n" + ) + + def test_summary_passes_only_complete_zero_runner_swap_evidence(self) -> None: + """A complete artifact can include background swap without failing the runner gate.""" + + with TemporaryDirectory() as directory: + root = Path(directory) + self._write_session(root, 1) + summary = summarize(root, expected_sessions=1) + + self.assertTrue(summary["passed"]) + self.assertEqual(summary["runner_swap_kib"]["max"], 0) + self.assertEqual(summary["whole_host_swap_kib"]["max"], 39088) + + def test_summary_rejects_swapped_runner_or_missing_session(self) -> None: + """Neither runner paging nor an incomplete series may be reported as endurance-qualified.""" + + with TemporaryDirectory() as directory: + root = Path(directory) + self._write_session(root, 1, runner_swap_kib=4) + summary = summarize(root, expected_sessions=2) + + self.assertFalse(summary["passed"]) + self.assertTrue(any("expected 2 sessions" in failure for failure in summary["failures"])) + self.assertTrue(any("runner swap=4" in failure for failure in summary["failures"])) + + +class LlamaCppControlScriptTests(unittest.TestCase): + """Protect the isolated ROCm llama.cpp control lifecycle and workload reuse.""" + + def test_control_uses_a_loopback_child_and_existing_fixed_harness(self) -> None: + """The control must terminate its own port-1921 child and reuse Qwen inputs.""" + + repository_root = Path(__file__).resolve().parents[2] + wrapper = repository_root / "scripts" / "gmk-evo-x2" / "run_qwen_llamacpp_rocm_control.sh" + contents = wrapper.read_text(encoding="utf-8") + + self.assertIn('readonly BASE_URL="http://127.0.0.1:1921/v1"', contents) + self.assertIn('trap cleanup_server EXIT', contents) + self.assertIn('GMK_EVO_X2_QWEN_BASE_URL="${BASE_URL}"', contents) + self.assertIn('run_qwen_scheduler_baseline.sh', contents) + self.assertIn('--port 1921', contents) + + def test_timeshare_control_requires_serving_state_before_returning(self) -> None: + """A port-1919 HTTP response is insufficient while FreeToken is loading.""" + + repository_root = Path(__file__).resolve().parents[2] + wrapper = repository_root / "scripts" / "gmk-evo-x2" / "run_qwen_llamacpp_rocm_timeshare_control.sh" + contents = wrapper.read_text(encoding="utf-8") + + self.assertIn('"status":"ok"', contents) + self.assertIn('find_freetoken_pid', contents) + self.assertIn('sudo swapoff -a', contents) + self.assertIn('bash "${RECOVERY_SCRIPT}"', contents) + + def test_gemma_control_releases_stale_swap_only_after_qwen_stops(self) -> None: + """Gemma must start from a clean state without changing host swap policy.""" + + repository_root = Path(__file__).resolve().parents[2] + wrapper = repository_root / "scripts" / "gmk-evo-x2" / "run_gemma4_gguf_text_control.sh" + contents = wrapper.read_text(encoding="utf-8") + + self.assertIn('sudo swapoff -a', contents) + self.assertIn('sudo swapon -a', contents) + self.assertIn('swap-after-qwen-release.txt', contents) + self.assertLess(contents.index('production_pid="$(port_pid'), contents.index('sudo swapoff -a')) diff --git a/tests/benchmarks/test_public_document_privacy.py b/tests/benchmarks/test_public_document_privacy.py new file mode 100644 index 0000000000..3830065f9b --- /dev/null +++ b/tests/benchmarks/test_public_document_privacy.py @@ -0,0 +1,24 @@ +"""Keep deployment identities out of public AMD documentation.""" + +from pathlib import Path +import re + + +def test_public_amd_documents_use_anonymous_deployment_examples(): + root = Path(__file__).resolve().parents[2] + patterns = [ + re.compile(r"\bLAN-\d+\b", re.I), + re.compile(r"\b192\.168\.\d+\.\d+\b"), + re.compile(r"/(?:home|media)/(?!operator(?:/|\b)|user(?:/|\b)|username(?:/|\b))[^/\s`]+"), + re.compile(r"[\w.+-]+@(?:gmail|outlook|hotmail)\.com", re.I), + ] + violations = [] + paths = list((root / "docs").rglob("*")) + list((root / "paper-draft").rglob("*")) + paths += [root / ".zenodo.json", root / "CITATION.cff"] + for path in paths: + if path.suffix not in {".md", ".json", ".yaml", ".yml", ".tex", ".bib", ".cff"}: + continue + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if any(pattern.search(line) for pattern in patterns): + violations.append(f"{path.name}:{number}") + assert not violations, "Deployment identity in public docs: " + ", ".join(violations) diff --git a/tests/daemon/test_activity.py b/tests/daemon/test_activity.py new file mode 100644 index 0000000000..274dd64920 --- /dev/null +++ b/tests/daemon/test_activity.py @@ -0,0 +1,252 @@ +# What: enable postponed evaluation of annotations; why: type hints in test_activity can reference runtime types without eager imports or forward-reference failures. +from __future__ import annotations + +# What: import base64 for test capture is opt in redacted binary safe and evicted with row using base64; why: test_capture_is_opt_in_redacted_binary_safe_and_evicted_with_row uses base64 b64decode, making that imported dependency available to its named operation. +import base64 +# What: import time for record using time; why: _record uses time monotonic, making that imported dependency available to its named operation. +import time + +# What: import activity store for record using freetoken and daemon and activity and activity store; why: _record uses the activity store annotation in record, making that imported dependency available to its named operation. +from freetoken.daemon.activity import ActivityStore + + +# What: define the _record test helper around store and model and body and cancelled; why: the record scenario calls this helper to produce or observe the exact behavior checked by its assertions. +def _record(store: ActivityStore, *, model="a", body=b"ok", cancelled=False): + # What: return record and store and model and cancelled from the _record test helper; why: the record scenario uses this helper result in its subsequent act or assertion. + return store.record( + # What: arrange model to store.record; why: the record scenario binds this model value to store.record's model input. + model=model, + # What: arrange route to store.record; why: the record scenario binds this v1 and chat and completions value to store.record's route input. + route="/v1/chat/completions", + # What: arrange method to store.record; why: the record scenario binds this post value to store.record's method input. + method="POST", + # What: arrange status to store.record; why: the record scenario binds this 200 value to store.record's status input. + status=200, + # What: arrange started to time.monotonic; why: the record scenario binds this monotonic and time and 0 01 value to time.monotonic's started input. + started=time.monotonic() - 0.01, + # What: arrange ttft s to store.record; why: the record scenario binds this 0 002 value to store.record's ttft s input. + ttft_s=0.002, + # What: arrange response bytes to len; why: the record scenario binds this len and body value to len's response bytes input. + response_bytes=len(body), + # What: arrange cancelled to store.record; why: the record scenario binds this cancelled value to store.record's cancelled input. + cancelled=cancelled, + # What: arrange request headers to store.record; why: the record scenario binds this authorization and x auth token and x trace and bearer and secret value to store.record's request headers input. + request_headers={ + # What: arrange the authorization field as bearer and secret; why: _record carries authorization into "Authorization": "Bearer secret", "X-Auth-Token": "also-secret". + "Authorization": "Bearer secret", "X-Auth-Token": "also-secret", + # What: arrange the x trace field as visible; why: _record carries x trace into "X-Trace": "visible". + "X-Trace": "visible", + # What: arrange the enclosing predicate mapping with authorization and x auth token and x trace; why: _record groups the supplied clauses as one _record expression mapping before its value is consumed. + }, + # What: arrange request body to store.record; why: the record scenario binds no value to store.record's request body input. + request_body=b"\x00prompt", + # What: arrange the set cookie field as private; why: _record carries set cookie into response_headers={"Set-Cookie": "private", "Content-Type": "application/. + response_headers={"Set-Cookie": "private", "Content-Type": "application/octet-stream"}, + # What: arrange response body to store.record; why: the record scenario binds this body value to store.record's response body input. + response_body=body, + # What: arrange the store.record call with model and route and method and status and started; why: _record groups the supplied clauses as one store.record call before its value is consumed. + ) + + +# What: define the test_activity_rows_are_bounded_filterable_and_aggregated test around local fixtures; why: this test groups the arrange, act, and assertions that protect the activity rows are bounded filterable and aggregated outcome. +def test_activity_rows_are_bounded_filterable_and_aggregated(): + # What: act by calling ActivityStore and capture store; why: the activity rows are bounded filterable and aggregated test asserts the response, state, or failure produced by this call. + store = ActivityStore(max_entries=2, capture_budget_bytes=0) + # What: arrange the exact record store model a body b fixture fragment; why: the activity rows are bounded filterable and aggregated scenario feeds this byte-preserved fragment through _record(store, model="a", body=b"1") before asserting its protocol or parser result. + _record(store, model="a", body=b"1") + # What: arrange the exact record store model b body b fixture fragment; why: the activity rows are bounded filterable and aggregated scenario feeds this byte-preserved fragment through _record(store, model="b", body=b"22") before asserting its protocol or parser result. + _record(store, model="b", body=b"22") + # What: act by calling _record and capture latest; why: the activity rows are bounded filterable and aggregated test asserts the response, state, or failure produced by this call. + latest = _record(store, model="a", body=b"333", cancelled=True) + + # What: act by calling store.list and capture page; why: the activity rows are bounded filterable and aggregated test asserts the response, state, or failure produced by this call. + page = store.list(limit=1) + # What: assert that row id for row in page equals latest id; why: this assertion protects the activity rows are bounded filterable and aggregated regression after the test's arranged inputs and exercised call. + assert [row["id"] for row in page["data"]] == [latest["id"]] + # What: assert that page next before id equals latest id; why: this assertion protects the activity rows are bounded filterable and aggregated regression after the test's arranged inputs and exercised call. + assert page["nextBeforeId"] == latest["id"] + # What: assert that row model for row in store list equals a b; why: this assertion protects the activity rows are bounded filterable and aggregated regression after the test's arranged inputs and exercised call. + assert [row["model"] for row in store.list(limit=10)["data"]] == ["a", "b"] + # What: assert the expected store stats model a == outcome; why: test activity test activity rows are bounded filterable and aggregated protects its regression by requiring this observable result after the exercised behavior. + assert store.stats(model="a") == { + # What: arrange count 1 cancelled 1 errors 0 for the scenario; why: test activity test activity rows are bounded filterable and aggregated requires this concrete input or helper state before exercising the behavior under test. + "count": 1, "cancelled": 1, "errors": 0, + # What: arrange responseBytes 3 averageDurationS latest durationS for the scenario; why: test duration s requires this concrete input or helper state before exercising the behavior under test. + "responseBytes": 3, "averageDurationS": latest["durationS"], + # What: arrange persistence enabled False healthy True error None for the scenario; why: test activity test activity rows are bounded filterable and aggregated requires this concrete input or helper state before exercising the behavior under test. + "persistence": {"enabled": False, "healthy": True, "error": None}, + # What: arrange the grouped source fragment for the scenario; why: test activity test activity rows are bounded filterable and aggregated requires this concrete input or helper state before exercising the behavior under test. + } + + +# What: define the test_capture_is_opt_in_redacted_binary_safe_and_evicted_with_row test around local fixtures; why: this test groups the arrange, act, and assertions that protect the capture is opt in redacted binary safe and evicted with row outcome. +def test_capture_is_opt_in_redacted_binary_safe_and_evicted_with_row(): + # What: act by calling ActivityStore and capture store; why: the capture is opt in redacted binary safe and evicted with row test asserts the response, state, or failure produced by this call. + store = ActivityStore(max_entries=1, capture_budget_bytes=1024) + # What: act by calling _record and capture first; why: the capture is opt in redacted binary safe and evicted with row test asserts the response, state, or failure produced by this call. + first = _record(store, body=b"\xffresult") + # What: act by calling store.capture and capture capture; why: the capture is opt in redacted binary safe and evicted with row test asserts the response, state, or failure produced by this call. + capture = store.capture(first["id"]) + + # What: assert that first has capture is true; why: this assertion protects the capture is opt in redacted binary safe and evicted with row regression after the test's arranged inputs and exercised call. + assert first["hasCapture"] is True + # What: assert the expected capture requestHeaders == outcome; why: test activity test capture is opt in redacted binary safe and evicted with row protects its regression by requiring this observable result after the exercised behavior. + assert capture["requestHeaders"] == { + # What: arrange Authorization REDACTED X Auth Token REDACTED for the scenario; why: test activity test capture is opt in redacted binary safe and evicted with row requires this concrete input or helper state before exercising the behavior under test. + "Authorization": "[REDACTED]", "X-Auth-Token": "[REDACTED]", + # What: arrange X Trace visible for the scenario; why: test activity test capture is opt in redacted binary safe and evicted with row requires this concrete input or helper state before exercising the behavior under test. + "X-Trace": "visible", + # What: arrange the grouped source fragment for the scenario; why: test activity test capture is opt in redacted binary safe and evicted with row requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that capture response headers set cookie equals redacted; why: this assertion protects the capture is opt in redacted binary safe and evicted with row regression after the test's arranged inputs and exercised call. + assert capture["responseHeaders"]["Set-Cookie"] == "[REDACTED]" + # What: assert that base64 b64decode capture request body base64 equals b x00prompt; why: this assertion protects the capture is opt in redacted binary safe and evicted with row regression after the test's arranged inputs and exercised call. + assert base64.b64decode(capture["requestBodyBase64"]) == b"\x00prompt" + # What: assert that base64 b64decode capture response body base64 equals b xffresult; why: this assertion protects the capture is opt in redacted binary safe and evicted with row regression after the test's arranged inputs and exercised call. + assert base64.b64decode(capture["responseBodyBase64"]) == b"\xffresult" + + # What: act by calling _record and capture second; why: the capture is opt in redacted binary safe and evicted with row test asserts the response, state, or failure produced by this call. + second = _record(store, body=b"next") + # What: assert that the evicted first row has no retained capture; why: capture eviction must remove sensitive body data together with its activity row. + assert store.capture(first["id"]) is None + # What: assert that the retained second row still has a capture; why: eviction must not remove the newest body data while its activity row remains. + assert store.capture(second["id"]) is not None + + +# What: define the test_capture_skips_cancelled_and_over_budget_items_and_reconfigures test around local fixtures; why: this test groups the arrange, act, and assertions that protect the capture skips cancelled and over budget items and reconfigures outcome. +def test_capture_skips_cancelled_and_over_budget_items_and_reconfigures(): + # What: act by calling ActivityStore and capture store; why: the capture skips cancelled and over budget items and reconfigures test asserts the response, state, or failure produced by this call. + store = ActivityStore(max_entries=5, capture_budget_bytes=8) + # What: assert that record store body b too large has capture is false; why: this assertion protects the capture skips cancelled and over budget items and reconfigures regression after the test's arranged inputs and exercised call. + assert _record(store, body=b"too-large")["hasCapture"] is False + # What: assert that record store body b x cancelled is false; why: this assertion protects the capture skips cancelled and over budget items and reconfigures regression after the test's arranged inputs and exercised call. + assert _record(store, body=b"x", cancelled=True)["hasCapture"] is False + # What: act by calling store.reconfigure with 1 and 0; why: the capture skips cancelled and over budget items and reconfigures scenario observes the store.reconfigure return value during page store list limit. + store.reconfigure(1, 0) + # What: act by calling store.list and capture page; why: the capture skips cancelled and over budget items and reconfigures test asserts the response, state, or failure produced by this call. + page = store.list(limit=10) + # What: assert that page count equals 1; why: this assertion protects the capture skips cancelled and over budget items and reconfigures regression after the test's arranged inputs and exercised call. + assert page["count"] == 1 + # What: assert that page data 0 has capture is false; why: this assertion protects the capture skips cancelled and over budget items and reconfigures regression after the test's arranged inputs and exercised call. + assert page["data"][0]["hasCapture"] is False + # What: assert that store capture item limit equals 0; why: this assertion protects the capture skips cancelled and over budget items and reconfigures regression after the test's arranged inputs and exercised call. + assert store.capture_item_limit == 0 + + # What: act by calling ActivityStore and capture retained; why: the capture skips cancelled and over budget items and reconfigures test asserts the response, state, or failure produced by this call. + retained = ActivityStore(max_entries=2, capture_budget_bytes=1024) + # What: act by calling _record and capture row; why: the capture skips cancelled and over budget items and reconfigures test asserts the response, state, or failure produced by this call. + row = _record(retained, body=b"captured") + # What: assert that row has capture is true; why: this assertion protects the capture skips cancelled and over budget items and reconfigures regression after the test's arranged inputs and exercised call. + assert row["hasCapture"] is True + # What: act by calling retained.reconfigure with 2 and 0; why: the capture skips cancelled and over budget items and reconfigures scenario observes the retained.reconfigure return value during assert retained list limit data has capture is. + retained.reconfigure(2, 0) + # What: assert that retained list limit 2 data 0 has capture is false; why: this assertion protects the capture skips cancelled and over budget items and reconfigures regression after the test's arranged inputs and exercised call. + assert retained.list(limit=2)["data"][0]["hasCapture"] is False + # What: assert that retained capture row id is group delimiter; why: this assertion protects the capture skips cancelled and over budget items and reconfigures regression after the test's arranged inputs and exercised call. + assert retained.capture(row["id"]) is None + + +# What: define the test_body_free_activity_survives_restart_and_compacts_corrupt_history test around tmp path; why: this test groups the arrange, act, and assertions that protect the body free activity survives restart and compacts corrupt history outcome. +def test_body_free_activity_survives_restart_and_compacts_corrupt_history(tmp_path): + # What: arrange path as tmp path and activity and jsonl; why: the body free activity survives restart and compacts corrupt history test consumes this named precondition before exercising the behavior. + path = tmp_path / "activity.jsonl" + # What: act by calling ActivityStore and capture first; why: the body free activity survives restart and compacts corrupt history test asserts the response, state, or failure produced by this call. + first = ActivityStore(2, 1024, str(path)) + # What: arrange the exact record first model a body b fixture fragment; why: the body free activity survives restart and compacts corrupt history scenario feeds this byte-preserved fragment through _record(first, model="a", body=b"first") before asserting its protocol or parser result. + _record(first, model="a", body=b"first") + # What: arrange the exact record first model b body b fixture fragment; why: the body free activity survives restart and compacts corrupt history scenario feeds this byte-preserved fragment through _record(first, model="b", body=b"second") before asserting its protocol or parser result. + _record(first, model="b", body=b"second") + # What: act by calling _record and capture latest; why: the body free activity survives restart and compacts corrupt history test asserts the response, state, or failure produced by this call. + latest = _record(first, model="c", body=b"third") + # What: enter the path.open managed context before target write truncated n; why: test_body_free_activity_survives_restart_and_compacts_corrupt_history releases this resource or lock after target write truncated n on both success and failure paths. + with path.open("a", encoding="utf-8") as target: + # What: arrange the exact target write truncated n fixture fragment; why: the body free activity survives restart and compacts corrupt history scenario feeds this byte-preserved fragment through target.write("truncated{\n") before asserting its protocol or parser result. + target.write("truncated{\n") + # What: arrange the exact target write x n fixture fragment; why: the body free activity survives restart and compacts corrupt history scenario feeds this byte-preserved fragment through target.write("x" * 9000 + "\n") before asserting its protocol or parser result. + target.write("x" * 9000 + "\n") + + # What: act by calling ActivityStore and capture recovered; why: the body free activity survives restart and compacts corrupt history test asserts the response, state, or failure produced by this call. + recovered = ActivityStore(2, 1024, str(path)) + # What: act by calling recovered.list and capture page; why: the body free activity survives restart and compacts corrupt history test asserts the response, state, or failure produced by this call. + page = recovered.list(limit=10) + + # What: assert that row model for row in page equals c b; why: this assertion protects the body free activity survives restart and compacts corrupt history regression after the test's arranged inputs and exercised call. + assert [row["model"] for row in page["data"]] == ["c", "b"] + # What: assert that all row has capture is false for row in page data; why: this assertion protects the body free activity survives restart and compacts corrupt history regression after the test's arranged inputs and exercised call. + assert all(row["hasCapture"] is False for row in page["data"]) + # What: assert that page persistence equals enabled true healthy true error; why: this assertion protects the body free activity survives restart and compacts corrupt history regression after the test's arranged inputs and exercised call. + assert page["persistence"] == {"enabled": True, "healthy": True, "error": None} + # What: assert that recovered capture latest id is group delimiter; why: this assertion protects the body free activity survives restart and compacts corrupt history regression after the test's arranged inputs and exercised call. + assert recovered.capture(latest["id"]) is None + # What: act by calling _record and capture next row; why: the body free activity survives restart and compacts corrupt history test asserts the response, state, or failure produced by this call. + next_row = _record(recovered, model="d") + # What: assert that next row id equals latest id 1; why: this assertion protects the body free activity survives restart and compacts corrupt history regression after the test's arranged inputs and exercised call. + assert next_row["id"] == latest["id"] + 1 + # What: assert that len path read text encoding utf 8 splitlines is at most 4; why: this assertion protects the body free activity survives restart and compacts corrupt history regression after the test's arranged inputs and exercised call. + assert len(path.read_text(encoding="utf-8").splitlines()) <= 4 + + +# What: define the test_persistence_failure_never_breaks_in_memory_activity test around tmp path; why: this test groups the arrange, act, and assertions that protect the persistence failure never breaks in memory activity outcome. +def test_persistence_failure_never_breaks_in_memory_activity(tmp_path): + # What: arrange missing parent as tmp path and activity and jsonl and missing; why: the persistence failure never breaks in memory activity test consumes this named precondition before exercising the behavior. + missing_parent = tmp_path / "missing" / "activity.jsonl" + # What: act by calling ActivityStore and capture store; why: the persistence failure never breaks in memory activity test asserts the response, state, or failure produced by this call. + store = ActivityStore(2, 0, str(missing_parent)) + # What: act by calling _record and capture row; why: the persistence failure never breaks in memory activity test asserts the response, state, or failure produced by this call. + row = _record(store) + + # What: act by calling store.list and capture page; why: the persistence failure never breaks in memory activity test asserts the response, state, or failure produced by this call. + page = store.list(limit=10) + # What: assert that page data 0 id equals row id; why: this assertion protects the persistence failure never breaks in memory activity regression after the test's arranged inputs and exercised call. + assert page["data"][0]["id"] == row["id"] + # What: assert the expected page persistence == outcome; why: test activity test persistence failure never breaks in memory activity protects its regression by requiring this observable result after the exercised behavior. + assert page["persistence"] == { + # What: arrange enabled True healthy False error write failed for the scenario; why: test activity test persistence failure never breaks in memory activity requires this concrete input or helper state before exercising the behavior under test. + "enabled": True, "healthy": False, "error": "write_failed", + # What: arrange the grouped source fragment for the scenario; why: test activity test persistence failure never breaks in memory activity requires this concrete input or helper state before exercising the behavior under test. + } + + +# What: define the test_load_failure_requires_atomic_rewrite_before_health_recovers test around tmp path and monkeypatch; why: this test groups the arrange, act, and assertions that protect the load failure requires atomic rewrite before health recovers outcome. +def test_load_failure_requires_atomic_rewrite_before_health_recovers(tmp_path, monkeypatch): + # What: arrange path as tmp path and activity and jsonl; why: the load failure requires atomic rewrite before health recovers test consumes this named precondition before exercising the behavior. + path = tmp_path / "activity.jsonl" + # What: arrange the exact path write text unread history n encoding utf 8 fixture fragment; why: the load failure requires atomic rewrite before health recovers scenario feeds this byte-preserved fragment through path.write_text('{"unread":"history"}\n', encoding="utf-8") before asserting its protocol or parser re. + path.write_text('{"unread":"history"}\n', encoding="utf-8") + # What: arrange real open as open; why: the load failure requires atomic rewrite before health recovers test consumes this named precondition before exercising the behavior. + real_open = open + + # What: define the fail_initial_read test helper around name; why: the load failure requires atomic rewrite before health recovers scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def fail_initial_read(name, *args, **kwargs): + # What: act on args and str and name and path before oserror; why: the load failure requires atomic rewrite before health recovers scenario admits oserror only for this predicate and excludes the opposite state. + if str(name) == str(path) and not args: + # What: raise OSError for the caller; why: fail_initial_read stops this rejected path before it can mutate state, dispatch work, or report success. + raise OSError("private path detail") + # What: return real open and name and args and kwargs from the fail_initial_read test helper; why: the load failure requires atomic rewrite before health recovers scenario uses this helper result in its subsequent act or assertion. + return real_open(name, *args, **kwargs) + + # What: arrange the exact monkeypatch setattr builtins open fail initial read fixture fragment; why: the load failure requires atomic rewrite before health recovers scenario feeds this byte-preserved fragment through monkeypatch.setattr("builtins.open", fail_initial_read) before asserting its protocol or parser re. + monkeypatch.setattr("builtins.open", fail_initial_read) + # What: act by calling ActivityStore and capture store; why: the load failure requires atomic rewrite before health recovers test asserts the response, state, or failure produced by this call. + store = ActivityStore(2, 0, str(path)) + # What: arrange the exact monkeypatch setattr builtins open real open fixture fragment; why: the load failure requires atomic rewrite before health recovers scenario feeds this byte-preserved fragment through monkeypatch.setattr("builtins.open", real_open) before asserting its protocol or parser result. + monkeypatch.setattr("builtins.open", real_open) + # What: assert the expected store list persistence == outcome; why: test activity test load failure requires atomic rewrite before health recovers protects its regression by requiring this observable result after the exercised behavior. + assert store.list()["persistence"] == { + # What: arrange enabled True healthy False error load failed for the scenario; why: test activity test load failure requires atomic rewrite before health recovers requires this concrete input or helper state before exercising the behavior under test. + "enabled": True, "healthy": False, "error": "load_failed", + # What: arrange the grouped source fragment for the scenario; why: test activity test load failure requires atomic rewrite before health recovers requires this concrete input or helper state before exercising the behavior under test. + } + + # What: act by calling _record and capture row; why: the load failure requires atomic rewrite before health recovers test asserts the response, state, or failure produced by this call. + row = _record(store) + # What: assert the expected store list persistence == outcome; why: test activity test load failure requires atomic rewrite before health recovers protects its regression by requiring this observable result after the exercised behavior. + assert store.list()["persistence"] == { + # What: arrange enabled True healthy True error None for the scenario; why: test activity test load failure requires atomic rewrite before health recovers requires this concrete input or helper state before exercising the behavior under test. + "enabled": True, "healthy": True, "error": None, + # What: arrange the grouped source fragment for the scenario; why: test activity test load failure requires atomic rewrite before health recovers requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that path read text encoding utf 8 count n equals 1; why: this assertion protects the load failure requires atomic rewrite before health recovers regression after the test's arranged inputs and exercised call. + assert path.read_text(encoding="utf-8").count("\n") == 1 + # What: assert that activity store 2 0 str path list equals row id; why: this assertion protects the load failure requires atomic rewrite before health recovers regression after the test's arranged inputs and exercised call. + assert ActivityStore(2, 0, str(path)).list()["data"][0]["id"] == row["id"] diff --git a/tests/daemon/test_catalog.py b/tests/daemon/test_catalog.py new file mode 100644 index 0000000000..87dcb0ce5f --- /dev/null +++ b/tests/daemon/test_catalog.py @@ -0,0 +1,1576 @@ +# What: enable postponed evaluation of annotations; why: type hints in test_catalog can reference runtime types without eager imports or forward-reference failures. +from __future__ import annotations + +# What: import pytest for module initialization using pytest; why: module initialization uses pytest mark parametrize, making that imported dependency available to its named operation. +import pytest +# What: arrange from concurrent futures import ThreadPoolExecutor for the scenario; why: test catalog requires this concrete input or helper state before exercising the behavior under test. +from concurrent.futures import ThreadPoolExecutor +# What: import test client for test profile api uses validated catalog and existing switch transaction using fastapi and testclient and test client; why: test_profile_api_uses_validated_catalog_and_existing_switch_transaction uses test client, making that imported dependency available to its named operation. +from fastapi.testclient import TestClient + +# What: arrange from freetoken daemon catalog import CatalogError ModelCapabilities ModelCatalog for the scenario; why: test catalog requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.catalog import CatalogError, ModelCapabilities, ModelCatalog +# What: import build app for test profile api uses validated catalog and existing switch transaction using freetoken and daemon and app and build app; why: test_profile_api_uses_validated_catalog_and_existing_switch_transaction uses build app, making that imported dependency available to its named operation. +from freetoken.daemon.app import build_app +# What: arrange from freetoken daemon import client as daemon client for the scenario; why: test catalog requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon import client as daemon_client +# What: import log ring for test profile api uses validated catalog and existing switch transaction using freetoken and daemon and logring and log ring; why: test_profile_api_uses_validated_catalog_and_existing_switch_transaction uses log ring, making that imported dependency available to its named operation. +from freetoken.daemon.logring import LogRing +# What: arrange from freetoken daemon readiness import wait for ready for the scenario; why: test catalog requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.readiness import wait_for_ready + + +# What: define the test_catalog_reads_named_profiles_without_shell_interpolation test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog reads named profiles without shell interpolation outcome. +def test_catalog_reads_named_profiles_without_shell_interpolation(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog reads named profiles without shell interpolation test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with models and qwen coder and model and models; why: the catalog reads named profiles without shell interpolation scenario observes the path.write_text return value during models qwen coder nmodel models qwen gguf nport nargs. + path.write_text( + # What: arrange the exact models qwen coder nmodel models qwen gguf nport nargs fixture fragment; why: the catalog reads named profiles without shell interpolation scenario feeds this byte-preserved fragment through """[models.qwen-coder]\nmodel = \"/models/qwen.gguf\"\nport = 1922\nargs before asserting its p. + """[models.qwen-coder]\nmodel = \"/models/qwen.gguf\"\nport = 1922\nargs = [\"--max-seq-len-override\", \"32768\"]\ndescription = \"coding profile\"\n""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog reads named profiles without shell interpolation scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_reads_named_profiles_without_shell_interpolation groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: act by calling ModelCatalog.load and capture catalog; why: the catalog reads named profiles without shell interpolation test asserts the response, state, or failure produced by this call. + catalog = ModelCatalog.load(str(path)) + # What: assert the expected catalog get qwen coder request == outcome; why: test catalog test catalog reads named profiles without shell interpolation protects its regression by requiring this observable result after the exercised behavior. + assert catalog.get("qwen-coder").request() == { + # What: arrange model models qwen gguf port 1922 args max seq len override 32768 for the scenario; why: test catalog test catalog reads named profiles without shell interpolation requires this concrete input or helper state before exercising the behavior under test. + "model": "/models/qwen.gguf", "port": 1922, "args": ["--max-seq-len-override", "32768"] + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog reads named profiles without shell interpolation requires this. + } + # What: assert the expected catalog public == outcome; why: test catalog test catalog reads named profiles without shell interpolation protects its regression by requiring this observable result after the exercised behavior. + assert catalog.public() == [{ + # What: arrange name qwen coder model models qwen gguf port 1922 for the scenario; why: test catalog test catalog reads named profiles without shell interpolation requires this concrete input or helper state before exercising the behavior under test. + "name": "qwen-coder", "model": "/models/qwen.gguf", "port": 1922, + # What: arrange args max seq len override 32768 description coding profile readyTimeoutS 120.0 for the scenario; why: test catalog reads named profiles without shell requires this concrete input or helper state before exercising the behavior under test. + "args": ["--max-seq-len-override", "32768"], "description": "coding profile", "readyTimeoutS": 120.0, + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog reads named profiles without shell interpolation requires this concrete input or helper state before exercising the. + }] + + +# What: define the test_catalog_validates_safe_upstream_no_activation_suffixes test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog validates safe upstream no activation suffixes outcome. +def test_catalog_validates_safe_upstream_no_activation_suffixes(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog validates safe upstream no activation suffixes test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with router and upstream no activation suffixes and wasm and map; why: the catalog validates safe upstream no activation suffixes scenario observes the path.write_text return value during router. + path.write_text( + # What: arrange the exact router fixture fragment; why: the catalog validates safe upstream no activation suffixes scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact upstream no activation suffixes wasm map fixture fragment; why: the catalog validates safe upstream no activation suffixes scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact models local fixture fragment; why: the catalog validates safe upstream no activation suffixes scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact model local gguf fixture fragment; why: the catalog validates safe upstream no activation suffixes scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the catalog validates safe upstream no activation suffixes scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + """[router] +upstream_no_activation_suffixes = [".wasm", ".map"] + +[models.local] +model = "local.gguf" +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog validates safe upstream no activation suffixes scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_validates_safe_upstream_no_activation_suffixes groups the supplied clauses as one path.write_text call before its value is consumed. + ) + + # What: act by calling ModelCatalog.load and capture settings; why: the catalog validates safe upstream no activation suffixes test asserts the response, state, or failure produced by this call. + settings = ModelCatalog.load(str(path)).settings + + # What: assert that settings upstream no activation suffixes equals wasm map; why: this assertion protects the catalog validates safe upstream no activation suffixes regression after the test's arranged inputs and exercised call. + assert settings.upstream_no_activation_suffixes == (".wasm", ".map") + + +# What: define the test_catalog_validates_bounded_activity_and_capture_settings test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog validates bounded activity and capture settings outcome. +def test_catalog_validates_bounded_activity_and_capture_settings(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog validates bounded activity and capture settings test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with router and activity max entries and capture buffer mb and activity session headers; why: the catalog validates bounded activity and capture settings scenario observes the path.write_text return value during router. + path.write_text( + # What: arrange the exact router fixture fragment; why: the catalog validates bounded activity and capture settings scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact activity max entries fixture fragment; why: the catalog validates bounded activity and capture settings scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact capture buffer mb fixture fragment; why: the catalog validates bounded activity and capture settings scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact activity session headers x conversation id fixture fragment; why: the catalog validates bounded activity and capture settings scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact performance disabled true fixture fragment; why: the catalog validates bounded activity and capture settings scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact performance every s fixture fragment; why: the catalog validates bounded activity and capture settings scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact models local fixture fragment; why: the catalog validates bounded activity and capture settings scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact model local gguf fixture fragment; why: the catalog validates bounded activity and capture settings scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the catalog validates bounded activity and capture settings scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + """[router] +activity_max_entries = 25 +capture_buffer_mb = 4 +activity_session_headers = ["X-Conversation-ID"] +performance_disabled = true +performance_every_s = 30 + +[models.local] +model = "local.gguf" +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog validates bounded activity and capture settings scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_validates_bounded_activity_and_capture_settings groups the supplied clauses as one path.write_text call before its value is consumed. + ) + + # What: act by calling ModelCatalog.load and capture settings; why: the catalog validates bounded activity and capture settings test asserts the response, state, or failure produced by this call. + settings = ModelCatalog.load(str(path)).settings + + # What: assert that settings activity max entries equals 25; why: this assertion protects the catalog validates bounded activity and capture settings regression after the test's arranged inputs and exercised call. + assert settings.activity_max_entries == 25 + # What: assert that settings capture buffer mb equals 4; why: this assertion protects the catalog validates bounded activity and capture settings regression after the test's arranged inputs and exercised call. + assert settings.capture_buffer_mb == 4 + # What: assert that settings activity session headers equals x conversation id; why: this assertion protects the catalog validates bounded activity and capture settings regression after the test's arranged inputs and exercised call. + assert settings.activity_session_headers == ("x-conversation-id",) + # What: assert that settings performance disabled is true; why: this assertion protects the catalog validates bounded activity and capture settings regression after the test's arranged inputs and exercised call. + assert settings.performance_disabled is True + # What: assert that settings performance every s equals 30; why: this assertion protects the catalog validates bounded activity and capture settings regression after the test's arranged inputs and exercised call. + assert settings.performance_every_s == 30 + + +# What: parameterize test_catalog_rejects_unbounded_activity_or_capture_settings with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects unbounded activity or capture settings. +@pytest.mark.parametrize("key,value", [ + # What: arrange activity max entries 0 for the scenario; why: test catalog test catalog rejects unbounded activity or capture settings requires this concrete input or helper state before exercising the behavior under test. + ("activity_max_entries", 0), + # What: arrange activity max entries 100001 for the scenario; why: test catalog test catalog rejects unbounded activity or capture settings requires this concrete input or helper state before exercising the behavior under test. + ("activity_max_entries", 100001), + # What: arrange capture buffer mb 1 for the scenario; why: test catalog test catalog rejects unbounded activity or capture settings requires this concrete input or helper state before exercising the behavior under test. + ("capture_buffer_mb", -1), + # What: arrange capture buffer mb 257 for the scenario; why: test catalog test catalog rejects unbounded activity or capture settings requires this concrete input or helper state before exercising the behavior under test. + ("capture_buffer_mb", 257), + # What: arrange performance every s 4 for the scenario; why: test catalog test catalog rejects unbounded activity or capture settings requires this concrete input or helper state before exercising the behavior under test. + ("performance_every_s", 4), + # What: arrange performance every s 3601 for the scenario; why: test catalog test catalog rejects unbounded activity or capture settings requires this concrete input or helper state before exercising the behavior under test. + ("performance_every_s", 3601), + # What: arrange the performance disabled portion of the enclosing predicate; why: this clause remains in the catalog rejects unbounded activity or capture settings scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("performance_disabled", 1), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_catalog_rejects_unbounded_activity_or_capture_settings groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_catalog_rejects_unbounded_activity_or_capture_settings test around tmp path and key and value; why: this test groups the arrange, act, and assertions that protect the catalog rejects unbounded activity or capture settings outcome. +def test_catalog_rejects_unbounded_activity_or_capture_settings(tmp_path, key, value): + # What: arrange path as tmp path and models and toml; why: the catalog rejects unbounded activity or capture settings test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with key and value and router and value and models; why: the catalog rejects unbounded activity or capture settings scenario observes the path.write_text return value during f router n key value n. + path.write_text( + # What: arrange the exact f router n key value n fixture fragment; why: the catalog rejects unbounded activity or capture settings scenario feeds this byte-preserved fragment through f'[router]\n{key} = {value}\n\n[models.local]\nmodel = "local.gguf"\n' before asserting its protocol or parser result. + f'[router]\n{key} = {value}\n\n[models.local]\nmodel = "local.gguf"\n', + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog rejects unbounded activity or capture settings scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_rejects_unbounded_activity_or_capture_settings groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: assert the pytest.raises failure context; why: the catalog rejects unbounded activity or capture settings scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match=key): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects unbounded activity or capture settings scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: parameterize test_catalog_rejects_credential_or_invalid_activity_session_headers with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects credential or invalid activity session headers. +@pytest.mark.parametrize("headers", [ + # What: arrange the authorization portion of the enclosing predicate; why: this clause remains in the catalog rejects credential or invalid activity session headers scenario\'s enclosing expression so its grouping and evaluation order stay intact. + '["Authorization"]', + # What: arrange the x auth token portion of the enclosing predicate; why: this clause remains in the catalog rejects credential or invalid activity session headers scenario\'s enclosing expression so its grouping and evaluation order stay intact. + '["X-Auth-Token"]', + # What: arrange the x api key portion of the enclosing predicate; why: this clause remains in the catalog rejects credential or invalid activity session headers scenario\'s enclosing expression so its grouping and evaluation order stay intact. + '["X-Api-Key"]', + # What: arrange the x session id x session id portion of the enclosing predicate; why: this clause remains in the catalog rejects credential or invalid activity session headers scenario\'s enclosing expression so its grouping and evaluation order stay intact. + '["X-Session-ID", "x-session-id"]', + # What: arrange the bad header portion of the enclosing predicate; why: this clause remains in the catalog rejects credential or invalid activity session headers scenario\'s enclosing expression so its grouping and evaluation order stay intact. + '["bad header"]', +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_catalog_rejects_credential_or_invalid_activity_session_headers groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_catalog_rejects_credential_or_invalid_activity_session_headers test around tmp path and headers; why: this test groups the arrange, act, and assertions that protect the catalog rejects credential or invalid activity session headers outcome. +def test_catalog_rejects_credential_or_invalid_activity_session_headers(tmp_path, headers): + # What: arrange path as tmp path and models and toml; why: the catalog rejects credential or invalid activity session headers test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with headers and router and activity session headers and models and local; why: the catalog rejects credential or invalid activity session headers scenario observes the path.write_text return value during f router nactivity session headers headers n n. + path.write_text( + # What: arrange the exact f router nactivity session headers headers n n fixture fragment; why: the catalog rejects credential or invalid activity session headers scenario feeds this byte-preserved fragment through f'[router]\nactivity_session_headers = {headers}\n\n[models.local]\nmode before asserting its pr. + f'[router]\nactivity_session_headers = {headers}\n\n[models.local]\nmodel = "local.gguf"\n', + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog rejects credential or invalid activity session headers scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_rejects_credential_or_invalid_activity_session_headers groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: assert the pytest.raises failure context; why: the catalog rejects credential or invalid activity session headers scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match="activity_session_headers"): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects credential or invalid activity session headers scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: parameterize test_catalog_rejects_unsafe_upstream_no_activation_suffixes with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects unsafe upstream no activation suffixes. +@pytest.mark.parametrize("value", [ + # What: arrange the js portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe upstream no activation suffixes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + '".js"', + # What: arrange the js portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe upstream no activation suffixes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + '["js"]', + # What: arrange the secret portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe upstream no activation suffixes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + '["../secret"]', + # What: arrange the js js portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe upstream no activation suffixes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + '[".js", ".js"]', +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_catalog_rejects_unsafe_upstream_no_activation_suffixes groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_catalog_rejects_unsafe_upstream_no_activation_suffixes test around tmp path and value; why: this test groups the arrange, act, and assertions that protect the catalog rejects unsafe upstream no activation suffixes outcome. +def test_catalog_rejects_unsafe_upstream_no_activation_suffixes(tmp_path, value): + # What: arrange path as tmp path and models and toml; why: the catalog rejects unsafe upstream no activation suffixes test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with value and router and upstream no activation suffixes and models and local; why: the catalog rejects unsafe upstream no activation suffixes scenario observes the path.write_text return value during f router. + path.write_text( + # What: arrange the exact f router fixture fragment; why: the catalog rejects unsafe upstream no activation suffixes scenario feeds this byte-preserved fragment through f"""[router] before asserting its protocol or parser result. + # What: arrange the exact upstream no activation suffixes value fixture fragment; why: the catalog rejects unsafe upstream no activation suffixes scenario feeds this byte-preserved fragment through f"""[router] before asserting its protocol or parser result. + # What: arrange the exact models local fixture fragment; why: the catalog rejects unsafe upstream no activation suffixes scenario feeds this byte-preserved fragment through f"""[router] before asserting its protocol or parser result. + # What: arrange the exact model local gguf fixture fragment; why: the catalog rejects unsafe upstream no activation suffixes scenario feeds this byte-preserved fragment through f"""[router] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the catalog rejects unsafe upstream no activation suffixes scenario feeds this byte-preserved fragment through f"""[router] before asserting its protocol or parser result. + f"""[router] +upstream_no_activation_suffixes = {value} + +[models.local] +model = "local.gguf" +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog rejects unsafe upstream no activation suffixes scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_rejects_unsafe_upstream_no_activation_suffixes groups the supplied clauses as one path.write_text call before its value is consumed. + ) + + # What: assert the pytest.raises failure context; why: the catalog rejects unsafe upstream no activation suffixes scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match="upstream_no_activation_suffixes"): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects unsafe upstream no activation suffixes scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: define the test_catalog_validates_custom_readiness_and_owned_loopback_proxy_targets test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog validates custom readiness and owned loopback proxy targets outcome. +def test_catalog_validates_custom_readiness_and_owned_loopback_proxy_targets(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog validates custom readiness and owned loopback proxy targets test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with models and coding and model and coding; why: the catalog validates custom readiness and owned loopback proxy targets scenario observes the path.write_text return value during models coding. + path.write_text( + # What: arrange the exact models coding fixture fragment; why: the catalog validates custom readiness and owned loopback proxy targets scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact model coding gguf fixture fragment; why: the catalog validates custom readiness and owned loopback proxy targets scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact port fixture fragment; why: the catalog validates custom readiness and owned loopback proxy targets scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact check endpoint ready fixture fragment; why: the catalog validates custom readiness and owned loopback proxy targets scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact proxy http port gateway v1 fixture fragment; why: the catalog validates custom readiness and owned loopback proxy targets scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the catalog validates custom readiness and owned loopback proxy targets scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + """[models.coding] +model = "coding.gguf" +port = 1922 +check_endpoint = "/ready" +proxy = "http://127.0.0.1:${PORT}/gateway/v1" +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog validates custom readiness and owned loopback proxy targets scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_validates_custom_readiness_and_owned_loopback_proxy_targets groups the supplied clauses as one path.write_text call before its value is consumed. + ) + + # What: act by calling operation.get and capture profile; why: the catalog validates custom readiness and owned loopback proxy targets test asserts the response, state, or failure produced by this call. + profile = ModelCatalog.load(str(path)).get("coding") + + # What: assert that profile check endpoint equals ready; why: this assertion protects the catalog validates custom readiness and owned loopback proxy targets regression after the test's arranged inputs and exercised call. + assert profile.check_endpoint == "/ready" + # What: assert that profile proxy base url 1922 equals http 127 0 0 1 1922 gateway v1; why: this assertion protects the catalog validates custom readiness and owned loopback proxy targets regression after the test's arranged inputs and exercised call. + assert profile.proxy_base_url(1922) == "http://127.0.0.1:1922/gateway/v1" + # What: assert that profile public check endpoint equals ready; why: this assertion protects the catalog validates custom readiness and owned loopback proxy targets regression after the test's arranged inputs and exercised call. + assert profile.public()["checkEndpoint"] == "/ready" + # What: assert that profile public proxy equals http 127 0 0 1 port gateway v1; why: this assertion protects the catalog validates custom readiness and owned loopback proxy targets regression after the test's arranged inputs and exercised call. + assert profile.public()["proxy"] == "http://127.0.0.1:${PORT}/gateway/v1" + + +# What: parameterize test_catalog_rejects_unsafe_readiness_or_proxy_targets with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects unsafe readiness or proxy targets. +@pytest.mark.parametrize("field,value,message", [ + # What: arrange the check endpoint ready absolute ascii path portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe readiness or proxy targets scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("check_endpoint", "ready", "absolute ASCII path"), + # What: arrange the check endpoint health absolute ascii path portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe readiness or proxy targets scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("check_endpoint", "/../health", "absolute ASCII path"), + # What: arrange the check endpoint health token x absolute ascii portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe readiness or proxy targets scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("check_endpoint", "/health?token=x", "absolute ASCII path"), + # What: arrange the proxy http portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe readiness or proxy targets scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("proxy", "http://127.0.0.1:1922", "127.0.0.1"), + # What: arrange the proxy http localhost port portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe readiness or proxy targets scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("proxy", "http://localhost:${PORT}", "127.0.0.1"), + # What: arrange the proxy https port portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe readiness or proxy targets scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("proxy", "https://127.0.0.1:${PORT}", "127.0.0.1"), + # What: arrange the proxy http port admin portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe readiness or proxy targets scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("proxy", "http://127.0.0.1:${PORT}/../admin", "127.0.0.1"), + # What: arrange the proxy http port api token x portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe readiness or proxy targets scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("proxy", "http://127.0.0.1:${PORT}/api?token=x", "127.0.0.1"), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_catalog_rejects_unsafe_readiness_or_proxy_targets groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_catalog_rejects_unsafe_readiness_or_proxy_targets test around tmp path and field and value and message; why: this test groups the arrange, act, and assertions that protect the catalog rejects unsafe readiness or proxy targets outcome. +def test_catalog_rejects_unsafe_readiness_or_proxy_targets( + # What: arrange the tmp path input for test_catalog_rejects_unsafe_readiness_or_proxy_targets; why: test_catalog_rejects_unsafe_readiness_or_proxy_targets consumes tmp path during path tmp path models toml, so callers must bind it with the other signature inputs. + tmp_path, field, value, message +# What: arrange the grouped source fragment for the scenario; why: test catalog rejects unsafe readiness or proxy targets requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange path as tmp path and models and toml; why: the catalog rejects unsafe readiness or proxy targets test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with field and value and models and bad and model; why: the catalog rejects unsafe readiness or proxy targets scenario observes the path.write_text return value during f models bad nmodel bad gguf n field. + path.write_text( + # What: arrange the exact f models bad nmodel bad gguf n field fixture fragment; why: the catalog rejects unsafe readiness or proxy targets scenario feeds this byte-preserved fragment through f'[models.bad]\nmodel = "bad.gguf"\n{field} = "{value}"\n' before asserting its protocol or parser result. + f'[models.bad]\nmodel = "bad.gguf"\n{field} = "{value}"\n', + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog rejects unsafe readiness or proxy targets scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_rejects_unsafe_readiness_or_proxy_targets groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: assert the pytest.raises failure context; why: the catalog rejects unsafe readiness or proxy targets scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match=message): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects unsafe readiness or proxy targets scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: define the test_catalog_validates_and_exposes_supported_listing_capabilities test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog validates and exposes supported listing capabilities outcome. +def test_catalog_validates_and_exposes_supported_listing_capabilities(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog validates and exposes supported listing capabilities test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with models and coding and model and coding; why: the catalog validates and exposes supported listing capabilities scenario observes the path.write_text return value during models coding. + path.write_text( + # What: arrange the exact models coding fixture fragment; why: the catalog validates and exposes supported listing capabilities scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact model coding gguf fixture fragment; why: the catalog validates and exposes supported listing capabilities scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact models coding capabilities fixture fragment; why: the catalog validates and exposes supported listing capabilities scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact in text fixture fragment; why: the catalog validates and exposes supported listing capabilities scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact out text fixture fragment; why: the catalog validates and exposes supported listing capabilities scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact tools true fixture fragment; why: the catalog validates and exposes supported listing capabilities scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact context fixture fragment; why: the catalog validates and exposes supported listing capabilities scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the catalog validates and exposes supported listing capabilities scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + """[models.coding] +model = "coding.gguf" + +[models.coding.capabilities] +in = ["text"] +out = ["text"] +tools = true +context = 32768 +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog validates and exposes supported listing capabilities scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_validates_and_exposes_supported_listing_capabilities groups the supplied clauses as one path.write_text call before its value is consumed. + ) + + # What: act by calling operation.get and capture profile; why: the catalog validates and exposes supported listing capabilities test asserts the response, state, or failure produced by this call. + profile = ModelCatalog.load(str(path)).get("coding") + + # What: assert that profile capabilities equals model capabilities text text true 32768; why: this assertion protects the catalog validates and exposes supported listing capabilities regression after the test's arranged inputs and exercised call. + assert profile.capabilities == ModelCapabilities(("text",), ("text",), True, 32768) + # What: assert the expected profile public capabilities == outcome; why: test catalog test catalog validates and exposes supported listing capabilities protects its regression by requiring this observable result after the exercised behavior. + assert profile.public()["capabilities"] == { + # What: arrange in text out text tools True context 32768 for the scenario; why: test catalog test catalog validates and exposes supported listing capabilities requires this concrete input or helper state before exercising the behavior under test. + "in": ["text"], "out": ["text"], "tools": True, "context": 32768, + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog validates and exposes supported listing capabilities requires this concrete input or helper state before exercising the behavior under test. + } + + +# What: define the test_catalog_validates_model_display_name_and_json_metadata test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog validates model display name and json metadata outcome. +def test_catalog_validates_model_display_name_and_json_metadata(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog validates model display name and json metadata test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with models and coding and model and coding; why: the catalog validates model display name and json metadata scenario observes the path.write_text return value during models coding. + path.write_text( + # What: arrange the exact models coding fixture fragment; why: the catalog validates model display name and json metadata scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact model coding gguf fixture fragment; why: the catalog validates model display name and json metadata scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact name coding model fixture fragment; why: the catalog validates model display name and json metadata scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact description fixture fragment; why: the catalog validates model display name and json metadata scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact upstream timeout s fixture fragment; why: the catalog validates model display name and json metadata scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact models coding metadata fixture fragment; why: the catalog validates model display name and json metadata scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact tier stable fixture fragment; why: the catalog validates model display name and json metadata scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact tags local text fixture fragment; why: the catalog validates model display name and json metadata scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact models coding metadata nested fixture fragment; why: the catalog validates model display name and json metadata scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact enabled true fixture fragment; why: the catalog validates model display name and json metadata scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the catalog validates model display name and json metadata scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + """[models.coding] +model = "coding.gguf" +name = " Coding Model " +description = " " +upstream_timeout_s = 45 + +[models.coding.metadata] +tier = "stable" +tags = ["local", "text"] + +[models.coding.metadata.nested] +enabled = true +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog validates model display name and json metadata scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_validates_model_display_name_and_json_metadata groups the supplied clauses as one path.write_text call before its value is consumed. + ) + + # What: act by calling operation.get and capture profile; why: the catalog validates model display name and json metadata test asserts the response, state, or failure produced by this call. + profile = ModelCatalog.load(str(path)).get("coding") + + # What: assert that profile display name equals coding model; why: this assertion protects the catalog validates model display name and json metadata regression after the test's arranged inputs and exercised call. + assert profile.display_name == "Coding Model" + # What: assert that profile description is group delimiter; why: this assertion protects the catalog validates model display name and json metadata regression after the test's arranged inputs and exercised call. + assert profile.description is None + # What: assert that profile upstream timeout s equals 45; why: this assertion protects the catalog validates model display name and json metadata regression after the test's arranged inputs and exercised call. + assert profile.upstream_timeout_s == 45 + # What: assert the expected profile metadata == outcome; why: test catalog test catalog validates model display name and json metadata protects its regression by requiring this observable result after the exercised behavior. + assert profile.metadata() == { + # What: arrange nested enabled True tags local text tier stable for the scenario; why: test catalog test catalog validates model display name and json metadata requires this concrete input or helper state before exercising the behavior under test. + "nested": {"enabled": True}, "tags": ["local", "text"], "tier": "stable", + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog validates model display name and json metadata requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that profile public display name equals coding model; why: this assertion protects the catalog validates model display name and json metadata regression after the test's arranged inputs and exercised call. + assert profile.public()["displayName"] == "Coding Model" + # What: assert that profile public metadata equals profile metadata; why: this assertion protects the catalog validates model display name and json metadata regression after the test's arranged inputs and exercised call. + assert profile.public()["metadata"] == profile.metadata() + # What: assert that profile public upstream timeout s equals 45; why: this assertion protects the catalog validates model display name and json metadata regression after the test's arranged inputs and exercised call. + assert profile.public()["upstreamTimeoutS"] == 45 + + +# What: define the test_catalog_validates_request_fields_and_creates_variant_aliases test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog validates request fields and creates variant aliases outcome. +def test_catalog_validates_request_fields_and_creates_variant_aliases(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog validates request fields and creates variant aliases test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with models and coding and model and coding; why: the catalog validates request fields and creates variant aliases scenario observes the path.write_text return value during models coding. + path.write_text( + # What: arrange the exact models coding fixture fragment; why: the catalog validates request fields and creates variant aliases scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact model coding gguf fixture fragment; why: the catalog validates request fields and creates variant aliases scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact use model name engine coding v1 fixture fragment; why: the catalog validates request fields and creates variant aliases scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact drop fields metadata private fixture fragment; why: the catalog validates request fields and creates variant aliases scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact models coding set fields fixture fragment; why: the catalog validates request fields and creates variant aliases scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange models coding for the scenario; why: test catalog test catalog validates request fields and creates variant aliases requires this concrete input or helper state before exercising the behavior under test. + # What: arrange the exact max tokens fixture fragment; why: the catalog validates request fields and creates variant aliases scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact chat template kwargs enable thinking true fixture fragment; why: the catalog validates request fields and creates variant aliases scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact models coding set fields by id coding high fixture fragment; why: the catalog validates request fields and creates variant aliases scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange models coding for the scenario; why: test catalog test catalog validates request fields and creates variant aliases requires this concrete input or helper state before exercising the behavior under test. + # What: arrange the exact chat template kwargs reasoning effort high fixture fragment; why: the catalog validates request fields and creates variant aliases scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the catalog validates request fields and creates variant aliases scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + """[models.coding] +model = "coding.gguf" +use_model_name = "engine/coding-v1" +drop_fields = ["metadata.private"] + +[models.coding.set_fields] +temperature = 0.2 +"max_tokens?" = 4096 +"chat_template_kwargs.enable_thinking?" = true + +[models.coding.set_fields_by_id."coding:high"] +temperature = 0.1 +"chat_template_kwargs.reasoning_effort" = "high" +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog validates request fields and creates variant aliases scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_validates_request_fields_and_creates_variant_aliases groups the supplied clauses as one path.write_text call before its value is consumed. + ) + + # What: act by calling ModelCatalog.load and capture catalog; why: the catalog validates request fields and creates variant aliases test asserts the response, state, or failure produced by this call. + catalog = ModelCatalog.load(str(path)) + # What: act by calling catalog.get and capture profile; why: the catalog validates request fields and creates variant aliases test asserts the response, state, or failure produced by this call. + profile = catalog.get("coding:high") + + # What: assert that profile is catalog get coding; why: this assertion protects the catalog validates request fields and creates variant aliases regression after the test's arranged inputs and exercised call. + assert profile is catalog.get("coding") + # What: assert that profile aliases equals coding high; why: this assertion protects the catalog validates request fields and creates variant aliases regression after the test's arranged inputs and exercised call. + assert profile.aliases == ("coding:high",) + # What: assert that profile use model name equals engine coding v1; why: this assertion protects the catalog validates request fields and creates variant aliases regression after the test's arranged inputs and exercised call. + assert profile.use_model_name == "engine/coding-v1" + # What: assert that profile drop fields equals metadata private; why: this assertion protects the catalog validates request fields and creates variant aliases regression after the test's arranged inputs and exercised call. + assert profile.drop_fields == ("metadata.private",) + # What: assert the expected profile public setFields == outcome; why: test catalog test catalog validates request fields and creates variant aliases protects its regression by requiring this observable result after the exercised behavior. + assert profile.public()["setFields"] == { + # What: arrange temperature 0.2 for the scenario; why: test catalog test catalog validates request fields and creates variant aliases requires this concrete input or helper state before exercising the behavior under test. + "temperature": 0.2, + # What: arrange chat template kwargs enable thinking True for the scenario; why: test catalog test catalog validates request fields and creates variant aliases requires this concrete input or helper state before exercising the behavior under test. + "chat_template_kwargs.enable_thinking?": True, + # What: arrange max tokens 4096 for the scenario; why: test catalog test catalog validates request fields and creates variant aliases requires this concrete input or helper state before exercising the behavior under test. + "max_tokens?": 4096, + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog validates request fields and creates variant aliases requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert the expected profile public setFieldsById == outcome; why: test catalog test catalog validates request fields and creates variant aliases protects its regression by requiring this observable result after the exercised behavior. + assert profile.public()["setFieldsById"] == { + # What: arrange coding high for the scenario; why: test catalog test catalog validates request fields and creates variant aliases requires this concrete input or helper state before exercising the behavior under test. + "coding:high": { + # What: arrange chat template kwargs reasoning effort high for the scenario; why: test catalog test catalog validates request fields and creates variant aliases requires this concrete input or helper state before exercising the behavior under test. + "chat_template_kwargs.reasoning_effort": "high", + # What: arrange temperature 0.1 for the scenario; why: test catalog test catalog validates request fields and creates variant aliases requires this concrete input or helper state before exercising the behavior under test. + "temperature": 0.1, + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog validates request fields and creates variant aliases requires this concrete input or helper state before exercising the behavior under test. + } + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog validates request fields and creates variant aliases requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that profile public use model name equals engine coding v1; why: this assertion protects the catalog validates request fields and creates variant aliases regression after the test's arranged inputs and exercised call. + assert profile.public()["useModelName"] == "engine/coding-v1" + + +# What: define the test_catalog_hard_request_field_wins_over_soft_spelling test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog hard request field wins over soft spelling outcome. +def test_catalog_hard_request_field_wins_over_soft_spelling(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog hard request field wins over soft spelling test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with models and coding and model and coding; why: the catalog hard request field wins over soft spelling scenario observes the path.write_text return value during models coding. + path.write_text( + # What: arrange the exact models coding fixture fragment; why: the catalog hard request field wins over soft spelling scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact model coding gguf fixture fragment; why: the catalog hard request field wins over soft spelling scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange the exact models coding set fields fixture fragment; why: the catalog hard request field wins over soft spelling scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + # What: arrange models coding for the scenario; why: test catalog test catalog hard request field wins over soft spelling requires this concrete input or helper state before exercising the behavior under test. + # What: arrange models coding for the scenario; why: test catalog test catalog hard request field wins over soft spelling requires this concrete input or helper state before exercising the behavior under test. + # What: arrange the exact the grouped expression fixture fragment; why: the catalog hard request field wins over soft spelling scenario feeds this byte-preserved fragment through """[models.coding] before asserting its protocol or parser result. + """[models.coding] +model = "coding.gguf" +[models.coding.set_fields] +max_tokens = 1000 +"max_tokens?" = 2000 +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog hard request field wins over soft spelling scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_hard_request_field_wins_over_soft_spelling groups the supplied clauses as one path.write_text call before its value is consumed. + ) + + # What: act by calling operation.get and capture fields; why: the catalog hard request field wins over soft spelling test asserts the response, state, or failure produced by this call. + fields = ModelCatalog.load(str(path)).get("coding").set_fields + + # What: assert the expected field key field value field soft for field in fields == outcome; why: test catalog test catalog hard request field wins over soft spelling protects its regression by requiring this observable result after the exercised behavior. + assert [(field.key, field.value(), field.soft) for field in fields] == [ + # What: arrange max tokens 1000 False for the scenario; why: test catalog test catalog hard request field wins over soft spelling requires this concrete input or helper state before exercising the behavior under test. + ("max_tokens", 1000, False) + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog hard request field wins over soft spelling requires this concrete input or helper state before exercising the behavior under test. + ] + + +# What: parameterize test_catalog_rejects_unsupported_or_malformed_capabilities with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects unsupported or malformed capabilities. +@pytest.mark.parametrize("declaration,message", [ + # What: arrange the in image unsupported modalities image portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or malformed capabilities scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('in = ["image"]', "unsupported modalities: image"), + # What: arrange the out audio unsupported modalities audio portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or malformed capabilities scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('out = ["audio"]', "unsupported modalities: audio"), + # What: arrange the out video unsupported modalities video portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or malformed capabilities scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('out = ["video"]', "unsupported modalities: video"), + # What: arrange the in text text must not contain portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or malformed capabilities scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('in = ["text", "text"]', "must not contain duplicates"), + # What: arrange the tools tools must be a boolean portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or malformed capabilities scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("tools = 1", "tools must be a boolean"), + # What: arrange the context context must be a nonnegative portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or malformed capabilities scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("context = -1", "context must be a nonnegative integer"), + # What: arrange the context true context must be a portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or malformed capabilities scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("context = true", "context must be a nonnegative integer"), + # What: arrange the reranker true unsupported keys reranker portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or malformed capabilities scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("reranker = true", "unsupported keys: reranker"), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_catalog_rejects_unsupported_or_malformed_capabilities groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_catalog_rejects_unsupported_or_malformed_capabilities test around tmp path and declaration and message; why: this test groups the arrange, act, and assertions that protect the catalog rejects unsupported or malformed capabilities outcome. +def test_catalog_rejects_unsupported_or_malformed_capabilities( + # What: arrange the tmp path input for test_catalog_rejects_unsupported_or_malformed_capabilities; why: test_catalog_rejects_unsupported_or_malformed_capabilities consumes tmp path during path tmp path models toml, so callers must bind it with the other signature inputs. + tmp_path, declaration, message +# What: arrange the grouped source fragment for the scenario; why: test catalog rejects unsupported or malformed capabilities requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange path as tmp path and models and toml; why: the catalog rejects unsupported or malformed capabilities test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with declaration and models and coding and model and coding; why: the catalog rejects unsupported or malformed capabilities scenario observes the path.write_text return value during f models coding nmodel coding gguf n. + path.write_text( + # What: arrange the exact f models coding nmodel coding gguf n fixture fragment; why: the catalog rejects unsupported or malformed capabilities scenario feeds this byte-preserved fragment through f'[models.coding]\nmodel = "coding.gguf"\n' before asserting its protocol or parser result. + # What: arrange the exact f models coding capabilities n declaration n fixture fragment; why: the catalog rejects unsupported or malformed capabilities scenario feeds this byte-preserved fragment through f'[models.coding]\nmodel = "coding.gguf"\n' before asserting its protocol or parser result. + f'[models.coding]\nmodel = "coding.gguf"\n' + f'[models.coding.capabilities]\n{declaration}\n', + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog rejects unsupported or malformed capabilities scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_rejects_unsupported_or_malformed_capabilities groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: assert the pytest.raises failure context; why: the catalog rejects unsupported or malformed capabilities scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match=message): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects unsupported or malformed capabilities scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: parameterize test_catalog_rejects_unsafe_request_field_configuration with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects unsafe request field configuration. +@pytest.mark.parametrize("declaration,message", [ + # What: arrange the models coding set fields nmodel other must not set portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe request field configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('[models.coding.set_fields]\nmodel = "other"', "must not set model"), + # What: arrange the models coding set fields n model other must not portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe request field configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('[models.coding.set_fields]\n"model?" = "other"', "must not set model"), + # What: arrange the models coding set fields n bad path safe dot delimited portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe request field configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('[models.coding.set_fields]\n"bad..path" = 1', "safe dot-delimited"), + # What: arrange the models coding set fields nstarted json compatible portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe request field configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('[models.coding.set_fields]\nstarted = 2026-09-14', "JSON-compatible"), + # What: arrange the enclosing predicate collection with models and coding and set fields by id and; why: test_catalog_rejects_unsafe_request_field_configuration groups the supplied clauses as one test_catalog_rejects_unsafe_request_field_configuration expression collection before its. + ( + # What: arrange the models coding set fields by id bad alias ntemperature portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe request field configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + '[models.coding.set_fields_by_id."bad//alias"]\ntemperature = 1', + # What: arrange the slash separated portion of the enclosing predicate; why: this clause remains in the catalog rejects unsafe request field configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "slash-separated", + # What: arrange the enclosing predicate collection with models and coding and set fields by id and; why: test_catalog_rejects_unsafe_request_field_configuration groups the supplied clauses as one test_catalog_rejects_unsafe_request_field_configuration expression collection before its. + ), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_catalog_rejects_unsafe_request_field_configuration groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_catalog_rejects_unsafe_request_field_configuration test around tmp path and declaration and message; why: this test groups the arrange, act, and assertions that protect the catalog rejects unsafe request field configuration outcome. +def test_catalog_rejects_unsafe_request_field_configuration( + # What: arrange the tmp path input for test_catalog_rejects_unsafe_request_field_configuration; why: test_catalog_rejects_unsafe_request_field_configuration consumes tmp path during path tmp path models toml, so callers must bind it with the other signature inputs. + tmp_path, declaration, message +# What: arrange the grouped source fragment for the scenario; why: test catalog rejects unsafe request field configuration requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange path as tmp path and models and toml; why: the catalog rejects unsafe request field configuration test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with declaration and models and coding and model and coding; why: the catalog rejects unsafe request field configuration scenario observes the path.write_text return value during f models coding nmodel coding gguf n declaration. + path.write_text( + # What: arrange the exact f models coding nmodel coding gguf n declaration fixture fragment; why: the catalog rejects unsafe request field configuration scenario feeds this byte-preserved fragment through f'[models.coding]\nmodel = "coding.gguf"\n{declaration}\n' before asserting its protocol or parser result. + f'[models.coding]\nmodel = "coding.gguf"\n{declaration}\n', + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog rejects unsafe request field configuration scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_rejects_unsafe_request_field_configuration groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: assert the pytest.raises failure context; why: the catalog rejects unsafe request field configuration scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match=message): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects unsafe request field configuration scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: define the test_filter_generated_alias_cannot_collide_with_another_profile test around tmp path; why: this test groups the arrange, act, and assertions that protect the filter generated alias cannot collide with another profile outcome. +def test_filter_generated_alias_cannot_collide_with_another_profile(tmp_path): + # What: arrange path as tmp path and models and toml; why: the filter generated alias cannot collide with another profile test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with models and one and model and one; why: the filter generated alias cannot collide with another profile scenario observes the path.write_text return value during models one. + path.write_text( + # What: arrange the exact models one fixture fragment; why: the filter generated alias cannot collide with another profile scenario feeds this byte-preserved fragment through """[models.one] before asserting its protocol or parser result. + # What: arrange the exact model one gguf fixture fragment; why: the filter generated alias cannot collide with another profile scenario feeds this byte-preserved fragment through """[models.one] before asserting its protocol or parser result. + # What: arrange the exact models one set fields by id two fixture fragment; why: the filter generated alias cannot collide with another profile scenario feeds this byte-preserved fragment through """[models.one] before asserting its protocol or parser result. + # What: arrange the exact temperature fixture fragment; why: the filter generated alias cannot collide with another profile scenario feeds this byte-preserved fragment through """[models.one] before asserting its protocol or parser result. + # What: arrange the exact models two fixture fragment; why: the filter generated alias cannot collide with another profile scenario feeds this byte-preserved fragment through """[models.one] before asserting its protocol or parser result. + # What: arrange the exact model two gguf fixture fragment; why: the filter generated alias cannot collide with another profile scenario feeds this byte-preserved fragment through """[models.one] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the filter generated alias cannot collide with another profile scenario feeds this byte-preserved fragment through """[models.one] before asserting its protocol or parser result. + """[models.one] +model = "one.gguf" +[models.one.set_fields_by_id.two] +temperature = 0.1 +[models.two] +model = "two.gguf" +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the filter generated alias cannot collide with another profile scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_filter_generated_alias_cannot_collide_with_another_profile groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: assert the pytest.raises failure context; why: the filter generated alias cannot collide with another profile scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match="conflicts with a configured profile"): + # What: act by calling ModelCatalog.load with str and path; why: the filter generated alias cannot collide with another profile scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: parameterize test_catalog_rejects_ambiguous_or_shell_style_profiles with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects ambiguous or shell style profiles. +@pytest.mark.parametrize("content, message", [ + # What: arrange the models bad nmodel m nargs port n portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or shell style profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[models.bad]\nmodel = 'm'\nargs = ['--port', '9']\n", "must not set --model or --port"), + # What: arrange the models bad nmodel m ncmd anything n portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or shell style profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[models.bad]\nmodel = 'm'\ncmd = 'anything'\n", "unsupported keys"), + # What: arrange the models bad nmodel n non empty string portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or shell style profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[models.bad]\nmodel = ''\n", "non-empty string"), + # What: arrange the models bad nmodel m nport n through portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or shell style profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[models.bad]\nmodel = 'm'\nport = -1\n", "0 through 65535"), + # What: arrange the models bad nmodel m nuse model name n non empty portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or shell style profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[models.bad]\nmodel = 'm'\nuse_model_name = ''\n", "non-empty trimmed"), + # What: arrange the models bad nmodel m nuse model name bad n portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or shell style profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[models.bad]\nmodel = 'm'\nuse_model_name = ' bad'\n", "non-empty trimmed"), + # What: arrange the models bad nmodel m nupstream timeout s n upstream timeout s portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or shell style profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[models.bad]\nmodel = 'm'\nupstream_timeout_s = 0\n", "upstream_timeout_s"), + # What: arrange the enclosing predicate collection with models and bad and model and m and json; why: test_catalog_rejects_ambiguous_or_shell_style_profiles groups the supplied clauses as one test_catalog_rejects_ambiguous_or_shell_style_profiles expression collection before its. + ( + # What: arrange the models bad nmodel m n models bad metadata ncreated portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or shell style profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "[models.bad]\nmodel = 'm'\n[models.bad.metadata]\ncreated = 2026-09-14\n", + # What: arrange the json compatible portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or shell style profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "JSON-compatible", + # What: arrange the enclosing predicate collection with models and bad and model and m and json; why: test_catalog_rejects_ambiguous_or_shell_style_profiles groups the supplied clauses as one test_catalog_rejects_ambiguous_or_shell_style_profiles expression collection before its. + ), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_catalog_rejects_ambiguous_or_shell_style_profiles groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_catalog_rejects_ambiguous_or_shell_style_profiles test around tmp path and content and message; why: this test groups the arrange, act, and assertions that protect the catalog rejects ambiguous or shell style profiles outcome. +def test_catalog_rejects_ambiguous_or_shell_style_profiles(tmp_path, content, message): + # What: arrange path as tmp path and models and toml; why: the catalog rejects ambiguous or shell style profiles test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text content encoding utf 8 fixture fragment; why: the catalog rejects ambiguous or shell style profiles scenario feeds this byte-preserved fragment through path.write_text(content, encoding="utf-8") before asserting its protocol or parser result. + path.write_text(content, encoding="utf-8") + # What: assert the pytest.raises failure context; why: the catalog rejects ambiguous or shell style profiles scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match=message): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects ambiguous or shell style profiles scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: define the test_catalog_unknown_profile_has_operator_facing_error test around local fixtures; why: this test groups the arrange, act, and assertions that protect the catalog unknown profile has operator facing error outcome. +def test_catalog_unknown_profile_has_operator_facing_error(): + # What: assert the pytest.raises failure context; why: the catalog unknown profile has operator facing error scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match="unknown model profile 'missing'"): + # What: arrange the exact model catalog empty get missing fixture fragment; why: the catalog unknown profile has operator facing error scenario feeds this byte-preserved fragment through ModelCatalog.empty().get("missing") before asserting its protocol or parser result. + ModelCatalog.empty().get("missing") + + +# What: define the test_catalog_marks_port_zero_as_an_explicit_dynamic_port test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog marks port zero as an explicit dynamic port outcome. +def test_catalog_marks_port_zero_as_an_explicit_dynamic_port(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog marks port zero as an explicit dynamic port test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text models dynamic nmodel m nport n fixture fragment; why: the catalog marks port zero as an explicit dynamic port scenario feeds this byte-preserved fragment through path.write_text("[models.dynamic]\nmodel = 'm'\nport = 0\n", encoding="u before asserting its protocol or pars. + path.write_text("[models.dynamic]\nmodel = 'm'\nport = 0\n", encoding="utf-8") + # What: act by calling operation.get and capture profile; why: the catalog marks port zero as an explicit dynamic port test asserts the response, state, or failure produced by this call. + profile = ModelCatalog.load(str(path)).get("dynamic") + # What: assert that profile port equals 0; why: this assertion protects the catalog marks port zero as an explicit dynamic port regression after the test's arranged inputs and exercised call. + assert profile.port == 0 + # What: assert that profile public dynamic port is true; why: this assertion protects the catalog marks port zero as an explicit dynamic port regression after the test's arranged inputs and exercised call. + assert profile.public()["dynamicPort"] is True + + +# What: define the test_catalog_resolves_collision_safe_aliases_and_hides_unlisted_models test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog resolves collision safe aliases and hides unlisted models outcome. +def test_catalog_resolves_collision_safe_aliases_and_hides_unlisted_models(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog resolves collision safe aliases and hides unlisted models test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with router and include aliases in list and true and models; why: the catalog resolves collision safe aliases and hides unlisted models scenario observes the path.write_text return value during router. + path.write_text( + # What: arrange the exact router fixture fragment; why: the catalog resolves collision safe aliases and hides unlisted models scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact include aliases in list true fixture fragment; why: the catalog resolves collision safe aliases and hides unlisted models scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact models visible fixture fragment; why: the catalog resolves collision safe aliases and hides unlisted models scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact model visible gguf fixture fragment; why: the catalog resolves collision safe aliases and hides unlisted models scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact aliases nickname compat id fixture fragment; why: the catalog resolves collision safe aliases and hides unlisted models scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact models hidden fixture fragment; why: the catalog resolves collision safe aliases and hides unlisted models scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact model hidden gguf fixture fragment; why: the catalog resolves collision safe aliases and hides unlisted models scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact aliases private name fixture fragment; why: the catalog resolves collision safe aliases and hides unlisted models scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact unlisted true fixture fragment; why: the catalog resolves collision safe aliases and hides unlisted models scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the catalog resolves collision safe aliases and hides unlisted models scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + """[router] +include_aliases_in_list = true + +[models.visible] +model = "visible.gguf" +aliases = ["nickname", "compat-id"] + +[models.hidden] +model = "hidden.gguf" +aliases = ["private-name"] +unlisted = true +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog resolves collision safe aliases and hides unlisted models scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_resolves_collision_safe_aliases_and_hides_unlisted_models groups the supplied clauses as one path.write_text call before its value is consumed. + ) + + # What: act by calling ModelCatalog.load and capture catalog; why: the catalog resolves collision safe aliases and hides unlisted models test asserts the response, state, or failure produced by this call. + catalog = ModelCatalog.load(str(path)) + + # What: assert that catalog get nickname is catalog get visible; why: this assertion protects the catalog resolves collision safe aliases and hides unlisted models regression after the test's arranged inputs and exercised call. + assert catalog.get("nickname") is catalog.get("visible") + # What: assert that catalog get private name is catalog get hidden; why: this assertion protects the catalog resolves collision safe aliases and hides unlisted models regression after the test's arranged inputs and exercised call. + assert catalog.get("private-name") is catalog.get("hidden") + # What: assert that catalog listed model ids equals visible nickname compat id; why: this assertion protects the catalog resolves collision safe aliases and hides unlisted models regression after the test's arranged inputs and exercised call. + assert catalog.listed_model_ids() == ("visible", "nickname", "compat-id") + # What: act by calling ModelCatalog and capture default listing; why: the catalog resolves collision safe aliases and hides unlisted models test asserts the response, state, or failure produced by this call. + default_listing = ModelCatalog({ + # What: arrange the visible field as get and catalog and visible; why: test_catalog_resolves_collision_safe_aliases_and_hides_unlisted_models carries visible through default listing into assert default listing listed model ids equals visible. + "visible": catalog.get("visible"), + # What: arrange the hidden field as get and catalog and hidden; why: test_catalog_resolves_collision_safe_aliases_and_hides_unlisted_models carries hidden through default listing into assert default listing listed model ids equals visible. + "hidden": catalog.get("hidden"), + # What: arrange the ModelCatalog call with get; why: test_catalog_resolves_collision_safe_aliases_and_hides_unlisted_models groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + # What: assert that default listing listed model ids equals visible; why: this assertion protects the catalog resolves collision safe aliases and hides unlisted models regression after the test's arranged inputs and exercised call. + assert default_listing.listed_model_ids() == ("visible",) + # What: act by calling catalog.public and capture public; why: the catalog resolves collision safe aliases and hides unlisted models test asserts the response, state, or failure produced by this call. + public = {profile["name"]: profile for profile in catalog.public()} + # What: assert that public visible aliases equals nickname compat id; why: this assertion protects the catalog resolves collision safe aliases and hides unlisted models regression after the test's arranged inputs and exercised call. + assert public["visible"]["aliases"] == ["nickname", "compat-id"] + # What: assert that public hidden unlisted is true; why: this assertion protects the catalog resolves collision safe aliases and hides unlisted models regression after the test's arranged inputs and exercised call. + assert public["hidden"]["unlisted"] is True + + +# What: parameterize test_catalog_rejects_ambiguous_or_invalid_model_aliases with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects ambiguous or invalid model aliases. +@pytest.mark.parametrize( + # What: arrange the models message portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or invalid model aliases scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "models,message", + # What: arrange the grouped source fragment for the scenario; why: test catalog rejects ambiguous or invalid model aliases requires this concrete input or helper state before exercising the behavior under test. + [ + # What: arrange the enclosing predicate collection with models and one and model and one and conflicts; why: test_catalog_rejects_ambiguous_or_invalid_model_aliases groups the supplied clauses as one test_catalog_rejects_ambiguous_or_invalid_model_aliases expression collection before its. + ( + # What: arrange the exact models one nmodel one gguf naliases two n fixture fragment; why: the catalog rejects ambiguous or invalid model aliases scenario feeds this byte-preserved fragment through "[models.one]\nmodel='one.gguf'\naliases=['two']\n" before asserting its protocol or parser result. + # What: arrange the exact models two nmodel two gguf n fixture fragment; why: the catalog rejects ambiguous or invalid model aliases scenario feeds this byte-preserved fragment through "[models.one]\nmodel='one.gguf'\naliases=['two']\n" before asserting its protocol or parser result. + "[models.one]\nmodel='one.gguf'\naliases=['two']\n" + "[models.two]\nmodel='two.gguf'\n", + # What: arrange the conflicts with a configured profile portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or invalid model aliases scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "conflicts with a configured profile", + # What: arrange the enclosing predicate collection with models and one and model and one and conflicts; why: test_catalog_rejects_ambiguous_or_invalid_model_aliases groups the supplied clauses as one test_catalog_rejects_ambiguous_or_invalid_model_aliases expression collection before its. + ), + # What: arrange the enclosing predicate collection with models and one and model and one and assigned; why: test_catalog_rejects_ambiguous_or_invalid_model_aliases groups the supplied clauses as one test_catalog_rejects_ambiguous_or_invalid_model_aliases expression collection before its. + ( + # What: arrange the exact models one nmodel one gguf naliases shared n fixture fragment; why: the catalog rejects ambiguous or invalid model aliases scenario feeds this byte-preserved fragment through "[models.one]\nmodel='one.gguf'\naliases=['shared']\n" before asserting its protocol or parser result. + # What: arrange the exact models two nmodel two gguf naliases shared n fixture fragment; why: the catalog rejects ambiguous or invalid model aliases scenario feeds this byte-preserved fragment through "[models.one]\nmodel='one.gguf'\naliases=['shared']\n" before asserting its protocol or parser result. + "[models.one]\nmodel='one.gguf'\naliases=['shared']\n" + "[models.two]\nmodel='two.gguf'\naliases=['shared']\n", + # What: arrange the assigned to both portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or invalid model aliases scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "assigned to both", + # What: arrange the enclosing predicate collection with models and one and model and one and assigned; why: test_catalog_rejects_ambiguous_or_invalid_model_aliases groups the supplied clauses as one test_catalog_rejects_ambiguous_or_invalid_model_aliases expression collection before its. + ), + # What: arrange the models one nmodel one gguf naliases bad name portion of the enclosing predicate; why: this clause remains in the catalog rejects ambiguous or invalid model aliases scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[models.one]\nmodel='one.gguf'\naliases=['bad//name']\n", "distinct valid"), + # What: arrange the grouped source fragment for the scenario; why: test catalog rejects ambiguous or invalid model aliases requires this concrete input or helper state before exercising the behavior under test. + ], +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_catalog_rejects_ambiguous_or_invalid_model_aliases groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +) +# What: define the test_catalog_rejects_ambiguous_or_invalid_model_aliases test around tmp path and models and message; why: this test groups the arrange, act, and assertions that protect the catalog rejects ambiguous or invalid model aliases outcome. +def test_catalog_rejects_ambiguous_or_invalid_model_aliases(tmp_path, models, message): + # What: arrange path as tmp path and models and toml; why: the catalog rejects ambiguous or invalid model aliases test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text models encoding utf 8 fixture fragment; why: the catalog rejects ambiguous or invalid model aliases scenario feeds this byte-preserved fragment through path.write_text(models, encoding="utf-8") before asserting its protocol or parser result. + path.write_text(models, encoding="utf-8") + # What: assert the pytest.raises failure context; why: the catalog rejects ambiguous or invalid model aliases scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match=message): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects ambiguous or invalid model aliases scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: parameterize test_catalog_rejects_unsafe_namespaced_model_ids with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects unsafe namespaced model ids. +@pytest.mark.parametrize("model_id", ["bad//name", "bad/../name", "/bad"]) +# What: define the test_catalog_rejects_unsafe_namespaced_model_ids test around tmp path and model id; why: this test groups the arrange, act, and assertions that protect the catalog rejects unsafe namespaced model ids outcome. +def test_catalog_rejects_unsafe_namespaced_model_ids(tmp_path, model_id): + # What: arrange path as tmp path and models and toml; why: the catalog rejects unsafe namespaced model ids test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with model id and models and model and model and gguf; why: the catalog rejects unsafe namespaced model ids scenario observes the path.write_text return value during f models model id nmodel model gguf n. + path.write_text( + # What: arrange the exact f models model id nmodel model gguf n fixture fragment; why: the catalog rejects unsafe namespaced model ids scenario feeds this byte-preserved fragment through f'[models."{model_id}"]\nmodel = "model.gguf"\n', encoding="utf-8" before asserting its protocol or parser result. + f'[models."{model_id}"]\nmodel = "model.gguf"\n', encoding="utf-8" + # What: arrange the path.write_text call with encoding; why: test_catalog_rejects_unsafe_namespaced_model_ids groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: assert the pytest.raises failure context; why: the catalog rejects unsafe namespaced model ids scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match="slash-separated"): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects unsafe namespaced model ids scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: define the test_catalog_accepts_colon_variant_model_ids test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog accepts colon variant model ids outcome. +def test_catalog_accepts_colon_variant_model_ids(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog accepts colon variant model ids test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with models and coding and high and model; why: the catalog accepts colon variant model ids scenario observes the path.write_text return value during models coding high nmodel coding gguf n. + path.write_text( + # What: arrange the exact models coding high nmodel coding gguf n fixture fragment; why: the catalog accepts colon variant model ids scenario feeds this byte-preserved fragment through '[models."coding:high"]\nmodel = "coding.gguf"\n', encoding="utf-8" before asserting its protocol or parser result. + '[models."coding:high"]\nmodel = "coding.gguf"\n', encoding="utf-8" + # What: arrange the path.write_text call with encoding; why: test_catalog_accepts_colon_variant_model_ids groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: assert that model catalog load str path get coding high equals coding high; why: this assertion protects the catalog accepts colon variant model ids regression after the test's arranged inputs and exercised call. + assert ModelCatalog.load(str(path)).get("coding:high").name == "coding:high" + + +# What: define the test_catalog_validates_pin_and_warm_selectors test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog validates pin and warm selectors outcome. +def test_catalog_validates_pin_and_warm_selectors(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog validates pin and warm selectors test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with models and a and model and a; why: the catalog validates pin and warm selectors scenario observes the path.write_text return value during models a. + path.write_text( + # What: arrange the exact models a fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact model a gguf fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact aliases a variant fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact models b fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact model b gguf fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact selectors public fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact strategy pin fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact targets a variant b fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact name public model fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact description stable local model fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact selectors public metadata fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact tier stable fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact type operator value fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact selectors available fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact strategy warm fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact targets a b fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact selectors hidden fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact strategy pin fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact targets a fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact unlisted true fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through """[models.a] before asserting its protocol or parser result. + """[models.a] +model = "a.gguf" +aliases = ["a:variant"] +[models.b] +model = "b.gguf" + +[selectors.public] +strategy = "pin" +targets = ["a:variant", "b"] +name = "Public Model" +description = "Stable local model" +[selectors.public.metadata] +tier = "stable" +type = "operator-value" + +[selectors.available] +strategy = "warm" +targets = ["a", "b"] + +[selectors.hidden] +strategy = "pin" +targets = ["a"] +unlisted = true +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog validates pin and warm selectors scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_validates_pin_and_warm_selectors groups the supplied clauses as one path.write_text call before its value is consumed. + ) + + # What: act by calling ModelCatalog.load and capture catalog; why: the catalog validates pin and warm selectors test asserts the response, state, or failure produced by this call. + catalog = ModelCatalog.load(str(path)) + + # What: assert that catalog selector public targets equals a variant b; why: this assertion protects the catalog validates pin and warm selectors regression after the test's arranged inputs and exercised call. + assert catalog.selector("public").targets == ("a:variant", "b") + # What: assert that catalog selector available strategy equals warm; why: this assertion protects the catalog validates pin and warm selectors regression after the test's arranged inputs and exercised call. + assert catalog.selector("available").strategy == "warm" + # What: assert the expected catalog public selectors == outcome; why: test catalog test catalog validates pin and warm selectors protects its regression by requiring this observable result after the exercised behavior. + assert catalog.public_selectors() == [ + # What: arrange name available strategy warm targets a b for the scenario; why: test catalog test catalog validates pin and warm selectors requires this concrete input or helper state before exercising the behavior under test. + {"name": "available", "strategy": "warm", "targets": ["a", "b"]}, + # What: arrange name hidden strategy pin targets a unlisted True for the scenario; why: test catalog test catalog validates pin and warm selectors requires this concrete input or helper state before exercising the behavior under test. + {"name": "hidden", "strategy": "pin", "targets": ["a"], "unlisted": True}, + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog validates pin and warm selectors requires this concrete input or helper state before exercising the behavior under test. + { + # What: arrange name public strategy pin targets a variant b for the scenario; why: test catalog test catalog validates pin and warm selectors requires this concrete input or helper state before exercising the behavior under test. + "name": "public", "strategy": "pin", "targets": ["a:variant", "b"], + # What: arrange displayName Public Model description Stable local model for the scenario; why: test catalog test catalog validates pin and warm selectors requires this concrete input or helper state before exercising the behavior under test. + "displayName": "Public Model", "description": "Stable local model", + # What: arrange metadata tier stable type operator value for the scenario; why: test catalog test catalog validates pin and warm selectors requires this concrete input or helper state before exercising the behavior under test. + "metadata": {"tier": "stable", "type": "operator-value"}, + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog validates pin and warm selectors requires this concrete input or helper state before exercising the behavior under test. + }, + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog validates pin and warm selectors requires this concrete input or helper state before exercising the behavior under test. + ] + # What: assert that catalog listed model ids equals a b available public; why: this assertion protects the catalog validates pin and warm selectors regression after the test's arranged inputs and exercised call. + assert catalog.listed_model_ids() == ("a", "b", "available", "public") + + +# What: define the test_catalog_validates_runtime_routing_profiles_and_selector_targets test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog validates runtime routing profiles and selector targets outcome. +def test_catalog_validates_runtime_routing_profiles_and_selector_targets(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog validates runtime routing profiles and selector targets test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with router and preload model and a and variant; why: the catalog validates runtime routing profiles and selector targets scenario observes the path.write_text return value during router. + path.write_text( + # What: arrange the exact router fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact preload model a variant fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact startup routing profile coding fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact models a fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact model a gguf fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact aliases a variant fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact selectors available fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact strategy warm fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact targets a fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact profiles coding fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact description coding mode fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact profiles coding pins fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact public available fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact direct a variant fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact disabled fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + """[router] +preload_model = "a:variant" +startup_routing_profile = "coding" + +[models.a] +model = "a.gguf" +aliases = ["a:variant"] + +[selectors.available] +strategy = "warm" +targets = ["a"] + +[profiles.coding] +description = "Coding mode" +[profiles.coding.pins] +public = "available" +direct = "a:variant" +disabled = "" +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog validates runtime routing profiles and selector targets scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_validates_runtime_routing_profiles_and_selector_targets groups the supplied clauses as one path.write_text call before its value is consumed. + ) + + # What: act by calling ModelCatalog.load and capture catalog; why: the catalog validates runtime routing profiles and selector targets test asserts the response, state, or failure produced by this call. + catalog = ModelCatalog.load(str(path)) + + # What: act by calling catalog.routing_profile and capture profile; why: the catalog validates runtime routing profiles and selector targets test asserts the response, state, or failure produced by this call. + profile = catalog.routing_profile("coding") + # What: assert that profile replacement public equals true available; why: this assertion protects the catalog validates runtime routing profiles and selector targets regression after the test's arranged inputs and exercised call. + assert profile.replacement("public") == (True, "available") + # What: assert that profile replacement disabled equals true; why: this assertion protects the catalog validates runtime routing profiles and selector targets regression after the test's arranged inputs and exercised call. + assert profile.replacement("disabled") == (True, None) + # What: assert that profile replacement other equals false; why: this assertion protects the catalog validates runtime routing profiles and selector targets regression after the test's arranged inputs and exercised call. + assert profile.replacement("other") == (False, None) + # What: assert that catalog settings preload model equals a; why: this assertion protects the catalog validates runtime routing profiles and selector targets regression after the test's arranged inputs and exercised call. + assert catalog.settings.preload_model == "a" + # What: assert that catalog settings startup routing profile equals coding; why: this assertion protects the catalog validates runtime routing profiles and selector targets regression after the test's arranged inputs and exercised call. + assert catalog.settings.startup_routing_profile == "coding" + # What: assert the expected catalog public routing profiles == outcome; why: test catalog test catalog validates runtime routing profiles and selector targets protects its regression by requiring this observable result after the exercised behavior. + assert catalog.public_routing_profiles() == [{ + # What: arrange name coding for the scenario; why: test catalog test catalog validates runtime routing profiles and selector targets requires this concrete input or helper state before exercising the behavior under test. + "name": "coding", + # What: arrange description Coding mode for the scenario; why: test catalog test catalog validates runtime routing profiles and selector targets requires this concrete input or helper state before exercising the behavior under test. + "description": "Coding mode", + # What: arrange pins direct a variant disabled None public available for the scenario; why: test catalog test catalog validates runtime routing profiles and selector targets requires this concrete input or helper state before exercising the behavior under test. + "pins": {"direct": "a:variant", "disabled": None, "public": "available"}, + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog validates runtime routing profiles and selector targets requires this concrete input or helper state before exercising the behavior under test. + }] + + +# What: parameterize test_catalog_rejects_invalid_runtime_routing_profiles with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects invalid runtime routing profiles. +@pytest.mark.parametrize("content,message", [ + # What: arrange the profiles empty npins n must contain at portion of the enclosing predicate; why: this clause remains in the catalog rejects invalid runtime routing profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('[profiles.empty]\npins = {}\n', "must contain at least one"), + # What: arrange the enclosing predicate collection with profiles and bad and pins and public and references; why: test_catalog_rejects_invalid_runtime_routing_profiles groups the supplied clauses as one test_catalog_rejects_invalid_runtime_routing_profiles expression collection before its. + ( + # What: arrange the profiles bad pins npublic missing n portion of the enclosing predicate; why: this clause remains in the catalog rejects invalid runtime routing profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + '[profiles.bad.pins]\npublic = "missing"\n', + # What: arrange the references unknown model portion of the enclosing predicate; why: this clause remains in the catalog rejects invalid runtime routing profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "references unknown model", + # What: arrange the enclosing predicate collection with profiles and bad and pins and public and references; why: test_catalog_rejects_invalid_runtime_routing_profiles groups the supplied clauses as one test_catalog_rejects_invalid_runtime_routing_profiles expression collection before its. + ), + # What: arrange the profiles bad pins npublic n model id or portion of the enclosing predicate; why: this clause remains in the catalog rejects invalid runtime routing profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('[profiles.bad.pins]\npublic = 7\n', "model ID or empty string"), + # What: arrange the profiles bad name pins npublic a portion of the enclosing predicate; why: this clause remains in the catalog rejects invalid runtime routing profiles scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('[profiles."bad/name".pins]\npublic = "a"\n', "profile name"), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_catalog_rejects_invalid_runtime_routing_profiles groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_catalog_rejects_invalid_runtime_routing_profiles test around tmp path and content and message; why: this test groups the arrange, act, and assertions that protect the catalog rejects invalid runtime routing profiles outcome. +def test_catalog_rejects_invalid_runtime_routing_profiles(tmp_path, content, message): + # What: arrange path as tmp path and models and toml; why: the catalog rejects invalid runtime routing profiles test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text models a nmodel a gguf n content fixture fragment; why: the catalog rejects invalid runtime routing profiles scenario feeds this byte-preserved fragment through path.write_text('[models.a]\nmodel = "a.gguf"\n' + content, encoding="ut before asserting its protocol or parser. + path.write_text('[models.a]\nmodel = "a.gguf"\n' + content, encoding="utf-8") + # What: assert the pytest.raises failure context; why: the catalog rejects invalid runtime routing profiles scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match=message): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects invalid runtime routing profiles scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: parameterize test_catalog_rejects_unknown_startup_targets with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects unknown startup targets. +@pytest.mark.parametrize("setting,message", [ + # What: arrange the preload model missing unknown model profile portion of the enclosing predicate; why: this clause remains in the catalog rejects unknown startup targets scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('preload_model = "missing"', "unknown model profile"), + # What: arrange the startup routing profile missing unknown profile portion of the enclosing predicate; why: this clause remains in the catalog rejects unknown startup targets scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('startup_routing_profile = "missing"', "unknown profile"), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_catalog_rejects_unknown_startup_targets groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_catalog_rejects_unknown_startup_targets test around tmp path and setting and message; why: this test groups the arrange, act, and assertions that protect the catalog rejects unknown startup targets outcome. +def test_catalog_rejects_unknown_startup_targets(tmp_path, setting, message): + # What: arrange path as tmp path and models and toml; why: the catalog rejects unknown startup targets test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text f router n setting n fixture fragment; why: the catalog rejects unknown startup targets scenario feeds this byte-preserved fragment through path.write_text(f"[router]\n{setting}\n[models.a]\nmodel='a.gguf'\n", en before asserting its protocol or parser result. + path.write_text(f"[router]\n{setting}\n[models.a]\nmodel='a.gguf'\n", encoding="utf-8") + # What: assert the pytest.raises failure context; why: the catalog rejects unknown startup targets scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match=message): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects unknown startup targets scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: parameterize test_catalog_rejects_unsupported_or_ambiguous_selectors with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects unsupported or ambiguous selectors. +@pytest.mark.parametrize("content,message", [ + # What: arrange the grouped source fragment for the scenario; why: test catalog rejects unsupported or ambiguous selectors requires this concrete input or helper state before exercising the behavior under test. + ( + # What: arrange the selectors bad nstrategy spillover ntargets a n portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or ambiguous selectors scenario\'s enclosing expression so its grouping and evaluation order stay intact. + '[selectors.bad]\nstrategy = "spillover"\ntargets = ["a"]\n', + # What: arrange the requires multi resident or peer capacity portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or ambiguous selectors scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "requires multi-resident or peer capacity", + # What: arrange the grouped source fragment for the scenario; why: test catalog rejects unsupported or ambiguous selectors requires this concrete input or helper state before exercising the behavior under test. + ), + # What: arrange the selectors bad nstrategy random ntargets a n portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or ambiguous selectors scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('[selectors.bad]\nstrategy = "random"\ntargets = ["a"]\n', "pin or warm"), + # What: arrange the selectors bad nstrategy pin ntargets n to portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or ambiguous selectors scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('[selectors.bad]\nstrategy = "pin"\ntargets = []\n', "1 to 64"), + # What: arrange the enclosing predicate collection with selectors and bad and strategy and pin and json; why: test_catalog_rejects_unsupported_or_ambiguous_selectors groups the supplied clauses as one test_catalog_rejects_unsupported_or_ambiguous_selectors expression collection before its. + ( + # What: arrange the exact selectors bad nstrategy pin ntargets a n fixture fragment; why: the catalog rejects unsupported or ambiguous selectors scenario feeds this byte-preserved fragment through '[selectors.bad]\nstrategy = "pin"\ntargets = ["a"]\n' before asserting its protocol or parser result. + # What: arrange the exact selectors bad metadata ncreated n fixture fragment; why: the catalog rejects unsupported or ambiguous selectors scenario feeds this byte-preserved fragment through '[selectors.bad]\nstrategy = "pin"\ntargets = ["a"]\n' before asserting its protocol or parser result. + '[selectors.bad]\nstrategy = "pin"\ntargets = ["a"]\n' + '[selectors.bad.metadata]\ncreated = 2026-09-14\n', + # What: arrange the json compatible portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or ambiguous selectors scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "JSON-compatible", + # What: arrange the enclosing predicate collection with selectors and bad and strategy and pin and json; why: test_catalog_rejects_unsupported_or_ambiguous_selectors groups the supplied clauses as one test_catalog_rejects_unsupported_or_ambiguous_selectors expression collection before its. + ), + # What: arrange the selectors bad nstrategy pin ntargets missing n portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or ambiguous selectors scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('[selectors.bad]\nstrategy = "pin"\ntargets = ["missing"]\n', "not a configured"), + # What: arrange the selectors a nstrategy pin ntargets a n portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or ambiguous selectors scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('[selectors.a]\nstrategy = "pin"\ntargets = ["a"]\n', "conflicts"), + # What: arrange the grouped source fragment for the scenario; why: test catalog rejects unsupported or ambiguous selectors requires this concrete input or helper state before exercising the behavior under test. + ( + # What: arrange the exact selectors first nstrategy pin ntargets second n fixture fragment; why: the catalog rejects unsupported or ambiguous selectors scenario feeds this byte-preserved fragment through '[selectors.first]\nstrategy = "pin"\ntargets = ["second"]\n' before asserting its protocol or parser resul. + # What: arrange the exact selectors second nstrategy warm ntargets a n fixture fragment; why: the catalog rejects unsupported or ambiguous selectors scenario feeds this byte-preserved fragment through '[selectors.first]\nstrategy = "pin"\ntargets = ["second"]\n' before asserting its protocol or parser result. + '[selectors.first]\nstrategy = "pin"\ntargets = ["second"]\n' + '[selectors.second]\nstrategy = "warm"\ntargets = ["a"]\n', + # What: arrange the cannot reference another selector portion of the enclosing predicate; why: this clause remains in the catalog rejects unsupported or ambiguous selectors scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "cannot reference another selector", + # What: arrange the grouped source fragment for the scenario; why: test catalog rejects unsupported or ambiguous selectors requires this concrete input or helper state before exercising the behavior under test. + ), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_catalog_rejects_unsupported_or_ambiguous_selectors groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_catalog_rejects_unsupported_or_ambiguous_selectors test around tmp path and content and message; why: this test groups the arrange, act, and assertions that protect the catalog rejects unsupported or ambiguous selectors outcome. +def test_catalog_rejects_unsupported_or_ambiguous_selectors( + # What: arrange the tmp path input for test_catalog_rejects_unsupported_or_ambiguous_selectors; why: test_catalog_rejects_unsupported_or_ambiguous_selectors consumes tmp path during path tmp path models toml, so callers must bind it with the other signature inputs. + tmp_path, content, message +# What: arrange the grouped source fragment for the scenario; why: test catalog rejects unsupported or ambiguous selectors requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange path as tmp path and models and toml; why: the catalog rejects unsupported or ambiguous selectors test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text models a nmodel a gguf n content fixture fragment; why: the catalog rejects unsupported or ambiguous selectors scenario feeds this byte-preserved fragment through path.write_text('[models.a]\nmodel = "a.gguf"\n' + content, encoding="ut before asserting its protocol or pars. + path.write_text('[models.a]\nmodel = "a.gguf"\n' + content, encoding="utf-8") + # What: assert the pytest.raises failure context; why: the catalog rejects unsupported or ambiguous selectors scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match=message): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects unsupported or ambiguous selectors scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: define the test_catalog_supports_namespaced_model_ids_and_longest_upstream_prefix test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog supports namespaced model ids and longest upstream prefix outcome. +def test_catalog_supports_namespaced_model_ids_and_longest_upstream_prefix(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog supports namespaced model ids and longest upstream prefix test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with router and include aliases in list and true and models; why: the catalog supports namespaced model ids and longest upstream prefix scenario observes the path.write_text return value during router. + path.write_text( + # What: arrange the exact router fixture fragment; why: the catalog supports namespaced model ids and longest upstream prefix scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact include aliases in list true fixture fragment; why: the catalog supports namespaced model ids and longest upstream prefix scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact models author fixture fragment; why: the catalog supports namespaced model ids and longest upstream prefix scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact model parent gguf fixture fragment; why: the catalog supports namespaced model ids and longest upstream prefix scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact models author model fixture fragment; why: the catalog supports namespaced model ids and longest upstream prefix scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact model exact gguf fixture fragment; why: the catalog supports namespaced model ids and longest upstream prefix scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact aliases org compat fixture fragment; why: the catalog supports namespaced model ids and longest upstream prefix scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the catalog supports namespaced model ids and longest upstream prefix scenario feeds this byte-preserved fragment through """[router] before asserting its protocol or parser result. + """[router] +include_aliases_in_list = true + +[models.author] +model = "parent.gguf" + +[models."author/model"] +model = "exact.gguf" +aliases = ["org/compat"] +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog supports namespaced model ids and longest upstream prefix scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_supports_namespaced_model_ids_and_longest_upstream_prefix groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: act by calling ModelCatalog.load and capture catalog; why: the catalog supports namespaced model ids and longest upstream prefix test asserts the response, state, or failure produced by this call. + catalog = ModelCatalog.load(str(path)) + + # What: assert that catalog get org compat name equals author model; why: this assertion protects the catalog supports namespaced model ids and longest upstream prefix regression after the test's arranged inputs and exercised call. + assert catalog.get("org/compat").name == "author/model" + # What: assert that catalog listed model ids equals author author model org compat; why: this assertion protects the catalog supports namespaced model ids and longest upstream prefix regression after the test's arranged inputs and exercised call. + assert catalog.listed_model_ids() == ("author", "author/model", "org/compat") + # What: act by evaluating requested profile remaining catalog resolve upstream path author model api x y; why: test catalog test captures the behavior or response that its following assertions inspect. + requested, profile, remaining = catalog.resolve_upstream_path("author/model/api/x/y") + # What: assert the expected requested profile name remaining == outcome; why: test catalog test catalog supports namespaced model ids and longest upstream prefix protects its regression by requiring this observable result after the exercised behavior. + assert (requested, profile.name, remaining) == ( + # What: arrange author model author model api x y for the scenario; why: test catalog test catalog supports namespaced model ids and longest upstream prefix requires this concrete input or helper state before exercising the behavior under test. + "author/model", "author/model", "/api/x/y", + # What: arrange the grouped source fragment for the scenario; why: test catalog test catalog supports namespaced model ids and longest upstream prefix requires this concrete input or helper state before exercising the behavior under test. + ) + # What: act by evaluating requested profile remaining catalog resolve upstream path org compat; why: test catalog test catalog supports namespaced model ids and longest upstream prefix captures the behavior or response that its following assertions inspect. + requested, profile, remaining = catalog.resolve_upstream_path("org/compat") + # What: assert that requested profile name remaining equals org compat author model; why: this assertion protects the catalog supports namespaced model ids and longest upstream prefix regression after the test's arranged inputs and exercised call. + assert (requested, profile.name, remaining) == ("org/compat", "author/model", "/") + # What: assert the pytest.raises failure context; why: the catalog supports namespaced model ids and longest upstream prefix scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match="does not begin"): + # What: arrange the exact catalog resolve upstream path missing model v1 chat fixture fragment; why: the catalog supports namespaced model ids and longest upstream prefix scenario feeds this byte-preserved fragment through catalog.resolve_upstream_path("missing/model/v1/chat") before asserting its protocol or. + catalog.resolve_upstream_path("missing/model/v1/chat") + + +# What: define the test_readiness_waits_for_engine_health_not_just_a_listening_process test around local fixtures; why: this test groups the arrange, act, and assertions that protect the readiness waits for engine health not just a listening process outcome. +def test_readiness_waits_for_engine_health_not_just_a_listening_process(): + # What: define Manager as the owner of status; why: daemon callers use this class boundary so those methods share one manager state invariant. + class Manager: + # What: define the status test helper around captured fixture state; why: the readiness waits for engine health not just a listening process scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def status(self): + # What: arrange the running field as true; why: Manager.status carries running into return {"running": True, "pid": 44}. + return {"running": True, "pid": 44} + + # What: define Probe as the owner of __init__ and fresh_health; why: daemon callers use this class boundary so those methods share one probe state invariant. + class Probe: + # What: define the __init__ test helper around captured fixture state; why: the readiness waits for engine health not just a listening process scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def __init__(self): + # What: act by calling iter and capture docs; why: the readiness waits for engine health not just a listening process test asserts the response, state, or failure produced by this call. + self.docs = iter([ + # What: arrange the reachable field as true; why: Probe.__init__ carries reachable through docs into the enclosing return or state update. + {"reachable": True, "status": "loading"}, + # What: arrange the reachable field as true; why: Probe.__init__ carries reachable through docs into the enclosing return or state update. + {"reachable": True, "status": "ok", "model": "m"}, + # What: arrange the iter call with ordered positional inputs; why: Probe.__init__ groups the supplied clauses as one iter call before its value is consumed. + ]) + + # What: arrange the def fresh health self port test helper boundary; why: test catalog test readiness waits for engine health not just a listening process uses this local double to isolate the behavior checked by its assertions. + def fresh_health(self, port): + # What: assert that port equals 1922; why: this assertion protects the readiness waits for engine health not just a listening process regression after the test's arranged inputs and exercised call. + assert port == 1922 + # What: return next and docs from the fresh_health test helper; why: the readiness waits for engine health not just a listening process scenario uses this helper result in its subsequent act or assertion. + return next(self.docs) + + # What: act by calling iter and capture clock; why: the readiness waits for engine health not just a listening process test asserts the response, state, or failure produced by this call. + clock = iter([0.0, 0.0, 0.1, 0.1]) + # What: act by calling wait_for_ready and capture result; why: the readiness waits for engine health not just a listening process test asserts the response, state, or failure produced by this call. + result = wait_for_ready(Manager(), Probe(), pid=44, port=1922, timeout_s=1, now=lambda: next(clock), sleep=lambda _: None) + # What: assert that result equals ready true health reachable true status; why: this assertion protects the readiness waits for engine health not just a listening process regression after the test's arranged inputs and exercised call. + assert result == {"ready": True, "health": {"reachable": True, "status": "ok", "model": "m"}} + + +# What: define the test_readiness_timeout_leaves_the_existing_engine_under_manager_control test around local fixtures; why: this test groups the arrange, act, and assertions that protect the readiness timeout leaves the existing engine under manager control outcome. +def test_readiness_timeout_leaves_the_existing_engine_under_manager_control(): + # What: define Manager as the owner of status; why: daemon callers use this class boundary so those methods share one manager state invariant. + class Manager: + # What: define the status test helper around captured fixture state; why: the readiness timeout leaves the existing engine under manager control scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def status(self): + # What: arrange the running field as true; why: Manager.status carries running into return {"running": True, "pid": 44}. + return {"running": True, "pid": 44} + + # What: define Probe as the owner of fresh_health; why: daemon callers use this class boundary so those methods share one probe state invariant. + class Probe: + # What: arrange the def fresh health self port test helper boundary; why: test catalog test readiness timeout leaves the existing engine under manager control uses this local double to isolate the behavior checked by its assertions. + def fresh_health(self, port): + # What: arrange the helper response as reachable True status loading; why: test catalog test readiness timeout leaves the existing engine under manager control feeds this result into the behavior whose outcome is asserted. + return {"reachable": True, "status": "loading"} + + # What: act by calling iter and capture clock; why: the readiness timeout leaves the existing engine under manager control test asserts the response, state, or failure produced by this call. + clock = iter([0.0, 0.0, 1.0]) + # What: act by calling wait_for_ready and capture result; why: the readiness timeout leaves the existing engine under manager control test asserts the response, state, or failure produced by this call. + result = wait_for_ready(Manager(), Probe(), pid=44, port=1922, timeout_s=1, now=lambda: next(clock), sleep=lambda _: None) + # What: assert the expected result == outcome; why: test catalog test readiness timeout leaves the existing engine under manager control protects its regression by requiring this observable result after the exercised behavior. + assert result == { + # What: arrange ready False for the scenario; why: test catalog test readiness timeout leaves the existing engine under manager control requires this concrete input or helper state before exercising the behavior under test. + "ready": False, + # What: arrange reason timeout for the scenario; why: test catalog test readiness timeout leaves the existing engine under manager control requires this concrete input or helper state before exercising the behavior under test. + "reason": "timeout", + # What: arrange health reachable True status loading for the scenario; why: test catalog test readiness timeout leaves the existing engine under manager control requires this concrete input or helper state before exercising the behavior under test. + "health": {"reachable": True, "status": "loading"}, + # What: arrange the grouped source fragment for the scenario; why: test catalog test readiness timeout leaves the existing engine under manager control requires this concrete input or helper state before exercising the behavior under test. + } + + +# What: define the test_profile_api_uses_validated_catalog_and_existing_switch_transaction test around tmp path; why: this test groups the arrange, act, and assertions that protect the profile api uses validated catalog and existing switch transaction outcome. +def test_profile_api_uses_validated_catalog_and_existing_switch_transaction(tmp_path): + # What: arrange path as tmp path and models and toml; why: the profile api uses validated catalog and existing switch transaction test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text models coding nmodel models coding gguf nport fixture fragment; why: the profile api uses validated catalog and existing switch transaction scenario feeds this byte-preserved fragment through path.write_text("[models.coding]\nmodel = '/models/coding.gguf'\nport = before as. + path.write_text("[models.coding]\nmodel = '/models/coding.gguf'\nport = 1922\ncheck_endpoint = '/ready'\nargs = ['--max-seq-len-override', '32768']\n", encoding="utf-8") + + # What: define Manager as the owner of __init__ and status and start and switch and switch_for_readiness; why: daemon callers use this class boundary so those methods share one manager state invariant. + class Manager: + # What: define the __init__ test helper around captured fixture state; why: the profile api uses validated catalog and existing switch transaction scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def __init__(self): + # What: arrange calls as the fixture input; why: the profile api uses validated catalog and existing switch transaction test consumes this named precondition before exercising the behavior. + self.calls = [] + # What: arrange running as false; why: the profile api uses validated catalog and existing switch transaction test consumes this named precondition before exercising the behavior. + self.running = False + + # What: define the status test helper around captured fixture state; why: the profile api uses validated catalog and existing switch transaction scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def status(self): + # What: arrange the running field as running; why: Manager.status carries running into return {"running": self.running, "pid": 101 if self.running else None, ". + return {"running": self.running, "pid": 101 if self.running else None, "port": 1922 if self.running else None} + + # What: define the start test helper around model and port and args; why: the profile api uses validated catalog and existing switch transaction scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def start(self, model, port, args): + # What: arrange the exact self calls append start model port args fixture fragment; why: the profile api uses validated catalog and existing switch transaction scenario feeds this byte-preserved fragment through self.calls.append(("start", model, port, args)) before asserting its protocol or parser result. + self.calls.append(("start", model, port, args)) + # What: arrange self running True for the scenario; why: test catalog test profile api uses validated catalog and existing switch transaction requires this concrete input or helper. + self.running = True + # What: arrange the started field as true; why: Manager.start carries started into return {"started": True, "model": model, "port": port}. + return {"started": True, "model": model, "port": port} + + # What: define the switch test helper around model and port and args and force; why: the profile api uses validated catalog and existing switch transaction scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def switch(self, model, port, args, force): + # What: arrange the exact self calls append switch model port args force fixture fragment; why: the profile api uses validated catalog and existing switch transaction scenario feeds this byte-preserved fragment through self.calls.append(("switch", model, port, args, force)) before asserting its protocol or. + self.calls.append(("switch", model, port, args, force)) + # What: arrange self running True for the scenario; why: test catalog test profile api uses validated catalog and existing switch transaction requires this concrete input or helper. + self.running = True + # What: arrange the switched field as true; why: Manager.switch carries switched into return {"switched": True, "model": model, "port": port}. + return {"switched": True, "model": model, "port": port} + + # What: define the switch_for_readiness test helper around captured fixture state; why: the profile api uses validated catalog and existing switch transaction scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def switch_for_readiness(self, *args): + # What: return switch and args from the switch_for_readiness test helper; why: the profile api uses validated catalog and existing switch transaction scenario uses this helper result in its subsequent act or assertion. + return self.switch(*args), None + + # What: define Probe as the owner of fresh_readiness; why: daemon callers use this class boundary so those methods share one probe state invariant. + class Probe: + # What: define the fresh_readiness test helper around port and target; why: the profile api uses validated catalog and existing switch transaction scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def fresh_readiness(self, port, target): + # What: assert that target equals ready; why: this assertion protects the profile api uses validated catalog and existing switch transaction regression after the test's arranged inputs and exercised call. + assert target == "/ready" + # What: arrange the helper response as reachable True ready True port port; why: test catalog test profile api uses validated catalog and existing switch transaction feeds this result into the behavior whose outcome is asserted. + return {"reachable": True, "ready": True, "port": port} + + # What: act by calling Manager and capture manager; why: the profile api uses validated catalog and existing switch transaction test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_profile_api_uses_validated_catalog_and_existing_switch_transaction releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the profile api uses validated catalog and existing switch transaction test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_profile_api_uses_validated_catalog_and_existing_switch_transaction; why: test_profile_api_uses_validated_catalog_and_existing_switch_transaction consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=Probe(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to ModelCatalog.load; why: the profile api uses validated catalog and existing switch transaction scenario binds this lifecycle value to ModelCatalog.load's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=ModelCatalog.load(str(path)), token="secret", + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_profile_api_uses_validated_catalog_and_existing_switch_transaction groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the profile api uses validated catalog and existing switch transaction test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: assert that client get router profiles status code equals 401; why: this assertion protects the profile api uses validated catalog and existing switch transaction regression after the test's arranged inputs and exercised call. + assert client.get("/router/profiles").status_code == 401 + # What: act by calling client.get and capture listing; why: the profile api uses validated catalog and existing switch transaction test asserts the response, state, or failure produced by this call. + listing = client.get("/router/profiles", headers={"X-FT-Token": "secret"}) + # What: assert that listing status code equals 200; why: this assertion protects the profile api uses validated catalog and existing switch transaction regression after the test's arranged inputs and exercised call. + assert listing.status_code == 200 + # What: assert that listing json data 0 name equals coding; why: this assertion protects the profile api uses validated catalog and existing switch transaction regression after the test's arranged inputs and exercised call. + assert listing.json()["data"][0]["name"] == "coding" + # What: act by calling client.get and capture public listing; why: the profile api uses validated catalog and existing switch transaction test asserts the response, state, or failure produced by this call. + public_listing = client.get("/models") + # What: assert that public listing status code equals 200; why: this assertion protects the profile api uses validated catalog and existing switch transaction regression after the test's arranged inputs and exercised call. + assert public_listing.status_code == 200 + # What: assert that public listing json data 0 id equals coding; why: this assertion protects the profile api uses validated catalog and existing switch transaction regression after the test's arranged inputs and exercised call. + assert public_listing.json()["data"][0]["id"] == "coding" + # What: assert that models coding gguf is absent from public listing text; why: this assertion protects the profile api uses validated catalog and existing switch transaction regression after the test's arranged inputs and exercised call. + assert "/models/coding.gguf" not in public_listing.text + # What: act by calling client.post and capture started; why: the profile api uses validated catalog and existing switch transaction test asserts the response, state, or failure produced by this call. + started = client.post("/engine/start-profile", json={"name": "coding"}, headers={"X-FT-Token": "secret"}) + # What: assert that started status code equals 200; why: this assertion protects the profile api uses validated catalog and existing switch transaction regression after the test's arranged inputs and exercised call. + assert started.status_code == 200 + # What: assert that started json profile equals coding; why: this assertion protects the profile api uses validated catalog and existing switch transaction regression after the test's arranged inputs and exercised call. + assert started.json()["profile"] == "coding" + # What: assert that started json readiness ready is true; why: this assertion protects the profile api uses validated catalog and existing switch transaction regression after the test's arranged inputs and exercised call. + assert started.json()["readiness"]["ready"] is True + # What: act by calling client.post and capture switched; why: the profile api uses validated catalog and existing switch transaction test asserts the response, state, or failure produced by this call. + switched = client.post("/engine/switch-profile", json={"name": "coding", "force": True}, headers={"X-FT-Token": "secret"}) + # What: assert that switched status code equals 200; why: this assertion protects the profile api uses validated catalog and existing switch transaction regression after the test's arranged inputs and exercised call. + assert switched.status_code == 200 + # What: assert the expected manager calls == outcome; why: test catalog test profile api uses validated catalog and existing switch transaction protects its regression by requiring this observable result after the exercised behavior. + assert manager.calls == [ + # What: arrange start models coding gguf 1922 max seq len override 32768 for the scenario; why: test catalog test profile api uses validated catalog and existing switch transaction requires this concrete input or helper state before exercising the behavior under test. + ("start", "/models/coding.gguf", 1922, ["--max-seq-len-override", "32768"]), + # What: arrange switch models coding gguf 1922 max seq len override 32768 True for the scenario; why: test catalog test profile api uses validated catalog and existing switch transaction requires this concrete input or helper state before exercising the behavior under test. + ("switch", "/models/coding.gguf", 1922, ["--max-seq-len-override", "32768"], True), + # What: arrange the grouped source fragment for the scenario; why: test catalog test profile api uses validated catalog and existing switch transaction requires this concrete input or helper state before exercising the behavior under test. + ] + + +# What: define the test_client_shutdown_uses_the_daemon_shutdown_transaction test around monkeypatch and capsys; why: this test groups the arrange, act, and assertions that protect the client shutdown uses the daemon shutdown transaction outcome. +def test_client_shutdown_uses_the_daemon_shutdown_transaction(monkeypatch, capsys): + # What: arrange seen as the fixture input; why: the client shutdown uses the daemon shutdown transaction test consumes this named precondition before exercising the behavior. + seen = {} + + # What: define the request test helper around method and url and path; why: the client shutdown uses the daemon shutdown transaction scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def request(method, url, path, **kwargs): + # What: arrange method to seen.update; why: the client shutdown uses the daemon shutdown transaction scenario binds this method value to seen.update's method input. + seen.update(method=method, url=url, path=path, **kwargs) + # What: arrange the stopping field as true; why: request carries stopping into return {"stopping": True}. + return {"stopping": True} + + # What: arrange the exact monkeypatch setattr daemon client request json request fixture fragment; why: the client shutdown uses the daemon shutdown transaction scenario feeds this byte-preserved fragment through monkeypatch.setattr(daemon_client, "_request_json", request) before asserting its protocol or parser r. + monkeypatch.setattr(daemon_client, "_request_json", request) + # What: assert that daemon client main shutdown url http daemon 1900 equals 0; why: this assertion protects the client shutdown uses the daemon shutdown transaction regression after the test's arranged inputs and exercised call. + assert daemon_client.main(["shutdown", "--url", "http://daemon:1900", "--force"]) == 0 + # What: assert the expected seen == outcome; why: test catalog test client shutdown uses the daemon shutdown transaction protects its regression by requiring this observable result after the exercised behavior. + assert seen == { + # What: arrange method POST url http daemon 1900 path shutdown for the scenario; why: test catalog test client shutdown uses the daemon shutdown transaction requires this concrete input or helper state before exercising the behavior under test. + "method": "POST", "url": "http://daemon:1900", "path": "/shutdown", + # What: arrange body force True token None timeout daemon client DEFAULT LIFECYCLE TIMEOUT for the scenario; why: test catalog test client shutdown uses the daemon shutdown transaction requires this concrete input or helper state before exercising the behavior under test. + "body": {"force": True}, "token": None, "timeout": daemon_client.DEFAULT_LIFECYCLE_TIMEOUT, + # What: arrange the grouped source fragment for the scenario; why: test catalog test client shutdown uses the daemon shutdown transaction requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that stopping true is present in capsys readouterr out; why: this assertion protects the client shutdown uses the daemon shutdown transaction regression after the test's arranged inputs and exercised call. + assert '"stopping": true' in capsys.readouterr().out + + +# What: define the test_client_models_uses_authenticated_profile_control_route test around monkeypatch and capsys; why: this test groups the arrange, act, and assertions that protect the client models uses authenticated profile control route outcome. +def test_client_models_uses_authenticated_profile_control_route(monkeypatch, capsys): + # What: arrange seen as the fixture input; why: the client models uses authenticated profile control route test consumes this named precondition before exercising the behavior. + seen = {} + + # What: define the request test helper around method and url and path; why: the client models uses authenticated profile control route scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def request(method, url, path, **kwargs): + # What: arrange method to seen.update; why: the client models uses authenticated profile control route scenario binds this method value to seen.update's method input. + seen.update(method=method, url=url, path=path, **kwargs) + # What: arrange the data field as name and coding; why: request carries data into return {"data": [{"name": "coding"}]}. + return {"data": [{"name": "coding"}]} + + # What: arrange the exact monkeypatch setattr daemon client request json request fixture fragment; why: the client models uses authenticated profile control route scenario feeds this byte-preserved fragment through monkeypatch.setattr(daemon_client, "_request_json", request) before asserting its protocol or parser. + monkeypatch.setattr(daemon_client, "_request_json", request) + # What: assert the expected daemon client main outcome; why: test catalog test client models uses authenticated profile control route protects its regression by requiring this observable result after the exercised behavior. + assert daemon_client.main([ + # What: arrange models url http daemon 1900 token control secret for the scenario; why: test catalog test client models uses authenticated profile control route requires this concrete input or helper state before exercising the behavior under test. + "models", "--url", "http://daemon:1900", "--token", "control-secret" + # What: arrange == 0 for the scenario; why: test catalog test client models uses authenticated profile control route requires this concrete input or helper state before exercising the behavior under test. + ]) == 0 + # What: assert the expected seen == outcome; why: test catalog test client models uses authenticated profile control route protects its regression by requiring this observable result after the exercised behavior. + assert seen == { + # What: arrange method GET url http daemon 1900 path router profiles for the scenario; why: test catalog test client models uses authenticated profile control route requires this concrete input or helper state before exercising the behavior under test. + "method": "GET", "url": "http://daemon:1900", "path": "/router/profiles", + # What: arrange body None token control secret timeout daemon client DEFAULT TIMEOUT for the scenario; why: test catalog test client models uses authenticated profile control route requires this concrete input or helper state before exercising the behavior under test. + "body": None, "token": "control-secret", "timeout": daemon_client.DEFAULT_TIMEOUT, + # What: arrange the grouped source fragment for the scenario; why: test catalog test client models uses authenticated profile control route requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that name coding is present in capsys readouterr out; why: this assertion protects the client models uses authenticated profile control route regression after the test's arranged inputs and exercised call. + assert '"name": "coding"' in capsys.readouterr().out + + +# What: define the test_readiness_supports_a_validated_non_health_endpoint test around local fixtures; why: this test groups the arrange, act, and assertions that protect the readiness supports a validated non health endpoint outcome. +def test_readiness_supports_a_validated_non_health_endpoint(): + # What: define Manager as the owner of status; why: daemon callers use this class boundary so those methods share one manager state invariant. + class Manager: + # What: define the status test helper around captured fixture state; why: the readiness supports a validated non health endpoint scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def status(self): + # What: arrange the running field as true; why: Manager.status carries running into return {"running": True, "pid": 44}. + return {"running": True, "pid": 44} + + # What: define Probe as the owner of fresh_readiness; why: daemon callers use this class boundary so those methods share one probe state invariant. + class Probe: + # What: define the fresh_readiness test helper around port and path; why: the readiness supports a validated non health endpoint scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def fresh_readiness(self, port, path): + # What: assert that port path equals 1922 ready; why: this assertion protects the readiness supports a validated non health endpoint regression after the test's arranged inputs and exercised call. + assert (port, path) == (1922, "/ready") + # What: arrange the reachable field as true; why: Probe.fresh_readiness carries reachable into return {"reachable": True, "ready": True}. + return {"reachable": True, "ready": True} + + # What: act by calling wait_for_ready and capture result; why: the readiness supports a validated non health endpoint test asserts the response, state, or failure produced by this call. + result = wait_for_ready( + # What: arrange pid to Manager; why: the readiness supports a validated non health endpoint scenario binds this 44 value to Manager's pid input. + Manager(), Probe(), pid=44, port=1922, timeout_s=1, path="/ready" + # What: arrange the wait_for_ready call with pid and port and timeout s and path; why: test_readiness_supports_a_validated_non_health_endpoint groups the supplied clauses as one wait_for_ready call before its value is consumed. + ) + # What: assert that result equals ready true health reachable true ready; why: this assertion protects the readiness supports a validated non health endpoint regression after the test's arranged inputs and exercised call. + assert result == {"ready": True, "health": {"reachable": True, "ready": True}} + + +# What: define the test_router_policy_is_strict_and_public_model_fields_are_safe test around tmp path; why: this test groups the arrange, act, and assertions that protect the router policy is strict and public model fields are safe outcome. +def test_router_policy_is_strict_and_public_model_fields_are_safe(tmp_path): + # What: arrange path as tmp path and models and toml; why: the router policy is strict and public model fields are safe test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact router fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact api keys one two fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact default ttl s fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange path write text for the scenario; why: test catalog test router policy is strict and public model fields are safe requires this concrete input or helper state before exercising the behavior under test. + # What: arrange the exact upstream timeout s fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact scheduler fifo fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact global concurrency limit fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact send loading state true fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact router groups interactive fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact members coding chat fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact swap true fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact exclusive true fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact models coding fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact model coding gguf fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact ttl s fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange path write text for the scenario; why: test catalog test router policy is strict and public model fields are safe requires this concrete input or helper state before exercising the behavior under test. + # What: arrange path write text for the scenario; why: test catalog test router policy is strict and public model fields are safe requires this concrete input or helper state before exercising the behavior under test. + # What: arrange the exact concurrency limit fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact send loading state false fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact group interactive fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact models chat fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact model chat gguf fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange path write text for the scenario; why: test catalog test router policy is strict and public model fields are safe requires this concrete input or helper state before exercising the behavior under test. + # What: arrange the exact encoding utf 8 fixture fragment; why: the router policy is strict and public model fields are safe scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + path.write_text(""" +[router] +api_keys = ["one", "two"] +default_ttl_s = 300 +unload_timeout_s = 45 +upstream_timeout_s = 42 +scheduler = "fifo" +global_concurrency_limit = 4 +send_loading_state = true + +[router.groups.interactive] +members = ["coding", "chat"] +swap = true +exclusive = true + +[models.coding] +model = "coding.gguf" +ttl_s = 0 +unload_timeout_s = 60 +priority = 10 +concurrency_limit = 2 +send_loading_state = false +group = "interactive" + +[models.chat] +model = "chat.gguf" +priority = -5 +""", encoding="utf-8") + # What: act by calling ModelCatalog.load and capture catalog; why: the router policy is strict and public model fields are safe test asserts the response, state, or failure produced by this call. + catalog = ModelCatalog.load(str(path)) + # What: assert that catalog settings api keys equals one two; why: this assertion protects the router policy is strict and public model fields are safe regression after the test's arranged inputs and exercised call. + assert catalog.settings.api_keys == ("one", "two") + # What: assert that catalog settings default ttl s equals 300; why: this assertion protects the router policy is strict and public model fields are safe regression after the test's arranged inputs and exercised call. + assert catalog.settings.default_ttl_s == 300 + # What: assert that catalog settings upstream timeout s equals 42; why: this assertion protects the router policy is strict and public model fields are safe regression after the test's arranged inputs and exercised call. + assert catalog.settings.upstream_timeout_s == 42 + # What: assert that catalog settings global concurrency limit equals 4; why: this assertion protects the router policy is strict and public model fields are safe regression after the test's arranged inputs and exercised call. + assert catalog.settings.global_concurrency_limit == 4 + # What: assert that catalog settings send loading state is true; why: this assertion protects the router policy is strict and public model fields are safe regression after the test's arranged inputs and exercised call. + assert catalog.settings.send_loading_state is True + # What: assert that catalog settings groups 0 members equals coding chat; why: this assertion protects the router policy is strict and public model fields are safe regression after the test's arranged inputs and exercised call. + assert catalog.settings.groups[0].members == ("coding", "chat") + # What: act by calling catalog.public and capture public; why: the router policy is strict and public model fields are safe test asserts the response, state, or failure produced by this call. + public = {item["name"]: item for item in catalog.public()} + # What: assert the expected public coding == outcome; why: test catalog test router policy is strict and public model fields are safe protects its regression by requiring this observable result after the exercised behavior. + assert public["coding"] == { + # What: arrange name coding model coding gguf args readyTimeoutS 120.0 for the scenario; why: test catalog test router policy is strict and public model fields are safe requires this concrete input or helper state before exercising the behavior under test. + "name": "coding", "model": "coding.gguf", "args": [], "readyTimeoutS": 120.0, + # What: arrange ttlS 0.0 unloadTimeoutS 60.0 priority 10 for the scenario; why: test catalog test router policy is strict and public model fields are safe requires this concrete input or helper state before exercising the behavior under test. + "ttlS": 0.0, "unloadTimeoutS": 60.0, "priority": 10, + # What: arrange group interactive concurrencyLimit 2 sendLoadingState False for the scenario; why: test catalog test router policy is strict and public model fields are safe requires this concrete input or helper state before exercising the behavior under test. + "group": "interactive", "concurrencyLimit": 2, "sendLoadingState": False, + # What: arrange the grouped source fragment for the scenario; why: test catalog test router policy is strict and public model fields are safe requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that api keys is absent from str public; why: this assertion protects the router policy is strict and public model fields are safe regression after the test's arranged inputs and exercised call. + assert "api_keys" not in str(public) + + +# What: parameterize test_router_policy_rejects_ambiguous_or_unsafe_configuration with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test router policy rejects ambiguous or unsafe configuration. +@pytest.mark.parametrize("router, message", [ + # What: arrange the router nscheduler lifo scheduler portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[router]\nscheduler = 'lifo'", "scheduler"), + # What: arrange the router nupstream timeout s upstream timeout s portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[router]\nupstream_timeout_s = 0", "upstream_timeout_s"), + # What: arrange the drop fields model drop fields portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("drop_fields = ['model']", "drop_fields"), + # What: arrange the router napi keys same same duplicates portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[router]\napi_keys = ['same', 'same']", "duplicates"), + # What: arrange the router ninclude aliases in list yes include aliases in list portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[router]\ninclude_aliases_in_list = 'yes'", "include_aliases_in_list"), + # What: arrange the router nglobal concurrency limit global concurrency limit portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[router]\nglobal_concurrency_limit = -1", "global_concurrency_limit"), + # What: arrange the router nsend loading state yes send loading state portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[router]\nsend_loading_state = 'yes'", "send_loading_state"), + # What: arrange the concurrency limit true concurrency limit portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("concurrency_limit = true", "concurrency_limit"), + # What: arrange the send loading state send loading state portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("send_loading_state = 1", "send_loading_state"), + # What: arrange the router groups bad name nmembers a router portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ('[router.groups."bad/name"]\nmembers = ["a"]', "router group names"), + # What: arrange the router groups g nmembers missing configured models portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[router.groups.g]\nmembers = ['missing']", "configured models"), + # What: arrange the router groups g nmembers a npersistent true persistent portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[router.groups.g]\nmembers = ['a']\npersistent = true", "persistent"), + # What: arrange the router groups g nmembers a nexclusive false exclusive portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[router.groups.g]\nmembers = ['a']\nexclusive = false", "exclusive"), + # What: arrange the router groups g nmembers a nswap false multi resident portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[router.groups.g]\nmembers = ['a']\nswap = false", "multi-resident"), + # What: arrange the models b nmodel b gguf n router groups g nmembers portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("[models.b]\nmodel = 'b.gguf'\n[router.groups.g]\nmembers = ['a', 'b']\npersistent = true\nswap = false", "exactly one"), + # What: arrange the group other must match portion of the enclosing predicate; why: this clause remains in the router policy rejects ambiguous or unsafe configuration scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("group = 'other'", "must match"), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_router_policy_rejects_ambiguous_or_unsafe_configuration groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_router_policy_rejects_ambiguous_or_unsafe_configuration test around tmp path and router and message; why: this test groups the arrange, act, and assertions that protect the router policy rejects ambiguous or unsafe configuration outcome. +def test_router_policy_rejects_ambiguous_or_unsafe_configuration(tmp_path, router, message): + # What: arrange path as tmp path and models and toml; why: the router policy rejects ambiguous or unsafe configuration test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text models a nmodel a gguf n router fixture fragment; why: the router policy rejects ambiguous or unsafe configuration scenario feeds this byte-preserved fragment through path.write_text("[models.a]\nmodel = 'a.gguf'\n" + router, encoding="utf before asserting its protocol or. + path.write_text("[models.a]\nmodel = 'a.gguf'\n" + router, encoding="utf-8") + # What: assert the pytest.raises failure context; why: the router policy rejects ambiguous or unsafe configuration scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match=message): + # What: act by calling ModelCatalog.load with str and path; why: the router policy rejects ambiguous or unsafe configuration scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: define the test_router_policy_accepts_a_singleton_persistent_protected_slot test around tmp path; why: this test groups the arrange, act, and assertions that protect the router policy accepts a singleton persistent protected slot outcome. +def test_router_policy_accepts_a_singleton_persistent_protected_slot(tmp_path): + # What: arrange path as tmp path and models and toml; why: the router policy accepts a singleton persistent protected slot test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text fixture fragment; why: the router policy accepts a singleton persistent protected slot scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact router groups resident fixture fragment; why: the router policy accepts a singleton persistent protected slot scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact members a fixture fragment; why: the router policy accepts a singleton persistent protected slot scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact swap false fixture fragment; why: the router policy accepts a singleton persistent protected slot scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact exclusive true fixture fragment; why: the router policy accepts a singleton persistent protected slot scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact persistent true fixture fragment; why: the router policy accepts a singleton persistent protected slot scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact models a fixture fragment; why: the router policy accepts a singleton persistent protected slot scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact model a gguf fixture fragment; why: the router policy accepts a singleton persistent protected slot scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact group resident fixture fragment; why: the router policy accepts a singleton persistent protected slot scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + # What: arrange the exact encoding utf 8 fixture fragment; why: the router policy accepts a singleton persistent protected slot scenario feeds this byte-preserved fragment through path.write_text(""" before asserting its protocol or parser result. + path.write_text(""" +[router.groups.resident] +members = ["a"] +swap = false +exclusive = true +persistent = true + +[models.a] +model = "a.gguf" +group = "resident" +""", encoding="utf-8") + # What: act by calling ModelCatalog.load and capture group; why: the router policy accepts a singleton persistent protected slot test asserts the response, state, or failure produced by this call. + group = ModelCatalog.load(str(path)).settings.groups[0] + # What: assert that group members group swap group exclusive group persistent equals a false true true; why: this assertion protects the router policy accepts a singleton persistent protected slot regression after the test's arranged inputs and exercised call. + assert (group.members, group.swap, group.exclusive, group.persistent) == (("a",), False, True, True) diff --git a/tests/daemon/test_daemon_import_safety.py b/tests/daemon/test_daemon_import_safety.py index 2771034316..a96d3c80fd 100644 --- a/tests/daemon/test_daemon_import_safety.py +++ b/tests/daemon/test_daemon_import_safety.py @@ -20,6 +20,10 @@ "freetoken.daemon", "freetoken.daemon.version", "freetoken.daemon.accounting", + # What: arrange the freetoken daemon catalog portion of daemon modules; why: the daemon import safety scenario uses this clause to evaluate daemon modules as one grouped value. + "freetoken.daemon.catalog", + # What: arrange the freetoken daemon readiness portion of daemon modules; why: the daemon import safety scenario uses this clause to evaluate daemon modules as one grouped value. + "freetoken.daemon.readiness", "freetoken.daemon.logfmt", "freetoken.daemon.logring", "freetoken.daemon.osproc", @@ -110,3 +114,32 @@ def test_daemon_package_imports_without_torch(): f"daemon import-safety child exited {proc.returncode}\n" f"--- stdout ---\n{proc.stdout}\n--- stderr ---\n{proc.stderr}" ) + +def test_non_loopback_authentication_policy_rejects_empty_and_template_keys() -> None: + # Import helpers directly so this safety gate remains independent of uvicorn and GPU dependencies. + from freetoken.daemon.server import _catalog_has_release_authentication, _is_loopback_host + + # Build the smallest catalog-shaped object needed to exercise the startup authentication decision. + catalog = type("Catalog", (), {"settings": type("Settings", (), {"api_keys": ()})()})() + # Confirm loopback spellings stay available for local development without mandatory credentials. + assert _is_loopback_host("127.0.0.1") and _is_loopback_host("::1") and _is_loopback_host("localhost") + # Confirm wildcard and ordinary interface binds are treated as network exposure. + assert not _is_loopback_host("0.0.0.0") and not _is_loopback_host("192.0.2.10") + # Confirm an empty catalog cannot satisfy the release authentication requirement. + assert not _catalog_has_release_authentication(catalog, None) + # Replace the key list with an empty value to prove presence alone cannot authorize a public bind. + catalog.settings.api_keys = ("",) + # Confirm an empty credential is rejected even when the tuple itself is non-empty. + assert not _catalog_has_release_authentication(catalog, None) + # Replace the key list with the shipped marker to prove an unedited example still fails closed. + catalog.settings.api_keys = ("REPLACE_WITH_A_GENERATED_SECRET",) + # Confirm the public template value is never accepted as a real secret. + assert not _catalog_has_release_authentication(catalog, None) + # Replace the marker with a generated-looking test secret to prove configured authentication is accepted. + catalog.settings.api_keys = ("test-only-generated-secret",) + # Confirm a configured inference key satisfies the startup gate. + assert _catalog_has_release_authentication(catalog, None) + # Remove inference keys so the management-only token cannot accidentally authorize public proxy routes. + catalog.settings.api_keys = () + # Confirm the dedicated daemon token alone is insufficient for a non-loopback inference bind. + assert not _catalog_has_release_authentication(catalog, "test-only-daemon-token") diff --git a/tests/daemon/test_daemon_serve_manager.py b/tests/daemon/test_daemon_serve_manager.py index fd9956ebba..fdef2c5b32 100644 --- a/tests/daemon/test_daemon_serve_manager.py +++ b/tests/daemon/test_daemon_serve_manager.py @@ -13,7 +13,8 @@ ) from freetoken.daemon.logring import LogRing from freetoken.daemon.pidfile import ServeState, ServeStateStore -from freetoken.daemon.serve_manager import Conflict, ExitInfo, ServeManager +# What: arrange from freetoken daemon serve manager import Conflict ExitInfo ServeManager SwitchLaunchError for the scenario; why: test daemon serve manager requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.serve_manager import Conflict, ExitInfo, ServeManager, SwitchLaunchError # --------------------------------------------------------------------------- test doubles @@ -117,6 +118,246 @@ def make_manager( # --------------------------------------------------------------------------- start / idempotency +# What: parameterize test_switch_spawn_failure_restores_exact_previous_launch with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test switch spawn failure restores exact previous launch. +@pytest.mark.parametrize("recovery_fails", [False, True]) +# What: define the test_switch_spawn_failure_restores_exact_previous_launch test around tmp path and recovery fails; why: this test groups the arrange, act, and assertions that protect the switch spawn failure restores exact previous launch outcome. +def test_switch_spawn_failure_restores_exact_previous_launch(tmp_path, recovery_fails): + # What: act by calling Spawner and capture sp; why: the switch spawn failure restores exact previous launch test asserts the response, state, or failure produced by this call. + sp = Spawner() + # What: arrange calls as the fixture input; why: the switch spawn failure restores exact previous launch test consumes this named precondition before exercising the behavior. + calls = [] + + # What: define the spawn test helper around model and port and args; why: the switch spawn failure restores exact previous launch scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def spawn(model, port, args): + # What: act by calling calls.append with model and port and list and args; why: the switch spawn failure restores exact previous launch scenario observes the calls.append return value during if model bad or recovery fails and. + calls.append((model, port, list(args))) + # What: act on model and recovery fails and len and calls before oserror; why: the switch spawn failure restores exact previous launch scenario admits oserror only for this predicate and excludes the opposite state. + if model == "bad" or (recovery_fails and len(calls) == 3): + # What: raise OSError for the caller; why: spawn stops this rejected path before it can mutate state, dispatch work, or report success. + raise OSError("injected launch failure") + # What: return sp and model and port and args from the spawn test helper; why: the switch spawn failure restores exact previous launch scenario uses this helper result in its subsequent act or assertion. + return sp(model, port, args) + + # What: act by calling make_manager and capture mgr and store and; why: the switch spawn failure restores exact previous launch test asserts the response, state, or failure produced by this call. + mgr, store, _ = make_manager( + # What: arrange the pid input for test_switch_spawn_failure_restores_exact_previous_launch; why: test_switch_spawn_failure_restores_exact_previous_launch consumes pid during signature binding, so callers must bind it with the other signature inputs. + tmp_path, spawn, signal_fn=lambda pid, sig: sp.by_pid(pid).die() + # What: arrange the make_manager call with signal fn; why: test_switch_spawn_failure_restores_exact_previous_launch groups the supplied clauses as one make_manager call before its value is consumed. + ) + # What: arrange the exact mgr start previous example fixture fragment; why: the switch spawn failure restores exact previous launch scenario feeds this byte-preserved fragment through mgr.start("previous", 1922, ["--example"]) before asserting its protocol or parser result. + mgr.start("previous", 1922, ["--example"]) + # What: assert the pytest.raises failure context; why: the switch spawn failure restores exact previous launch scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(SwitchLaunchError) as failed: + # What: arrange the exact mgr switch bad fixture fragment; why: the switch spawn failure restores exact previous launch scenario feeds this byte-preserved fragment through mgr.switch("bad", 1923, []) before asserting its protocol or parser result. + mgr.switch("bad", 1923, []) + # What: assert the expected calls == previous 1922 example outcome; why: test daemon serve manager test switch spawn failure restores exact previous launch protects its regression by requiring this observable result after the exercised behavior. + assert calls == [("previous", 1922, ["--example"]), + # What: arrange bad 1923 previous 1922 example for the scenario; why: test daemon serve manager test switch spawn failure restores exact previous launch requires this concrete input or helper state before exercising the behavior under test. + ("bad", 1923, []), ("previous", 1922, ["--example"])] + # What: assert that failed value rollback attempted is true; why: this assertion protects the switch spawn failure restores exact previous launch regression after the test's arranged inputs and exercised call. + assert failed.value.rollback["attempted"] is True + # What: assert that failed value rollback launched is not recovery fails; why: this assertion protects the switch spawn failure restores exact previous launch regression after the test's arranged inputs and exercised call. + assert failed.value.rollback["launched"] is (not recovery_fails) + # What: assert that failed value accounting is not group delimiter; why: this assertion protects the switch spawn failure restores exact previous launch regression after the test's arranged inputs and exercised call. + assert failed.value.accounting is not None + # What: assert that mgr status running is not recovery fails; why: this assertion protects the switch spawn failure restores exact previous launch regression after the test's arranged inputs and exercised call. + assert mgr.status()["running"] is (not recovery_fails) + # What: act on recovery fails before model and load and store; why: the switch spawn failure restores exact previous launch scenario admits model and load and store only for this predicate and excludes the opposite state. + if not recovery_fails: + # What: assert that store load model equals previous; why: this assertion protects the switch spawn failure restores exact previous launch regression after the test's arranged inputs and exercised call. + assert store.load().model == "previous" + # What: act by calling mgr.stop with the declared inputs; why: the switch spawn failure restores exact previous launch scenario observes the mgr.stop return value during the enclosing return. + mgr.stop() + + +# What: define the test_switch_spawn_failure_without_previous_does_not_retry test around tmp path; why: this test groups the arrange, act, and assertions that protect the switch spawn failure without previous does not retry outcome. +def test_switch_spawn_failure_without_previous_does_not_retry(tmp_path): + # What: define the spawn test helper around captured fixture state; why: the switch spawn failure without previous does not retry scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def spawn(*args): + # What: raise OSError for the caller; why: spawn stops this rejected path before it can mutate state, dispatch work, or report success. + raise OSError("injected launch failure") + + # What: act by calling make_manager and capture mgr and and; why: the switch spawn failure without previous does not retry test asserts the response, state, or failure produced by this call. + mgr, _, _ = make_manager(tmp_path, spawn) + # What: assert the pytest.raises failure context; why: the switch spawn failure without previous does not retry scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(SwitchLaunchError) as failed: + # What: arrange the exact mgr switch bad fixture fragment; why: the switch spawn failure without previous does not retry scenario feeds this byte-preserved fragment through mgr.switch("bad", 1922) before asserting its protocol or parser result. + mgr.switch("bad", 1922) + # What: assert that failed value rollback equals attempted false launched false; why: this assertion protects the switch spawn failure without previous does not retry regression after the test's arranged inputs and exercised call. + assert failed.value.rollback == {"attempted": False, "launched": False} + # What: assert that mgr status running is false; why: this assertion protects the switch spawn failure without previous does not retry regression after the test's arranged inputs and exercised call. + assert not mgr.status()["running"] + + +# What: parameterize test_readiness_recovery_never_overrides_newer_lifecycle with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test readiness recovery never overrides newer lifecycle. +@pytest.mark.parametrize("newer_action", [None, "stop", "switch", "shutdown", "start"]) +# What: define the test_readiness_recovery_never_overrides_newer_lifecycle test around tmp path and newer action; why: this test groups the arrange, act, and assertions that protect the readiness recovery never overrides newer lifecycle outcome. +def test_readiness_recovery_never_overrides_newer_lifecycle(tmp_path, newer_action): + # What: act by calling Spawner and capture sp; why: the readiness recovery never overrides newer lifecycle test asserts the response, state, or failure produced by this call. + sp = Spawner() + # What: act by calling make_manager and capture mgr and and; why: the readiness recovery never overrides newer lifecycle test asserts the response, state, or failure produced by this call. + mgr, _, _ = make_manager(tmp_path, sp, + # What: arrange the pid input for test_readiness_recovery_never_overrides_newer_lifecycle; why: test_readiness_recovery_never_overrides_newer_lifecycle consumes pid during signature binding, so callers must bind it with the other signature inputs. + signal_fn=lambda pid, sig: sp.by_pid(pid).die()) + # What: arrange the exact mgr start previous original fixture fragment; why: the readiness recovery never overrides newer lifecycle scenario feeds this byte-preserved fragment through mgr.start("previous", 1922, ["--original"]) before asserting its protocol or parser result. + mgr.start("previous", 1922, ["--original"]) + # What: act by calling mgr.switch_for_readiness and capture and ticket; why: the readiness recovery never overrides newer lifecycle test asserts the response, state, or failure produced by this call. + _, ticket = mgr.switch_for_readiness("replacement", 1923) + # What: act on newer action before stop and mgr; why: the readiness recovery never overrides newer lifecycle scenario admits stop and mgr only for this predicate and excludes the opposite state. + if newer_action == "stop": + # What: act by calling mgr.stop with the declared inputs; why: the readiness recovery never overrides newer lifecycle scenario observes the mgr.stop return value during elif newer action shutdown. + mgr.stop() + # What: act on newer action before shutdown and mgr; why: the readiness recovery never overrides newer lifecycle scenario admits shutdown and mgr only for this predicate and excludes the opposite state. + elif newer_action == "shutdown": + # What: act by calling mgr.shutdown with the declared inputs; why: the readiness recovery never overrides newer lifecycle scenario observes the mgr.shutdown return value during elif newer action switch. + mgr.shutdown() + # What: act on newer action before switch and mgr; why: the readiness recovery never overrides newer lifecycle scenario admits switch and mgr only for this predicate and excludes the opposite state. + elif newer_action == "switch": + # What: arrange the exact mgr switch newer fixture fragment; why: the readiness recovery never overrides newer lifecycle scenario feeds this byte-preserved fragment through mgr.switch("newer", 1924) before asserting its protocol or parser result. + mgr.switch("newer", 1924) + # What: act on newer action before start and mgr; why: the readiness recovery never overrides newer lifecycle scenario admits start and mgr only for this predicate and excludes the opposite state. + elif newer_action == "start": + # What: arrange the exact mgr start replacement even explicit idempotent intent fixture fragment; why: the readiness recovery never overrides newer lifecycle scenario feeds this byte-preserved fragment through mgr.start("replacement", 1923) # even explicit idempotent intent wins before asserting its protocol o. + mgr.start("replacement", 1923) # even explicit idempotent intent wins + # What: act by calling mgr.recover_switch and capture result; why: the readiness recovery never overrides newer lifecycle test asserts the response, state, or failure produced by this call. + result = mgr.recover_switch(ticket) + # What: assert that result launched is newer action is; why: this assertion protects the readiness recovery never overrides newer lifecycle regression after the test's arranged inputs and exercised call. + assert result["launched"] is (newer_action is None) + # What: act on newer action before result; why: the readiness recovery never overrides newer lifecycle scenario admits result only for this predicate and excludes the opposite state. + if newer_action is not None: + # What: assert that result reason equals superseded; why: this assertion protects the readiness recovery never overrides newer lifecycle regression after the test's arranged inputs and exercised call. + assert result["reason"] == "superseded" + # What: select the remaining branch that performs assert sp calls previous original; why: test_readiness_recovery_never_overrides_newer_lifecycle covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: assert that sp calls 1 equals previous 1922 original; why: this assertion protects the readiness recovery never overrides newer lifecycle regression after the test's arranged inputs and exercised call. + assert sp.calls[-1] == ("previous", 1922, ["--original"]) + # Recovery is single-use even when a delayed caller repeats the request. + # What: assert that mgr recover switch ticket reason equals superseded; why: this assertion protects the readiness recovery never overrides newer lifecycle regression after the test's arranged inputs and exercised call. + assert mgr.recover_switch(ticket)["reason"] == "superseded" + # What: act by calling mgr.stop with the declared inputs; why: the readiness recovery never overrides newer lifecycle scenario observes the mgr.stop return value during the enclosing return. + mgr.stop() + + +# What: define the test_readiness_recovery_preserves_engine_when_accounting_fails test around tmp path; why: this test groups the arrange, act, and assertions that protect the readiness recovery preserves engine when accounting fails outcome. +def test_readiness_recovery_preserves_engine_when_accounting_fails(tmp_path): + # What: act by calling Spawner and capture sp; why: the readiness recovery preserves engine when accounting fails test asserts the response, state, or failure produced by this call. + sp = Spawner() + # What: act by calling make_manager and capture mgr and and; why: the readiness recovery preserves engine when accounting fails test asserts the response, state, or failure produced by this call. + mgr, _, _ = make_manager(tmp_path, sp, + # What: arrange the pid input for test_readiness_recovery_preserves_engine_when_accounting_fails; why: test_readiness_recovery_preserves_engine_when_accounting_fails consumes pid during signature binding, so callers must bind it with the other signature inputs. + signal_fn=lambda pid, sig: sp.by_pid(pid).die()) + # What: arrange the exact mgr start previous fixture fragment; why: the readiness recovery preserves engine when accounting fails scenario feeds this byte-preserved fragment through mgr.start("previous", 1922) before asserting its protocol or parser result. + mgr.start("previous", 1922) + # What: act by calling mgr.switch_for_readiness and capture and ticket; why: the readiness recovery preserves engine when accounting fails test asserts the response, state, or failure produced by this call. + _, ticket = mgr.switch_for_readiness("replacement", 1923) + + # What: define the unavailable test helper around port; why: the readiness recovery preserves engine when accounting fails scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def unavailable(port): + # What: raise AccountingPrepareError for the caller; why: unavailable stops this rejected path before it can mutate state, dispatch work, or report success. + raise AccountingPrepareError("injected unavailable accounting") + + # What: arrange prepare stop as unavailable; why: the readiness recovery preserves engine when accounting fails test consumes this named precondition before exercising the behavior. + mgr._prepare_stop = unavailable + # What: act by calling mgr.recover_switch and capture result; why: the readiness recovery preserves engine when accounting fails test asserts the response, state, or failure produced by this call. + result = mgr.recover_switch(ticket) + # What: assert that result attempted and not result launched; why: this assertion protects the readiness recovery preserves engine when accounting fails regression after the test's arranged inputs and exercised call. + assert result["attempted"] and not result["launched"] + # What: assert that result engine preserved; why: this assertion protects the readiness recovery preserves engine when accounting fails regression after the test's arranged inputs and exercised call. + assert result["enginePreserved"] + # What: assert that mgr status model equals replacement; why: this assertion protects the readiness recovery preserves engine when accounting fails regression after the test's arranged inputs and exercised call. + assert mgr.status()["model"] == "replacement" + # What: arrange prepare stop as the fixture input; why: the readiness recovery preserves engine when accounting fails test consumes this named precondition before exercising the behavior. + mgr._prepare_stop = None + # What: act by calling mgr.stop with the declared inputs; why: the readiness recovery preserves engine when accounting fails scenario observes the mgr.stop return value during the enclosing return. + mgr.stop() + + +# What: define the test_readiness_recovery_can_restore_after_replacement_exits test around tmp path; why: this test groups the arrange, act, and assertions that protect the readiness recovery can restore after replacement exits outcome. +def test_readiness_recovery_can_restore_after_replacement_exits(tmp_path): + # What: act by calling Spawner and capture sp; why: the readiness recovery can restore after replacement exits test asserts the response, state, or failure produced by this call. + sp = Spawner() + # What: act by calling make_manager and capture mgr and and; why: the readiness recovery can restore after replacement exits test asserts the response, state, or failure produced by this call. + mgr, _, _ = make_manager(tmp_path, sp, + # What: arrange the pid input for test_readiness_recovery_can_restore_after_replacement_exits; why: test_readiness_recovery_can_restore_after_replacement_exits consumes pid during signature binding, so callers must bind it with the other signature inputs. + signal_fn=lambda pid, sig: sp.by_pid(pid).die()) + # What: arrange the exact mgr start previous fixture fragment; why: the readiness recovery can restore after replacement exits scenario feeds this byte-preserved fragment through mgr.start("previous", 1922) before asserting its protocol or parser result. + mgr.start("previous", 1922) + # What: act by calling mgr.switch_for_readiness and capture replacement and ticket; why: the readiness recovery can restore after replacement exits test asserts the response, state, or failure produced by this call. + replacement, ticket = mgr.switch_for_readiness("replacement", 1923) + # What: act by calling sp.by_pid and capture child; why: the readiness recovery can restore after replacement exits test asserts the response, state, or failure produced by this call. + child = sp.by_pid(replacement["pid"]) + # What: act by calling child.die with 1; why: the readiness recovery can restore after replacement exits scenario observes the child.die return value during assert child reaped wait. + child.die(1) + # What: assert that child reaped wait 3; why: this assertion protects the readiness recovery can restore after replacement exits regression after the test's arranged inputs and exercised call. + assert child.reaped.wait(3) + # What: assert that mgr recover switch ticket launched; why: this assertion protects the readiness recovery can restore after replacement exits regression after the test's arranged inputs and exercised call. + assert mgr.recover_switch(ticket)["launched"] + # What: assert that mgr status model equals previous; why: this assertion protects the readiness recovery can restore after replacement exits regression after the test's arranged inputs and exercised call. + assert mgr.status()["model"] == "previous" + # What: act by calling mgr.stop with the declared inputs; why: the readiness recovery can restore after replacement exits scenario observes the mgr.stop return value during the enclosing return. + mgr.stop() + + +# What: define the test_recovery_waits_for_old_pidfile_cleanup test around tmp path; why: this test groups the arrange, act, and assertions that protect the recovery waits for old pidfile cleanup outcome. +def test_recovery_waits_for_old_pidfile_cleanup(tmp_path): + # What: act by calling Spawner and capture sp; why: the recovery waits for old pidfile cleanup test asserts the response, state, or failure produced by this call. + sp = Spawner() + # What: act by calling make_manager and capture mgr and store and; why: the recovery waits for old pidfile cleanup test asserts the response, state, or failure produced by this call. + mgr, store, _ = make_manager(tmp_path, sp, + # What: arrange the pid input for test_recovery_waits_for_old_pidfile_cleanup; why: test_recovery_waits_for_old_pidfile_cleanup consumes pid during signature binding, so callers must bind it with the other signature inputs. + signal_fn=lambda pid, sig: sp.by_pid(pid).die()) + # What: arrange the exact mgr start previous fixture fragment; why: the recovery waits for old pidfile cleanup scenario feeds this byte-preserved fragment through mgr.start("previous", 1922) before asserting its protocol or parser result. + mgr.start("previous", 1922) + # What: act by calling mgr.switch_for_readiness and capture replacement and ticket; why: the recovery waits for old pidfile cleanup test asserts the response, state, or failure produced by this call. + replacement, ticket = mgr.switch_for_readiness("replacement", 1923) + # What: act by calling threading.Event and capture entered and release; why: the recovery waits for old pidfile cleanup test asserts the response, state, or failure produced by this call. + entered, release = threading.Event(), threading.Event() + # What: arrange clear as clear and store; why: the recovery waits for old pidfile cleanup test consumes this named precondition before exercising the behavior. + clear = store.clear + + # What: define the delayed_clear test helper around captured fixture state; why: the recovery waits for old pidfile cleanup scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def delayed_clear(): + # What: act by calling entered.set with the declared inputs; why: the recovery waits for old pidfile cleanup scenario observes the entered.set return value during assert release wait. + entered.set() + # What: assert that release wait 5; why: this assertion protects the recovery waits for old pidfile cleanup regression after the test's arranged inputs and exercised call. + assert release.wait(5) + # What: act by calling clear with the declared inputs; why: the recovery waits for old pidfile cleanup scenario observes the clear return value during the enclosing return. + clear() + + # What: arrange clear as delayed clear; why: the recovery waits for old pidfile cleanup test consumes this named precondition before exercising the behavior. + store.clear = delayed_clear + # What: arrange the exact sp by pid replacement pid die fixture fragment; why: the recovery waits for old pidfile cleanup scenario feeds this byte-preserved fragment through sp.by_pid(replacement["pid"]).die(1) before asserting its protocol or parser result. + sp.by_pid(replacement["pid"]).die(1) + # What: assert that entered wait 3; why: this assertion protects the recovery waits for old pidfile cleanup regression after the test's arranged inputs and exercised call. + assert entered.wait(3) + # What: arrange result as the fixture input; why: the recovery waits for old pidfile cleanup test consumes this named precondition before exercising the behavior. + result = {} + # What: act by calling threading.Thread and capture recovery; why: the recovery waits for old pidfile cleanup test asserts the response, state, or failure produced by this call. + recovery = threading.Thread(target=lambda: result.update(mgr.recover_switch(ticket))) + # What: act by calling recovery.start with the declared inputs; why: the recovery waits for old pidfile cleanup scenario observes the recovery.start return value during try. + recovery.start() + # What: establish the handler boundary for the protected operation; why: test_recovery_waits_for_old_pidfile_cleanup routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: assert that len sp calls equals 2; why: this assertion protects the recovery waits for old pidfile cleanup regression after the test's arranged inputs and exercised call. + assert len(sp.calls) == 2 + # What: run release set on every exit path; why: test_recovery_waits_for_old_pidfile_cleanup performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: act by calling release.set with the declared inputs; why: the recovery waits for old pidfile cleanup scenario observes the release.set return value during recovery join. + release.set() + # What: act by calling recovery.join with 3; why: the recovery waits for old pidfile cleanup scenario observes the recovery.join return value during assert not recovery is alive. + recovery.join(3) + # What: assert that recovery is alive is false; why: this assertion protects the recovery waits for old pidfile cleanup regression after the test's arranged inputs and exercised call. + assert not recovery.is_alive() + # What: assert that result launched; why: this assertion protects the recovery waits for old pidfile cleanup regression after the test's arranged inputs and exercised call. + assert result["launched"] + # What: assert that store load model equals previous; why: this assertion protects the recovery waits for old pidfile cleanup regression after the test's arranged inputs and exercised call. + assert store.load().model == "previous" + # What: arrange clear as clear; why: the recovery waits for old pidfile cleanup test consumes this named precondition before exercising the behavior. + store.clear = clear + # What: act by calling mgr.stop with the declared inputs; why: the recovery waits for old pidfile cleanup scenario observes the mgr.stop return value during the enclosing return. + mgr.stop() + + def test_start_reports_running(tmp_path): sp = Spawner() mgr, store, _ = make_manager(tmp_path, sp) @@ -761,3 +1002,40 @@ def sig(pid, s): assert wait_until(lambda: mgr.status()["running"] is False) # monitor threads must all have exited (daemon threads terminate after _reap) assert wait_until(lambda: threading.active_count() <= baseline + 2) + +# What: verify shutdown bounds lifecycle ownership waits; why: systemd cleanup must not hang behind a stuck start or switch. +def test_shutdown_timeout_bounds_lifecycle_lock_acquisition(tmp_path) -> None: + # Create an idle fake manager so only lifecycle-lock contention controls this test. + manager, _store, _ring = make_manager(tmp_path, Spawner()) + # Signal when the background owner holds the lock so the timeout test is deterministic. + locked = threading.Event() + # Hold the competing lifecycle owner until the main test has observed the timeout. + release = threading.Event() + + # Define the competing lifecycle operation; why: an in-flight start or switch owns this same lock. + def hold_lifecycle() -> None: + # Acquire the private lock exactly as production lifecycle methods do. + with manager._lifecycle: + # Announce ownership only after shutdown is guaranteed to contend. + locked.set() + # Wait for test cleanup so ownership persists beyond the short shutdown deadline. + release.wait(timeout=2.0) + + # Start the competing owner in another thread because the lifecycle lock is reentrant per thread. + holder = threading.Thread(target=hold_lifecycle) + # Launch the owner before invoking shutdown so lock acquisition must wait. + holder.start() + # Require the ownership signal so scheduler timing cannot make the test flaky. + assert locked.wait(timeout=1.0) + try: + # Require shutdown to return a concrete timeout rather than blocking indefinitely. + with pytest.raises(TimeoutError, match="lifecycle ownership"): + # Use a bounded deadline that is much shorter than the background hold. + manager.shutdown(timeout=0.01, force=True) + finally: + # Release the background owner on success or failure so the test cannot strand a thread. + release.set() + # Join the owner to prove cleanup completed before the test returns. + holder.join(timeout=1.0) + # Confirm the competing lifecycle thread exited after cleanup. + assert not holder.is_alive() diff --git a/tests/daemon/test_metrics.py b/tests/daemon/test_metrics.py new file mode 100644 index 0000000000..0b26816773 --- /dev/null +++ b/tests/daemon/test_metrics.py @@ -0,0 +1,176 @@ +# What: import json for test amd smi process vram parses multi gpu json using json; why: test_amd_smi_process_vram_parses_multi_gpu_json uses json dumps, making that imported dependency available to its named operation. +import json + +# What: import simple namespace for test amd smi process vram parses multi gpu json using types and simple namespace; why: test_amd_smi_process_vram_parses_multi_gpu_json uses simple namespace, making that imported dependency available to its named operation. +from types import SimpleNamespace + +# What: import metrics for test vram measurement falls through to amd smi using freetoken and daemon and metrics; why: test_vram_measurement_falls_through_to_amd_smi uses the metrics annotation in test vram measurement falls through to amd smi, making that imported dependency available to its named operation. +from freetoken.daemon import metrics + + +# What: define DRM fdinfo deduplication coverage; why: AMD APU process memory must include VRAM and GTT exactly once per client even when several descriptors share it. +def test_drm_fdinfo_process_vram_deduplicates_clients(tmp_path): + # What: create the representative owned PID fdinfo directory; why: the probe reads the same procfs shape exposed by Linux DRM drivers. + fdinfo = tmp_path / "41" / "fdinfo" + # What: create parent directories for the fixture; why: individual descriptor records need a concrete procfs-like location. + fdinfo.mkdir(parents=True) + # What: define one shared DRM accounting record; why: duplicate descriptors for one client must not inflate measured GPU memory. + shared = "drm-client-id:\t7\ndrm-pdev:\t0000:64:00.0\ndrm-memory-vram:\t2 MiB\ndrm-memory-gtt:\t3 MiB\n" + # What: write the first descriptor for the shared client; why: the probe must count this valid VRAM plus GTT record. + (fdinfo / "3").write_text(shared, encoding="utf-8") + # What: write a duplicate descriptor for the shared client; why: deduplication must prevent counting identical client totals twice. + (fdinfo / "4").write_text(shared, encoding="utf-8") + # What: write a second distinct DRM client; why: separate clients owned by the same PID must still be aggregated. + (fdinfo / "5").write_text("drm-client-id:\t8\ndrm-pdev:\t0000:64:00.0\ndrm-memory-gtt:\t1 GiB\n", encoding="utf-8") + # What: require the deduplicated byte total; why: the result must combine local and aperture allocations without host-wide telemetry. + assert metrics._drm_fdinfo_process_vram([41], tmp_path) == {41: 5 * 1024**2 + 1024**3} + + +# What: define DRM probe precedence coverage; why: privacy-bounded native Linux accounting should satisfy AMD hosts before vendor command fallbacks. +def test_vram_measurement_prefers_owned_drm_fdinfo(monkeypatch): + # What: provide a measured owned-process DRM mapping; why: the selector should accept authoritative fdinfo without invoking broader probes. + monkeypatch.setattr(metrics, "_drm_fdinfo_process_vram", lambda pids: {41: 123}) + # What: require the DRM bytes, availability, and source label; why: API consumers need explicit provenance for the live measurement. + assert metrics._vram_measurement_for_pids([41]) == (123, True, "drm-fdinfo-vram-gtt") + + +# What: define the test_amd_smi_process_vram_parses_multi_gpu_json test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the amd smi process vram parses multi gpu json outcome. +def test_amd_smi_process_vram_parses_multi_gpu_json(monkeypatch): + # What: arrange doc as gpu and process list and 0 and gpu and process list; why: the amd smi process vram parses multi gpu json test consumes this named precondition before exercising the behavior. + doc = [ + # What: arrange the gpu field as 0; why: test_amd_smi_process_vram_parses_multi_gpu_json carries gpu through doc into returncode 0 stdout json dumps doc. + {"gpu": 0, "process_list": [{"process_info": { + # What: arrange the pid portion of doc; why: the amd smi process vram parses multi gpu json scenario uses this clause to evaluate doc as one grouped value. + "pid": 41, + # What: arrange the vram mem field as value and unit and 2 and gi b; why: test_amd_smi_process_vram_parses_multi_gpu_json carries vram mem through doc into returncode 0 stdout json dumps doc. + "memory_usage": {"vram_mem": {"value": 2, "unit": "GiB"}}, + # What: arrange the doc mapping with gpu and process list; why: test_amd_smi_process_vram_parses_multi_gpu_json groups the supplied clauses as one doc mapping before its value is consumed. + }}]}, + # What: arrange the process info field as pid and memory usage and 41 and vram mem and value; why: test_amd_smi_process_vram_parses_multi_gpu_json carries process info through doc into returncode 0 stdout json dumps doc. + {"gpu": 1, "process_list": [{"process_info": { + # What: arrange the pid portion of doc; why: the amd smi process vram parses multi gpu json scenario uses this clause to evaluate doc as one grouped value. + "pid": 41, + # What: arrange the vram mem field as value and unit and 512 and mi b; why: test_amd_smi_process_vram_parses_multi_gpu_json carries vram mem through doc into returncode 0 stdout json dumps doc. + "memory_usage": {"vram_mem": {"value": 512, "unit": "MiB"}}, + # What: arrange the process info field as pid and memory usage and 42 and vram mem and gb; why: test_amd_smi_process_vram_parses_multi_gpu_json carries process info through doc into returncode 0 stdout json dumps doc. + }}, {"process_info": { + # What: arrange the pid portion of doc; why: the amd smi process vram parses multi gpu json scenario uses this clause to evaluate doc as one grouped value. + "pid": 42, + # What: arrange the vram mem field as gb; why: test_amd_smi_process_vram_parses_multi_gpu_json carries vram mem through doc into returncode 0 stdout json dumps doc. + "memory_usage": {"vram_mem": "1.5 GB"}, + # What: arrange the doc mapping with process info; why: test_amd_smi_process_vram_parses_multi_gpu_json groups the supplied clauses as one doc mapping before its value is consumed. + }}]}, + # What: arrange the doc collection with gpu and process list and 0 and process info and pid and gpu and process list and 1 and process info and process info; why: test_amd_smi_process_vram_parses_multi_gpu_json groups the supplied clauses as one doc collection before its value is consumed. + ] + # What: arrange the exact monkeypatch setattr metrics subprocess run lambda args kwargs fixture fragment; why: the amd smi process vram parses multi gpu json scenario feeds this byte-preserved fragment through monkeypatch.setattr(metrics.subprocess, "run", lambda *args, **kwargs: S before asserting its protocol or. + monkeypatch.setattr(metrics.subprocess, "run", lambda *args, **kwargs: SimpleNamespace( + # What: arrange returncode to json.dumps; why: the amd smi process vram parses multi gpu json scenario binds this 0 value to json.dumps's returncode input. + returncode=0, stdout=json.dumps(doc) + # What: arrange the monkeypatch.setattr call with subprocess and simple namespace; why: test_amd_smi_process_vram_parses_multi_gpu_json groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + )) + + # What: assert the expected metrics amd smi process vram == outcome; why: test metrics test amd smi process vram parses multi gpu json protects its regression by requiring this observable result after the exercised behavior. + assert metrics._amd_smi_process_vram() == { + # What: arrange 41 2 1024 3 512 1024 2 for the scenario; why: test metrics test amd smi process vram parses multi gpu json requires this concrete input or helper state before exercising the behavior under test. + 41: 2 * 1024**3 + 512 * 1024**2, + # What: assert that metrics amd smi process vram equals 41 2 1024 3 512 1024; why: this assertion protects the amd smi process vram parses multi gpu json regression after the test's arranged inputs and exercised call. + 42: 1_500_000_000, + # What: arrange the grouped source fragment for the scenario; why: test metrics test amd smi process vram parses multi gpu json requires this concrete input or helper state before exercising the behavior under test. + } + + +# What: define the test_amd_smi_process_vram_distinguishes_empty_from_unavailable test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the amd smi process vram distinguishes empty from unavailable outcome. +def test_amd_smi_process_vram_distinguishes_empty_from_unavailable(monkeypatch): + # What: arrange the exact monkeypatch setattr metrics subprocess run lambda args kwargs fixture fragment; why: the amd smi process vram distinguishes empty from unavailable scenario feeds this byte-preserved fragment through monkeypatch.setattr(metrics.subprocess, "run", lambda *args, **kwargs: S before asserting. + monkeypatch.setattr(metrics.subprocess, "run", lambda *args, **kwargs: SimpleNamespace( + # What: arrange returncode 0 stdout for the scenario; why: test metrics test amd smi process vram distinguishes empty from unavailable requires this concrete input or helper state before exercising the behavior under test. + returncode=0, stdout="[]" + # What: arrange the monkeypatch.setattr call with subprocess and simple namespace; why: test_amd_smi_process_vram_distinguishes_empty_from_unavailable groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + )) + # What: assert that metrics amd smi process vram equals group delimiter; why: this assertion protects the amd smi process vram distinguishes empty from unavailable regression after the test's arranged inputs and exercised call. + assert metrics._amd_smi_process_vram() == {} + + # What: arrange the exact monkeypatch setattr metrics subprocess run lambda args kwargs fixture fragment; why: the amd smi process vram distinguishes empty from unavailable scenario feeds this byte-preserved fragment through monkeypatch.setattr(metrics.subprocess, "run", lambda *args, **kwargs: S before asserting. + monkeypatch.setattr(metrics.subprocess, "run", lambda *args, **kwargs: SimpleNamespace( + # What: arrange returncode 1 stdout for the scenario; why: test metrics test amd smi process vram distinguishes empty from unavailable requires this concrete input or helper state before exercising the behavior under test. + returncode=1, stdout="" + # What: arrange the monkeypatch.setattr call with subprocess and simple namespace; why: test_amd_smi_process_vram_distinguishes_empty_from_unavailable groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + )) + # What: assert that metrics amd smi process vram is group delimiter; why: this assertion protects the amd smi process vram distinguishes empty from unavailable regression after the test's arranged inputs and exercised call. + assert metrics._amd_smi_process_vram() is None + + # What: arrange the exact monkeypatch setattr metrics subprocess run lambda args kwargs fixture fragment; why: the amd smi process vram distinguishes empty from unavailable scenario feeds this byte-preserved fragment through monkeypatch.setattr(metrics.subprocess, "run", lambda *args, **kwargs: S before asserting. + monkeypatch.setattr(metrics.subprocess, "run", lambda *args, **kwargs: SimpleNamespace( + # What: arrange the exact returncode stdout process info pid memory usage fixture fragment; why: the amd smi process vram distinguishes empty from unavailable scenario feeds this byte-preserved fragment through returncode=0, stdout='[{"process_info":{"pid":41,"memory_usage":{}}}]' before asserting its protocol. + returncode=0, stdout='[{"process_info":{"pid":41,"memory_usage":{}}}]' + # What: arrange the monkeypatch.setattr call with subprocess and simple namespace; why: test_amd_smi_process_vram_distinguishes_empty_from_unavailable groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + )) + # What: assert that metrics amd smi process vram is group delimiter; why: this assertion protects the amd smi process vram distinguishes empty from unavailable regression after the test's arranged inputs and exercised call. + assert metrics._amd_smi_process_vram() is None + + +# What: define the test_vram_measurement_falls_through_to_amd_smi test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the vram measurement falls through to amd smi outcome. +def test_vram_measurement_falls_through_to_amd_smi(monkeypatch): + # What: make DRM fdinfo unavailable in this fallback scenario; why: the test specifically protects progression through the legacy probe chain. + monkeypatch.setattr(metrics, "_drm_fdinfo_process_vram", lambda pids: None) + # What: arrange the exact monkeypatch setattr metrics nvml process vram lambda fixture fragment; why: the vram measurement falls through to amd smi scenario feeds this byte-preserved fragment through monkeypatch.setattr(metrics, "_nvml_process_vram", lambda: None) before asserting its protocol or parser result. + monkeypatch.setattr(metrics, "_nvml_process_vram", lambda: None) + # What: arrange the exact monkeypatch setattr metrics smi process vram lambda fixture fragment; why: the vram measurement falls through to amd smi scenario feeds this byte-preserved fragment through monkeypatch.setattr(metrics, "_smi_process_vram", lambda: {}) before asserting its protocol or parser result. + monkeypatch.setattr(metrics, "_smi_process_vram", dict) + # What: arrange the 41 field as 123; why: test_vram_measurement_falls_through_to_amd_smi carries 41 into monkeypatch.setattr(metrics, "_amd_smi_process_vram", lambda: {41: 123,. + monkeypatch.setattr(metrics, "_amd_smi_process_vram", lambda: {41: 123, 42: 456}) + + # What: assert that metrics vram measurement for pids 41 equals 123 true amd smi; why: this assertion protects the vram measurement falls through to amd smi regression after the test's arranged inputs and exercised call. + assert metrics._vram_measurement_for_pids([41]) == (123, True, "amd-smi") + + +# What: define the test_engine_footprint_reports_sources_and_availability test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the engine footprint reports sources and availability outcome. +def test_engine_footprint_reports_sources_and_availability(monkeypatch): + # What: arrange the exact monkeypatch setattr metrics osproc tree pids lambda pid pid fixture fragment; why: the engine footprint reports sources and availability scenario feeds this byte-preserved fragment through monkeypatch.setattr(metrics.osproc, "tree_pids", lambda pid: [pid, pid + before asserting its protoc. + monkeypatch.setattr(metrics.osproc, "tree_pids", lambda pid: [pid, pid + 1]) + # What: arrange the exact monkeypatch setattr metrics osproc read pss bytes if available lambda pid pid fixture f; why: the engine footprint reports sources and availability scenario feeds this byte-preserved fragment through monkeypatch.setattr(metrics.osproc, "read_pss_bytes_if_available", lambd before asserting. + monkeypatch.setattr(metrics.osproc, "read_pss_bytes_if_available", lambda pid: pid * 10) + # What: act by calling monkeypatch.setattr with metrics and vram measurement for pids and 1234 and true and amd smi; why: the engine footprint reports sources and availability scenario observes the monkeypatch.setattr return value during metrics vram measurement for pids lambda pids amd smi. + monkeypatch.setattr( + # What: arrange the exact metrics vram measurement for pids lambda pids amd smi fixture fragment; why: the engine footprint reports sources and availability scenario feeds this byte-preserved fragment through metrics, "_vram_measurement_for_pids", lambda pids: (1234, True, "amd-sm before asserting its protocol. + metrics, "_vram_measurement_for_pids", lambda pids: (1234, True, "amd-smi") + # What: arrange the monkeypatch.setattr call with metrics; why: test_engine_footprint_reports_sources_and_availability groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + + # What: assert the expected metrics engine footprint 10 == outcome; why: test metrics test engine footprint reports sources and availability protects its regression by requiring this observable result after the exercised behavior. + assert metrics.engine_footprint(10) == { + # What: arrange ramBytes 210 for the scenario; why: test metrics test engine footprint reports sources and availability requires this concrete input or helper state before exercising the behavior under test. + "ramBytes": 210, + # What: arrange vramBytes 1234 for the scenario; why: test metrics test engine footprint reports sources and availability requires this concrete input or helper state before exercising the behavior under test. + "vramBytes": 1234, + # What: arrange pids 10 11 for the scenario; why: test metrics test engine footprint reports sources and availability requires this concrete input or helper state before exercising the behavior under test. + "pids": [10, 11], + # What: arrange ramAvailable True for the scenario; why: test metrics test engine footprint reports sources and availability requires this concrete input or helper state before exercising the behavior under test. + "ramAvailable": True, + # What: arrange vramAvailable True for the scenario; why: test metrics test engine footprint reports sources and availability requires this concrete input or helper state before exercising the behavior under test. + "vramAvailable": True, + # What: arrange ramSource proc smaps rollup pss for the scenario; why: test metrics test engine footprint reports sources and availability requires this concrete input or helper state before exercising the behavior under test. + "ramSource": "proc-smaps-rollup-pss", + # What: arrange vramSource amd smi for the scenario; why: test metrics test engine footprint reports sources and availability requires this concrete input or helper state before exercising the behavior under test. + "vramSource": "amd-smi", + # What: arrange the grouped source fragment for the scenario; why: test metrics test engine footprint reports sources and availability requires this concrete input or helper state before exercising the behavior under test. + } + + +# What: define the test_engine_footprint_does_not_label_fallback_zero_as_measured test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the engine footprint does not label fallback zero as measured outcome. +def test_engine_footprint_does_not_label_fallback_zero_as_measured(monkeypatch): + # What: arrange the exact monkeypatch setattr metrics osproc tree pids lambda pid pid fixture fragment; why: the engine footprint does not label fallback zero as measured scenario feeds this byte-preserved fragment through monkeypatch.setattr(metrics.osproc, "tree_pids", lambda pid: [pid]) before asserting its pro. + monkeypatch.setattr(metrics.osproc, "tree_pids", lambda pid: [pid]) + # What: arrange the exact monkeypatch setattr metrics osproc read pss bytes if available lambda pid fixture fragm; why: the engine footprint does not label fallback zero as measured scenario feeds this byte-preserved fragment through monkeypatch.setattr(metrics.osproc, "read_pss_bytes_if_available", lambd before a. + monkeypatch.setattr(metrics.osproc, "read_pss_bytes_if_available", lambda pid: None) + # What: arrange the exact monkeypatch setattr metrics vram measurement for pids lambda pids fixture fragment; why: the engine footprint does not label fallback zero as measured scenario feeds this byte-preserved fragment through monkeypatch.setattr(metrics, "_vram_measurement_for_pids", lambda pids: before asserti. + monkeypatch.setattr(metrics, "_vram_measurement_for_pids", lambda pids: (0, False, None)) + + # What: act by calling metrics.engine_footprint and capture footprint; why: the engine footprint does not label fallback zero as measured test asserts the response, state, or failure produced by this call. + footprint = metrics.engine_footprint(10) + # What: assert that footprint ram bytes equals footprint vram bytes equals 0; why: this assertion protects the engine footprint does not label fallback zero as measured regression after the test's arranged inputs and exercised call. + assert footprint["ramBytes"] == footprint["vramBytes"] == 0 + # What: assert that footprint ram available is footprint vram available is false; why: this assertion protects the engine footprint does not label fallback zero as measured regression after the test's arranged inputs and exercised call. + assert footprint["ramAvailable"] is footprint["vramAvailable"] is False + # What: assert that footprint ram source is footprint vram source is group delimiter; why: this assertion protects the engine footprint does not label fallback zero as measured regression after the test's arranged inputs and exercised call. + assert footprint["ramSource"] is footprint["vramSource"] is None diff --git a/tests/daemon/test_performance.py b/tests/daemon/test_performance.py new file mode 100644 index 0000000000..b7bd54c84e --- /dev/null +++ b/tests/daemon/test_performance.py @@ -0,0 +1,132 @@ +# What: enable postponed evaluation of annotations; why: type hints in test_performance can reference runtime types without eager imports or forward-reference failures. +from __future__ import annotations + +# What: arrange from datetime import datetime timezone for the scenario; why: test performance requires this concrete input or helper state before exercising the behavior under test. +from datetime import datetime, timezone +# What: import threading for test performance sampler stops and reconfigures without old generation resuming using threading; why: test_performance_sampler_stops_and_reconfigures_without_old_generation_resuming uses threading event, making that imported dependency available to its named operation. +import threading +# What: import time for test performance sampler stops and reconfigures without old generation resuming using time; why: test_performance_sampler_stops_and_reconfigures_without_old_generation_resuming uses time sleep, making that imported dependency available to its named operation. +import time + +# What: arrange from freetoken daemon performance import PerformanceMonitor for the scenario; why: test performance requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.performance import PerformanceMonitor + + +# What: define the test_performance_history_is_one_hour_bounded_and_filterable test around local fixtures; why: this test groups the arrange, act, and assertions that protect the performance history is one hour bounded and filterable outcome. +def test_performance_history_is_one_hour_bounded_and_filterable(): + # What: arrange clock as 1700000000 0; why: the performance history is one hour bounded and filterable test consumes this named precondition before exercising the behavior. + clock = [1_700_000_000.0] + # What: arrange values as 10 and 20 and 30; why: the performance history is one hour bounded and filterable test consumes this named precondition before exercising the behavior. + values = [10, 20, 30] + # What: act by calling PerformanceMonitor and capture monitor; why: the performance history is one hour bounded and filterable test asserts the response, state, or failure produced by this call. + monitor = PerformanceMonitor( + # What: arrange the lambda portion of monitor; why: the performance history is one hour bounded and filterable scenario uses this clause to evaluate monitor as one grouped value. + lambda: { + # What: arrange the ram bytes field as pop and values and 0; why: test_performance_history_is_one_hour_bounded_and_filterable carries ram bytes through monitor into monitor sample once. + "ramBytes": values.pop(0), "vramBytes": 7, + # What: arrange the ram available field as true; why: test_performance_history_is_one_hour_bounded_and_filterable carries ram available through monitor into monitor sample once. + "ramAvailable": True, "vramAvailable": False, + # What: arrange the ram source field as test pss; why: test_performance_history_is_one_hour_bounded_and_filterable carries ram source through monitor into monitor sample once. + "ramSource": "test-pss", "vramSource": None, + # What: arrange the pids field as 123; why: test_performance_history_is_one_hour_bounded_and_filterable carries pids through monitor into monitor sample once. + "pids": [123], + # What: arrange the monitor mapping with ram bytes and vram bytes and ram available and vram available and ram source; why: test_performance_history_is_one_hour_bounded_and_filterable groups the supplied clauses as one monitor mapping before its value is consumed. + }, + # What: arrange every s to PerformanceMonitor; why: the performance history is one hour bounded and filterable scenario binds this 1800 value to PerformanceMonitor's every s input. + every_s=1800, + # What: arrange wall now to PerformanceMonitor; why: the performance history is one hour bounded and filterable scenario binds this clock and 0 value to PerformanceMonitor's wall now input. + wall_now=lambda: clock[0], + # What: arrange the PerformanceMonitor call with every s and wall now; why: test_performance_history_is_one_hour_bounded_and_filterable groups the supplied clauses as one PerformanceMonitor call before its value is consumed. + ) + # What: act across the computed value to perform timestamp and clock; why: the performance history is one hour bounded and filterable scenario repeats the body only while or for the loop header admits an iteration. + for timestamp in (1_700_000_000.0, 1_700_001_800.0, 1_700_003_600.0): + # What: arrange clock entry as timestamp; why: the performance history is one hour bounded and filterable test consumes this named precondition before exercising the behavior. + clock[0] = timestamp + # What: act by calling monitor.sample_once with the declared inputs; why: the performance history is one hour bounded and filterable scenario observes the monitor.sample_once return value during result monitor current. + monitor.sample_once() + + # What: act by calling monitor.current and capture result; why: the performance history is one hour bounded and filterable test asserts the response, state, or failure produced by this call. + result = monitor.current() + # What: assert that row ram bytes for row in result equals 20 30; why: this assertion protects the performance history is one hour bounded and filterable regression after the test's arranged inputs and exercised call. + assert [row["ram_bytes"] for row in result["sys_stats"]] == [20, 30] + # What: assert that result gpu stats equals group delimiter; why: this assertion protects the performance history is one hour bounded and filterable regression after the test's arranged inputs and exercised call. + assert result["gpu_stats"] == [] + # What: assert that result retention s equals 3600; why: this assertion protects the performance history is one hour bounded and filterable regression after the test's arranged inputs and exercised call. + assert result["retentionS"] == 3600 + # What: assert that pids is absent from result sys stats 0; why: this assertion protects the performance history is one hour bounded and filterable regression after the test's arranged inputs and exercised call. + assert "pids" not in result["sys_stats"][0] + # What: assert that result sys stats 0 scope equals engine process tree; why: this assertion protects the performance history is one hour bounded and filterable regression after the test's arranged inputs and exercised call. + assert result["sys_stats"][0]["scope"] == "engine-process-tree" + + # What: act by calling datetime.fromtimestamp and capture after; why: the performance history is one hour bounded and filterable test asserts the response, state, or failure produced by this call. + after = datetime.fromtimestamp(1_700_001_800.0, timezone.utc) + # What: assert that row ram bytes for row in monitor current equals 30; why: this assertion protects the performance history is one hour bounded and filterable regression after the test's arranged inputs and exercised call. + assert [row["ram_bytes"] for row in monitor.current(after=after)["sys_stats"]] == [30] + + +# What: define the test_performance_probe_failure_is_generic_and_recovers test around local fixtures; why: this test groups the arrange, act, and assertions that protect the performance probe failure is generic and recovers outcome. +def test_performance_probe_failure_is_generic_and_recovers(): + # What: arrange fail as true; why: the performance probe failure is generic and recovers test consumes this named precondition before exercising the behavior. + fail = [True] + + # What: define the sample test helper around captured fixture state; why: the performance probe failure is generic and recovers scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def sample(): + # What: act on fail before runtime error; why: the performance probe failure is generic and recovers scenario admits runtime error only for this predicate and excludes the opposite state. + if fail[0]: + # What: raise RuntimeError for the caller; why: sample stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("private probe detail") + # What: return no value from the sample test helper; why: the performance probe failure is generic and recovers scenario uses this helper result in its subsequent act or assertion. + return {} + + # What: act by calling PerformanceMonitor and capture monitor; why: the performance probe failure is generic and recovers test asserts the response, state, or failure produced by this call. + monitor = PerformanceMonitor(sample) + # What: act by calling monitor.sample_once with the declared inputs; why: the performance probe failure is generic and recovers scenario observes the monitor.sample_once return value during assert monitor current error sample failed. + monitor.sample_once() + # What: assert that monitor current error equals sample failed; why: this assertion protects the performance probe failure is generic and recovers regression after the test's arranged inputs and exercised call. + assert monitor.current()["error"] == "sample_failed" + # What: arrange fail entry as false; why: the performance probe failure is generic and recovers test consumes this named precondition before exercising the behavior. + fail[0] = False + # What: act by calling monitor.sample_once with the declared inputs; why: the performance probe failure is generic and recovers scenario observes the monitor.sample_once return value during assert monitor current healthy is. + monitor.sample_once() + # What: assert that monitor current healthy is true; why: this assertion protects the performance probe failure is generic and recovers regression after the test's arranged inputs and exercised call. + assert monitor.current()["healthy"] is True + # What: assert that monitor current error is group delimiter; why: this assertion protects the performance probe failure is generic and recovers regression after the test's arranged inputs and exercised call. + assert monitor.current()["error"] is None + + +# What: define the test_performance_sampler_stops_and_reconfigures_without_old_generation_resuming test around local fixtures; why: this test groups the arrange, act, and assertions that protect the performance sampler stops and reconfigures without old generation resuming outcome. +def test_performance_sampler_stops_and_reconfigures_without_old_generation_resuming(): + # What: act by calling threading.Event and capture sampled; why: the performance sampler stops and reconfigures without old generation resuming test asserts the response, state, or failure produced by this call. + sampled = threading.Event() + # What: arrange calls as the fixture input; why: the performance sampler stops and reconfigures without old generation resuming test consumes this named precondition before exercising the behavior. + calls = [] + + # What: define the sample test helper around captured fixture state; why: the performance sampler stops and reconfigures without old generation resuming scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def sample(): + # What: act by calling calls.append with len and calls; why: the performance sampler stops and reconfigures without old generation resuming scenario observes the calls.append return value during sampled set. + calls.append(len(calls)) + # What: act by calling sampled.set with the declared inputs; why: the performance sampler stops and reconfigures without old generation resuming scenario observes the sampled.set return value during return. + sampled.set() + # What: return no value from the sample test helper; why: the performance sampler stops and reconfigures without old generation resuming scenario uses this helper result in its subsequent act or assertion. + return {} + + # What: act by calling PerformanceMonitor and capture monitor; why: the performance sampler stops and reconfigures without old generation resuming test asserts the response, state, or failure produced by this call. + monitor = PerformanceMonitor(sample, every_s=0.01) + # What: act by calling monitor.start with the declared inputs; why: the performance sampler stops and reconfigures without old generation resuming scenario observes the monitor.start return value during assert sampled wait. + monitor.start() + # What: assert that sampled wait 1; why: this assertion protects the performance sampler stops and reconfigures without old generation resuming regression after the test's arranged inputs and exercised call. + assert sampled.wait(1) + # What: act by calling monitor.reconfigure with 0 02 and false; why: the performance sampler stops and reconfigures without old generation resuming scenario observes the monitor.reconfigure return value during sampled clear. + monitor.reconfigure(0.02, False) + # What: act by calling sampled.clear with the declared inputs; why: the performance sampler stops and reconfigures without old generation resuming scenario observes the sampled.clear return value during assert sampled wait. + sampled.clear() + # What: assert that sampled wait 1; why: this assertion protects the performance sampler stops and reconfigures without old generation resuming regression after the test's arranged inputs and exercised call. + assert sampled.wait(1) + # What: act by calling monitor.stop with the declared inputs; why: the performance sampler stops and reconfigures without old generation resuming scenario observes the monitor.stop return value during stopped at len calls. + monitor.stop() + # What: act by calling len and capture stopped at; why: the performance sampler stops and reconfigures without old generation resuming test asserts the response, state, or failure produced by this call. + stopped_at = len(calls) + # What: act by calling time.sleep with 0 05; why: the performance sampler stops and reconfigures without old generation resuming scenario observes the time.sleep return value during assert len calls stopped at. + time.sleep(0.05) + # What: assert that len calls equals stopped at; why: this assertion protects the performance sampler stops and reconfigures without old generation resuming regression after the test's arranged inputs and exercised call. + assert len(calls) == stopped_at diff --git a/tests/daemon/test_real_process_recovery.py b/tests/daemon/test_real_process_recovery.py new file mode 100644 index 0000000000..777b383bb1 --- /dev/null +++ b/tests/daemon/test_real_process_recovery.py @@ -0,0 +1,599 @@ +"""Linux CPU integration: real process groups, HTTP probes, and durable recovery. + +No model runtime or production endpoint is used. The subprocess below is a small +test HTTP server, not a substitute for the separate GPU qualification gates. +""" +# What: document linux cpu integration real process groups in the test_real_process_recovery docstring; why: introspection and maintainers read this exact docstring fragment to understand test real process recovery behavior without executing it. +# What: document no model runtime or production endpoint in the test_real_process_recovery docstring; why: introspection and maintainers read this exact docstring fragment to understand test real process recovery behavior without executing it. +# What: document test http server not a substitute in the test_real_process_recovery docstring; why: introspection and maintainers read this exact docstring fragment to understand test real process recovery behavior without executing it. +# What: preserve the paragraph boundary in the the test_real_process_recovery docstring; why: introspection and maintainers read this paragraph break to understand test real process recovery behavior without executing it. + +# What: import json for json get using json; why: json_get uses json load, making that imported dependency available to its named operation. +import json +# What: import os for test real readiness rollback and process group cleanup using os; why: test_real_readiness_rollback_and_process_group_cleanup uses os killpg, making that imported dependency available to its named operation. +import os +# What: import socket for test real readiness rollback and process group cleanup using socket; why: test_real_readiness_rollback_and_process_group_cleanup uses socket socket, making that imported dependency available to its named operation. +import socket +# What: import subprocess for test native router binds and routes a real readopted child using subprocess; why: test_native_router_binds_and_routes_a_real_readopted_child uses subprocess popen, making that imported dependency available to its named operation. +import subprocess +# What: import sys for module initialization using sys; why: module initialization uses sys platform, making that imported dependency available to its named operation. +import sys +# What: import time for test native router binds and routes a real readopted child using time; why: test_native_router_binds_and_routes_a_real_readopted_child uses time monotonic, making that imported dependency available to its named operation. +import time +# What: import urllib request for json get using urllib and request; why: json_get uses urllib request urlopen, making that imported dependency available to its named operation. +import urllib.request +# What: import thread pool executor for test native router supervises a real child and relays sse using concurrent and futures and thread pool executor; why: test_native_router_supervises_a_real_child_and_relays_sse uses thread pool executor, making that imported dependency available to its named operation. +from concurrent.futures import ThreadPoolExecutor + +# What: import pytest for module initialization using pytest; why: module initialization uses pytest mark skipif, making that imported dependency available to its named operation. +import pytest +# What: import test client for test native router uses fresh dynamic ports for real child reactivation using fastapi and testclient and test client; why: test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation uses test client, making that imported dependency available to its named operation. +from fastapi.testclient import TestClient + +# What: import build app for test native router supervises a real child and relays sse using freetoken and daemon and app and build app; why: test_native_router_supervises_a_real_child_and_relays_sse uses build app, making that imported dependency available to its named operation. +from freetoken.daemon.app import build_app +# What: arrange from freetoken daemon catalog import ModelCatalog ModelProfile RouterSettings for the scenario; why: test real process recovery requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.catalog import ModelCatalog, ModelProfile, RouterSettings +# What: import log ring for test real readiness rollback and process group cleanup using freetoken and daemon and logring and log ring; why: test_real_readiness_rollback_and_process_group_cleanup uses log ring, making that imported dependency available to its named operation. +from freetoken.daemon.logring import LogRing +# What: arrange from freetoken daemon pidfile import ServeState ServeStateStore for the scenario; why: test real process recovery requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.pidfile import ServeState, ServeStateStore +# What: import serve probe for test real readiness rollback and process group cleanup using freetoken and daemon and proxy and serve probe; why: test_real_readiness_rollback_and_process_group_cleanup uses serve probe, making that imported dependency available to its named operation. +from freetoken.daemon.proxy import ServeProbe +# What: import wait for ready for test real readiness rollback and process group cleanup using freetoken and daemon and readiness and wait for ready; why: test_real_readiness_rollback_and_process_group_cleanup uses wait for ready, making that imported dependency available to its named operation. +from freetoken.daemon.readiness import wait_for_ready +# What: arrange from freetoken daemon router import RoutingCoordinator for the scenario; why: test real process recovery requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.router import RoutingCoordinator +# What: arrange from freetoken daemon serve manager import AdoptedChild PopenChild ServeManager for the scenario; why: test real process recovery requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.serve_manager import AdoptedChild, PopenChild, ServeManager + + +# What: act by calling pytest.mark.skipif and capture pytestmark; why: the real process recovery test asserts the response, state, or failure produced by this call. +pytestmark = pytest.mark.skipif(sys.platform != "linux", reason="Linux process-group integration") + +# What: arrange server as import and json and os and signal; why: the real process recovery test consumes this named precondition before exercising the behavior. +# What: arrange the exact import json os signal subprocess sys fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact from http server import base httprequest handler httpserver fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact model port resistant sys argv fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact worker subprocess popen sys executable c import time fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact if resistant yes fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact signal signal signal sigterm signal sig ign fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact class handler base httprequest handler fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact def log message args pass fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact def do get fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact if self path health fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact body status error if model bad fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact maintenance serving worker pid worker pid fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact else fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact body requests prompt tokens total completion tokens total fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact uptime s reachable fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact data json dumps body encode fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact self send response fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact self send header content length str len data fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact self end headers fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact self wfile write data fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact def do post fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact if self path v1 chat completions fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact length int self headers get content length fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact request json loads self rfile read length or b fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact body data model s echo s fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact data done n n model json dumps fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact data body encode fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact self send response fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact self send header content type text event stream fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact self send header content length str len data fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact self end headers fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact self wfile write data fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact return fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact self send error fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact httpserver int port handler serve forever fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +# What: arrange the exact the grouped expression fixture fragment; why: the real process recovery scenario feeds this byte-preserved fragment through server before asserting its protocol or parser result. +SERVER = r''' +import json, os, signal, subprocess, sys +from http.server import BaseHTTPRequestHandler, HTTPServer +model, port, resistant = sys.argv[1:] +worker = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"]) +if resistant == "yes": + signal.signal(signal.SIGTERM, signal.SIG_IGN) +class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): pass + def do_GET(self): + if self.path == "/health": + body = {"status": "error" if model == "bad" else "ok", + "maintenance": "serving", "worker_pid": worker.pid} + else: + body = {"requests": {"promptTokensTotal": 0, "completionTokensTotal": 0}, + "uptimeS": 0, "reachable": True} + data = json.dumps(body).encode() + self.send_response(200) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + def do_POST(self): + if self.path == "/v1/chat/completions": + length = int(self.headers.get("Content-Length", "0")) + request = json.loads(self.rfile.read(length) or b"{}") + body = ('data: {"model":"%s","echo":%s}\n\n' + 'data: [DONE]\n\n') % (model, json.dumps(request.get("model"))) + data = body.encode() + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + return + self.send_error(404) +HTTPServer(("127.0.0.1", int(port)), Handler).serve_forever() +''' + + +# What: define the json_get test helper around port and path; why: the json get scenario calls this helper to produce or observe the exact behavior checked by its assertions. +def json_get(port, path): + # What: enter the urllib.request.urlopen managed context before return json load response; why: json_get releases this resource or lock after return json load response on both success and failure paths. + with urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout=2) as response: + # What: return load and response and json from the json_get test helper; why: the json get scenario uses this helper result in its subsequent act or assertion. + return json.load(response) + + +# What: define the running test helper around pid; why: the running scenario calls this helper to produce or observe the exact behavior checked by its assertions. +def running(pid): + # What: establish the handler boundary for the protected operation; why: running routes failures to file not found error and process lookup error while preserving cleanup and success flow. + try: + # Zombies have exited even if the host's init has not reaped them yet. + # What: enter the open managed context before return source read rsplit split z; why: running releases this resource or lock after return source read rsplit split z on both success and failure paths. + with open(f"/proc/{pid}/stat") as source: + # What: return split and rsplit and read and source and z from the running test helper; why: the running scenario uses this helper result in its subsequent act or assertion. + return source.read().rsplit(")", 1)[1].split()[0] != "Z" + # What: handle file not found error and process lookup error by return false; why: running converts that failure into this concrete recovery, response, or cleanup behavior. + except (FileNotFoundError, ProcessLookupError): + # What: return false from the running test helper; why: the running scenario uses this helper result in its subsequent act or assertion. + return False + + +# What: parameterize test_real_readiness_rollback_and_process_group_cleanup with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test real readiness rollback and process group cleanup. +@pytest.mark.parametrize("resistant", [False, True]) +# What: define the test_real_readiness_rollback_and_process_group_cleanup test around tmp path and resistant; why: this test groups the arrange, act, and assertions that protect the real readiness rollback and process group cleanup outcome. +def test_real_readiness_rollback_and_process_group_cleanup(tmp_path, resistant): + # What: enter the socket.socket managed context before reservation bind; why: test_real_readiness_rollback_and_process_group_cleanup releases this resource or lock after reservation bind on both success and failure paths. + with socket.socket() as reservation: + # What: arrange the exact reservation bind fixture fragment; why: the real readiness rollback and process group cleanup scenario feeds this byte-preserved fragment through reservation.bind(("127.0.0.1", 0)) before asserting its protocol or parser result. + reservation.bind(("127.0.0.1", 0)) + # What: act by calling reservation.getsockname and capture port; why: the real readiness rollback and process group cleanup test asserts the response, state, or failure produced by this call. + port = reservation.getsockname()[1] + # What: arrange children as the fixture input; why: the real readiness rollback and process group cleanup test consumes this named precondition before exercising the behavior. + children = [] + # What: arrange workers as the fixture input; why: the real readiness rollback and process group cleanup test consumes this named precondition before exercising the behavior. + workers = [] + + # What: define the spawn test helper around model and actual port and args; why: the real readiness rollback and process group cleanup scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def spawn(model, actual_port, args): + # What: act by calling subprocess.Popen and capture proc; why: the real readiness rollback and process group cleanup test asserts the response, state, or failure produced by this call. + proc = subprocess.Popen([sys.executable, "-u", "-c", SERVER, model, str(actual_port), + # What: arrange start new session to subprocess.Popen; why: the real readiness rollback and process group cleanup scenario binds this true value to subprocess.Popen's start new session input. + "yes" if resistant else "no"], start_new_session=True, + # What: arrange stdin to subprocess.Popen; why: the real readiness rollback and process group cleanup scenario binds this devnull and subprocess value to subprocess.Popen's stdin input. + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + # What: arrange stderr to subprocess.Popen; why: the real readiness rollback and process group cleanup scenario binds this devnull and subprocess value to subprocess.Popen's stderr input. + stderr=subprocess.DEVNULL) + # What: act by calling PopenChild and capture child; why: the real readiness rollback and process group cleanup test asserts the response, state, or failure produced by this call. + child = PopenChild(proc, None) + # What: act by calling children.append with child; why: the real readiness rollback and process group cleanup scenario observes the children.append return value during return child. + children.append(child) + # What: return child from the spawn test helper; why: the real readiness rollback and process group cleanup scenario uses this helper result in its subsequent act or assertion. + return child + + # What: act by calling ServeStateStore and capture store; why: the real readiness rollback and process group cleanup test asserts the response, state, or failure produced by this call. + store = ServeStateStore(str(tmp_path / "serve.json")) + # What: act by calling ServeManager and capture manager; why: the real readiness rollback and process group cleanup test asserts the response, state, or failure produced by this call. + manager = ServeManager(LogRing(), store, spawn_fn=spawn, apply_oom=False, + # What: arrange grace s to ServeManager; why: the real readiness rollback and process group cleanup scenario binds this 0 2 value to ServeManager's grace s input. + grace_s=0.2, reap_wait_s=3, + # What: arrange the p input for test_real_readiness_rollback_and_process_group_cleanup; why: test_real_readiness_rollback_and_process_group_cleanup consumes p during signature binding, so callers must bind it with the other signature inputs. + read_stats=lambda p: json_get(p, "/v1/stats")) + # What: act by calling ServeProbe and capture probe; why: the real readiness rollback and process group cleanup test asserts the response, state, or failure produced by this call. + probe = ServeProbe() + # What: establish the handler boundary for the protected operation; why: test_real_readiness_rollback_and_process_group_cleanup routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: act by calling manager.start and capture first; why: the real readiness rollback and process group cleanup test asserts the response, state, or failure produced by this call. + first = manager.start("good", port, ["original-argument"]) + # What: assert that wait for ready manager probe pid first pid port port timeout s 5; why: this assertion protects the real readiness rollback and process group cleanup regression after the test's arranged inputs and exercised call. + assert wait_for_ready(manager, probe, pid=first["pid"], port=port, timeout_s=5)["ready"] + # What: arrange the exact workers append json get port health worker pid fixture fragment; why: the real readiness rollback and process group cleanup scenario feeds this byte-preserved fragment through workers.append(json_get(port, "/health")["worker_pid"]) before asserting its protocol or parser result. + workers.append(json_get(port, "/health")["worker_pid"]) + # What: act by calling manager.switch_for_readiness and capture replacement and ticket; why: the real readiness rollback and process group cleanup test asserts the response, state, or failure produced by this call. + replacement, ticket = manager.switch_for_readiness("bad", port) + # What: act by calling wait_for_ready and capture failed; why: the real readiness rollback and process group cleanup test asserts the response, state, or failure produced by this call. + failed = wait_for_ready(manager, probe, pid=replacement["pid"], port=port, timeout_s=5) + # What: assert that failed reason equals engine error; why: this assertion protects the real readiness rollback and process group cleanup regression after the test's arranged inputs and exercised call. + assert failed["reason"] == "engine-error" + # What: arrange the exact workers append json get port health worker pid fixture fragment; why: the real readiness rollback and process group cleanup scenario feeds this byte-preserved fragment through workers.append(json_get(port, "/health")["worker_pid"]) before asserting its protocol or parser result. + workers.append(json_get(port, "/health")["worker_pid"]) + # What: act by calling manager.recover_switch and capture recovery; why: the real readiness rollback and process group cleanup test asserts the response, state, or failure produced by this call. + recovery = manager.recover_switch(ticket) + # What: assert that recovery launched; why: this assertion protects the real readiness rollback and process group cleanup regression after the test's arranged inputs and exercised call. + assert recovery["launched"] + # What: assert that wait for ready manager probe pid recovery pid port port timeout s 5; why: this assertion protects the real readiness rollback and process group cleanup regression after the test's arranged inputs and exercised call. + assert wait_for_ready(manager, probe, pid=recovery["pid"], port=port, timeout_s=5)["ready"] + # What: arrange the exact workers append json get port health worker pid fixture fragment; why: the real readiness rollback and process group cleanup scenario feeds this byte-preserved fragment through workers.append(json_get(port, "/health")["worker_pid"]) before asserting its protocol or parser result. + workers.append(json_get(port, "/health")["worker_pid"]) + # What: act by calling store.load and capture saved; why: the real readiness rollback and process group cleanup test asserts the response, state, or failure produced by this call. + saved = store.load() + # What: assert that saved pid equals recovery pid and saved model equals good; why: this assertion protects the real readiness rollback and process group cleanup regression after the test's arranged inputs and exercised call. + assert saved.pid == recovery["pid"] and saved.model == "good" + # What: assert that saved args equals original argument; why: this assertion protects the real readiness rollback and process group cleanup regression after the test's arranged inputs and exercised call. + assert saved.args == ["original-argument"] + # What: assert that len manager pending accounting equals 2; why: this assertion protects the real readiness rollback and process group cleanup regression after the test's arranged inputs and exercised call. + assert len(manager.pending_accounting()) == 2 + # What: act by calling manager.stop with the declared inputs; why: the real readiness rollback and process group cleanup scenario observes the manager.stop return value during assert store load is. + manager.stop() + # What: assert that store load is group delimiter; why: this assertion protects the real readiness rollback and process group cleanup regression after the test's arranged inputs and exercised call. + assert store.load() is None + # What: assert that all child reaped is set and child proc poll is not for child in children; why: this assertion protects the real readiness rollback and process group cleanup regression after the test's arranged inputs and exercised call. + assert all(child.reaped.is_set() and child.proc.poll() is not None for child in children) + # What: assert that all child proc returncode equals 9 if resistant else 15 for child; why: this assertion protects the real readiness rollback and process group cleanup regression after the test's arranged inputs and exercised call. + assert all(child.proc.returncode == (-9 if resistant else -15) for child in children) + # What: act by calling time.monotonic and capture deadline; why: the real readiness rollback and process group cleanup test asserts the response, state, or failure produced by this call. + deadline = time.monotonic() + 3 + # What: act across any and deadline and monotonic and running and pid to perform sleep and time; why: the real readiness rollback and process group cleanup scenario repeats the body only while or for the loop header admits an iteration. + while any(running(pid) for pid in workers) and time.monotonic() < deadline: + # What: act by calling time.sleep with 0 05; why: the real readiness rollback and process group cleanup scenario observes the time.sleep return value during assert not any running pid for. + time.sleep(0.05) + # What: assert that any running pid for pid in workers is false; why: this assertion protects the real readiness rollback and process group cleanup regression after the test's arranged inputs and exercised call. + assert not any(running(pid) for pid in workers) + # What: enter the socket.socket managed context before assert connection connect ex port; why: test_real_readiness_rollback_and_process_group_cleanup releases this resource or lock after assert connection connect ex port on both success and failure paths. + with socket.socket() as connection: + # What: assert that connection connect ex 127 0 0 1 port differs from 0; why: this assertion protects the real readiness rollback and process group cleanup regression after the test's arranged inputs and exercised call. + assert connection.connect_ex(("127.0.0.1", port)) != 0 + # What: run test owned sessions only always clean up on every exit path; why: test_real_readiness_rollback_and_process_group_cleanup performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # Test-owned sessions only. Always clean up even if an assertion fails. + # What: act across children to perform process lookup error and killpg and pid and os and child; why: the real readiness rollback and process group cleanup scenario repeats the body only while or for the loop header admits an iteration. + for child in children: + # What: establish the handler boundary for the protected operation; why: test_real_readiness_rollback_and_process_group_cleanup routes failures to process lookup error while preserving cleanup and success flow. + try: + # What: act by calling os.killpg with pid and child and 9; why: the real readiness rollback and process group cleanup scenario observes the os.killpg return value during except process lookup error. + os.killpg(child.pid, 9) + # What: handle process lookup error by pass; why: test_real_readiness_rollback_and_process_group_cleanup converts that failure into this concrete recovery, response, or cleanup behavior. + except ProcessLookupError: + # What: ignore the anticipated exception handled by this branch; why: spawn continues its retry or cleanup path instead of re-raising that transient failure. + pass + # What: act on poll and proc and child before wait and proc and child; why: the real readiness rollback and process group cleanup scenario admits wait and proc and child only for this predicate and excludes the opposite state. + if child.proc.poll() is None: + # What: arrange timeout to child.proc.wait; why: the real readiness rollback and process group cleanup scenario binds this 3 value to child.proc.wait's timeout input. + child.proc.wait(timeout=3) + + +# What: define the test_native_router_supervises_a_real_child_and_relays_sse test around tmp path; why: this test groups the arrange, act, and assertions that protect the native router supervises a real child and relays sse outcome. +def test_native_router_supervises_a_real_child_and_relays_sse(tmp_path): + # What: enter the socket.socket managed context before reservation bind; why: test_native_router_supervises_a_real_child_and_relays_sse releases this resource or lock after reservation bind on both success and failure paths. + with socket.socket() as reservation: + # What: arrange the exact reservation bind fixture fragment; why: the native router supervises a real child and relays sse scenario feeds this byte-preserved fragment through reservation.bind(("127.0.0.1", 0)) before asserting its protocol or parser result. + reservation.bind(("127.0.0.1", 0)) + # What: act by calling reservation.getsockname and capture port; why: the native router supervises a real child and relays sse test asserts the response, state, or failure produced by this call. + port = reservation.getsockname()[1] + # What: arrange children as the fixture input; why: the native router supervises a real child and relays sse test consumes this named precondition before exercising the behavior. + children = [] + + # What: define the spawn test helper around model and actual port and args; why: the native router supervises a real child and relays sse scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def spawn(model, actual_port, args): + # What: act by calling subprocess.Popen and capture proc; why: the native router supervises a real child and relays sse test asserts the response, state, or failure produced by this call. + proc = subprocess.Popen( + # What: act by calling str with actual port; why: the native router supervises a real child and relays sse scenario observes the str return value during start new session stdin subprocess devnull stdout subprocess devnull. + [sys.executable, "-u", "-c", SERVER, model, str(actual_port), "no"], + # What: arrange start new session to subprocess.Popen; why: the native router supervises a real child and relays sse scenario binds this true value to subprocess.Popen's start new session input. + start_new_session=True, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + # What: arrange stderr to subprocess.Popen; why: the native router supervises a real child and relays sse scenario binds this devnull and subprocess value to subprocess.Popen's stderr input. + stderr=subprocess.DEVNULL, + # What: arrange the grouped source fragment for the scenario; why: test real process recovery test native router supervises a real child and relays sse requires this concrete input or helper state before exercising the behavior under test. + ) + # What: act by calling PopenChild and capture child; why: the native router supervises a real child and relays sse test asserts the response, state, or failure produced by this call. + child = PopenChild(proc, None) + # What: act by calling children.append with child; why: the native router supervises a real child and relays sse scenario observes the children.append return value during return child. + children.append(child) + # What: return child from the spawn test helper; why: the native router supervises a real child and relays sse scenario uses this helper result in its subsequent act or assertion. + return child + + # What: act by calling ServeStateStore and capture store; why: the native router supervises a real child and relays sse test asserts the response, state, or failure produced by this call. + store = ServeStateStore(str(tmp_path / "serve.json")) + # What: act by calling ServeManager and capture manager; why: the native router supervises a real child and relays sse test asserts the response, state, or failure produced by this call. + manager = ServeManager(LogRing(), store, spawn_fn=spawn, apply_oom=False, + # What: arrange grace s to ServeManager; why: the native router supervises a real child and relays sse scenario binds this 0 2 value to ServeManager's grace s input. + grace_s=0.2, reap_wait_s=3, + # What: arrange the p input for test_native_router_supervises_a_real_child_and_relays_sse; why: test_native_router_supervises_a_real_child_and_relays_sse consumes p during signature binding, so callers must bind it with the other signature inputs. + read_stats=lambda p: json_get(p, "/v1/stats")) + # What: act by calling ServeProbe and capture probe; why: the native router supervises a real child and relays sse test asserts the response, state, or failure produced by this call. + probe = ServeProbe() + # What: act by calling ModelCatalog and capture catalog; why: the native router supervises a real child and relays sse test asserts the response, state, or failure produced by this call. + catalog = ModelCatalog( + # What: arrange the good field as model profile and port and good and good; why: test_native_router_supervises_a_real_child_and_relays_sse carries good through catalog into lifecycle pool lifecycle proxy pool proxy catalog catalog. + {"good": ModelProfile("good", "good", (), port=port)}, + # What: arrange settings to RouterSettings; why: the native router supervises a real child and relays sse scenario binds this router settings and true value to RouterSettings's settings input. + settings=RouterSettings(send_loading_state=True), + # What: arrange the ModelCatalog call with settings; why: test_native_router_supervises_a_real_child_and_relays_sse groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: establish the handler boundary for the protected operation; why: test_native_router_supervises_a_real_child_and_relays_sse routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_native_router_supervises_a_real_child_and_relays_sse releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(2) as proxy: + # What: act by calling build_app and capture app; why: the native router supervises a real child and relays sse test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_native_router_supervises_a_real_child_and_relays_sse; why: test_native_router_supervises_a_real_child_and_relays_sse consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=probe, footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the native router supervises a real child and relays sse scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_native_router_supervises_a_real_child_and_relays_sse groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture response; why: the native router supervises a real child and relays sse test asserts the response, state, or failure produced by this call. + response = TestClient(app).post( + # What: arrange the model field as good; why: test_native_router_supervises_a_real_child_and_relays_sse sends this field through response so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "good", "stream": True}, + # What: arrange the operation.post call with json; why: test_native_router_supervises_a_real_child_and_relays_sse groups the supplied clauses as one operation.post call before its value is consumed. + ) + # What: assert that response status code equals 200; why: this assertion protects the native router supervises a real child and relays sse regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + # What: assert that b reasoning content freetoken swap loading model good is present in response content; why: this assertion protects the native router supervises a real child and relays sse regression after the test's arranged inputs and exercised call. + assert b'"reasoning_content":"freetoken-swap loading model: good\\n"' in response.content + # What: assert that b model good is present in response content; why: this assertion protects the native router supervises a real child and relays sse regression after the test's arranged inputs and exercised call. + assert b'"model":"good"' in response.content + # What: assert that response content endswith b data done n n; why: this assertion protects the native router supervises a real child and relays sse regression after the test's arranged inputs and exercised call. + assert response.content.endswith(b"data: [DONE]\n\n") + # What: assert that manager status running is true; why: this assertion protects the native router supervises a real child and relays sse regression after the test's arranged inputs and exercised call. + assert manager.status()["running"] is True + # What: assert that store load is not group delimiter; why: this assertion protects the native router supervises a real child and relays sse regression after the test's arranged inputs and exercised call. + assert store.load() is not None + # What: act by calling manager.stop with the declared inputs; why: the native router supervises a real child and relays sse scenario observes the manager.stop return value during assert store load is. + manager.stop() + # What: assert that store load is group delimiter; why: this assertion protects the native router supervises a real child and relays sse regression after the test's arranged inputs and exercised call. + assert store.load() is None + # What: assert that children 0 reaped is set; why: this assertion protects the native router supervises a real child and relays sse regression after the test's arranged inputs and exercised call. + assert children[0].reaped.is_set() + # What: run for child in children on every exit path; why: test_native_router_supervises_a_real_child_and_relays_sse performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: act across children to perform process lookup error and killpg and pid and os and child; why: the native router supervises a real child and relays sse scenario repeats the body only while or for the loop header admits an iteration. + for child in children: + # What: establish the handler boundary for the protected operation; why: test_native_router_supervises_a_real_child_and_relays_sse routes failures to process lookup error while preserving cleanup and success flow. + try: + # What: act by calling os.killpg with pid and child and 9; why: the native router supervises a real child and relays sse scenario observes the os.killpg return value during except process lookup error. + os.killpg(child.pid, 9) + # What: handle process lookup error by pass; why: test_native_router_supervises_a_real_child_and_relays_sse converts that failure into this concrete recovery, response, or cleanup behavior. + except ProcessLookupError: + # What: ignore the anticipated exception handled by this branch; why: spawn continues its retry or cleanup path instead of re-raising that transient failure. + pass + # What: act on poll and proc and child before wait and proc and child; why: the native router supervises a real child and relays sse scenario admits wait and proc and child only for this predicate and excludes the opposite state. + if child.proc.poll() is None: + # What: arrange timeout to child.proc.wait; why: the native router supervises a real child and relays sse scenario binds this 3 value to child.proc.wait's timeout input. + child.proc.wait(timeout=3) + + +# What: define the test_native_router_binds_and_routes_a_real_readopted_child test around tmp path; why: this test groups the arrange, act, and assertions that protect the native router binds and routes a real readopted child outcome. +def test_native_router_binds_and_routes_a_real_readopted_child(tmp_path): + # What: enter the socket.socket managed context before reservation bind; why: test_native_router_binds_and_routes_a_real_readopted_child releases this resource or lock after reservation bind on both success and failure paths. + with socket.socket() as reservation: + # What: arrange the exact reservation bind fixture fragment; why: the native router binds and routes a real readopted child scenario feeds this byte-preserved fragment through reservation.bind(("127.0.0.1", 0)) before asserting its protocol or parser result. + reservation.bind(("127.0.0.1", 0)) + # What: act by calling reservation.getsockname and capture port; why: the native router binds and routes a real readopted child test asserts the response, state, or failure produced by this call. + port = reservation.getsockname()[1] + # What: act by calling subprocess.Popen and capture proc; why: the native router binds and routes a real readopted child test asserts the response, state, or failure produced by this call. + proc = subprocess.Popen( + # What: act by calling str with port; why: the native router binds and routes a real readopted child scenario observes the str return value during start new session stdin subprocess devnull stdout subprocess devnull. + [sys.executable, "-u", "-c", SERVER, "good", str(port), "no"], + # What: arrange start new session to subprocess.Popen; why: the native router binds and routes a real readopted child scenario binds this true value to subprocess.Popen's start new session input. + start_new_session=True, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + # What: arrange stderr to subprocess.Popen; why: the native router binds and routes a real readopted child scenario binds this devnull and subprocess value to subprocess.Popen's stderr input. + stderr=subprocess.DEVNULL, + # What: arrange the subprocess.Popen call with start new session and stdin and stdout and stderr; why: test_native_router_binds_and_routes_a_real_readopted_child groups the supplied clauses as one subprocess.Popen call before its value is consumed. + ) + # What: act by calling ServeStateStore and capture store; why: the native router binds and routes a real readopted child test asserts the response, state, or failure produced by this call. + store = ServeStateStore(str(tmp_path / "serve.json")) + # What: arrange the exact store save serve state model good port port fixture fragment; why: the native router binds and routes a real readopted child scenario feeds this byte-preserved fragment through store.save(ServeState(model="good", port=port, pid=proc.pid, args=["--ad before asserting its protocol or parser. + store.save(ServeState(model="good", port=port, pid=proc.pid, args=["--adopted"])) + # What: act by calling ServeProbe and capture probe; why: the native router binds and routes a real readopted child test asserts the response, state, or failure produced by this call. + probe = ServeProbe() + # What: act by calling time.monotonic and capture deadline; why: the native router binds and routes a real readopted child test asserts the response, state, or failure produced by this call. + deadline = time.monotonic() + 5 + # What: act across deadline and monotonic and time to perform oserror and sleep and json get and port and time; why: the native router binds and routes a real readopted child scenario repeats the body only while or for the loop header admits an iteration. + while time.monotonic() < deadline: + # What: establish the handler boundary for the protected operation; why: test_native_router_binds_and_routes_a_real_readopted_child routes failures to oserror while preserving cleanup and success flow. + try: + # What: act on json get and port before the computed value; why: the native router binds and routes a real readopted child scenario admits the computed value only for this predicate and excludes the opposite state. + if json_get(port, "/health")["status"] == "ok": + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the native router binds and routes a real readopted child scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: handle oserror by time sleep 0 05; why: test_native_router_binds_and_routes_a_real_readopted_child converts that failure into this concrete recovery, response, or cleanup behavior. + except OSError: + # What: act by calling time.sleep with 0 05; why: the native router binds and routes a real readopted child scenario observes the time.sleep return value during else. + time.sleep(0.05) + # What: select the remaining branch that performs raise assertion error test child did not; why: test_native_router_binds_and_routes_a_real_readopted_child covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: raise AssertionError for the caller; why: test_native_router_binds_and_routes_a_real_readopted_child stops this rejected path before it can mutate state, dispatch work, or report success. + raise AssertionError("test child did not become ready") + + # What: act by calling AdoptedChild and capture adopted; why: the native router binds and routes a real readopted child test asserts the response, state, or failure produced by this call. + adopted = AdoptedChild( + # What: arrange alive check to running; why: the native router binds and routes a real readopted child scenario binds this running and pid and proc value to running's alive check input. + proc.pid, port, None, None, alive_check=lambda: running(proc.pid), + # What: arrange sleep to AdoptedChild; why: the native router binds and routes a real readopted child scenario binds this sleep and time value to AdoptedChild's sleep input. + sleep=time.sleep, poll_interval=0.05, + # What: arrange the AdoptedChild call with alive check and sleep and poll interval; why: test_native_router_binds_and_routes_a_real_readopted_child groups the supplied clauses as one AdoptedChild call before its value is consumed. + ) + # What: act by calling ServeManager and capture manager; why: the native router binds and routes a real readopted child test asserts the response, state, or failure produced by this call. + manager = ServeManager( + # What: act by calling LogRing with the declared inputs; why: the native router binds and routes a real readopted child scenario observes the LogRing return value during spawn fn lambda args value for value. + LogRing(), store, + # What: arrange the args input for test_native_router_binds_and_routes_a_real_readopted_child; why: test_native_router_binds_and_routes_a_real_readopted_child consumes args during signature binding, so callers must bind it with the other signature inputs. + spawn_fn=lambda *args: (_ for _ in ()).throw(AssertionError("unexpected second spawn")), + # What: arrange the state input for test_native_router_binds_and_routes_a_real_readopted_child; why: test_native_router_binds_and_routes_a_real_readopted_child consumes state during signature binding, so callers must bind it with the other signature inputs. + adopt_fn=lambda state: adopted, + # What: arrange apply oom to ServeManager; why: the native router binds and routes a real readopted child scenario binds this false value to ServeManager's apply oom input. + apply_oom=False, grace_s=0.2, reap_wait_s=3, + # What: arrange the p input for test_native_router_binds_and_routes_a_real_readopted_child; why: test_native_router_binds_and_routes_a_real_readopted_child consumes p during signature binding, so callers must bind it with the other signature inputs. + read_stats=lambda p: json_get(p, "/v1/stats"), + # What: arrange the ServeManager call with spawn fn and adopt fn and apply oom and grace s and reap wait s; why: test_native_router_binds_and_routes_a_real_readopted_child groups the supplied clauses as one ServeManager call before its value is consumed. + ) + # What: act by calling ModelCatalog and capture catalog; why: the native router binds and routes a real readopted child test asserts the response, state, or failure produced by this call. + catalog = ModelCatalog({ + # What: arrange the good field as model profile and port and good and good and adopted; why: test_native_router_binds_and_routes_a_real_readopted_child carries good through catalog into router routing coordinator manager catalog probe. + "good": ModelProfile("good", "good", ("--adopted",), port=port), + # What: arrange the ModelCatalog call with model profile; why: test_native_router_binds_and_routes_a_real_readopted_child groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + # What: establish the handler boundary for the protected operation; why: test_native_router_binds_and_routes_a_real_readopted_child routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: assert that manager readopt is true; why: this assertion protects the native router binds and routes a real readopted child regression after the test's arranged inputs and exercised call. + assert manager.readopt() is True + # What: act by calling RoutingCoordinator and capture router; why: the native router binds and routes a real readopted child test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog, probe) + # What: assert that router status active profile equals good; why: this assertion protects the native router binds and routes a real readopted child regression after the test's arranged inputs and exercised call. + assert router.status()["activeProfile"] == "good" + # What: assert that router status active identity matches engine is true; why: this assertion protects the native router binds and routes a real readopted child regression after the test's arranged inputs and exercised call. + assert router.status()["activeIdentityMatchesEngine"] is True + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_native_router_binds_and_routes_a_real_readopted_child releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(2) as proxy: + # What: act by calling build_app and capture app; why: the native router binds and routes a real readopted child test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_native_router_binds_and_routes_a_real_readopted_child; why: test_native_router_binds_and_routes_a_real_readopted_child consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=probe, footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the native router binds and routes a real readopted child scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_native_router_binds_and_routes_a_real_readopted_child groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture response; why: the native router binds and routes a real readopted child test asserts the response, state, or failure produced by this call. + response = TestClient(app).post( + # What: arrange the model field as good; why: test_native_router_binds_and_routes_a_real_readopted_child sends this field through response so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "good", "stream": True}, + # What: arrange the operation.post call with json; why: test_native_router_binds_and_routes_a_real_readopted_child groups the supplied clauses as one operation.post call before its value is consumed. + ) + # What: assert that response status code equals 200; why: this assertion protects the native router binds and routes a real readopted child regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + # What: assert that response content endswith b data done n n; why: this assertion protects the native router binds and routes a real readopted child regression after the test's arranged inputs and exercised call. + assert response.content.endswith(b"data: [DONE]\n\n") + # What: assert that manager status pid equals proc pid and manager status adopted is true; why: this assertion protects the native router binds and routes a real readopted child regression after the test's arranged inputs and exercised call. + assert manager.status()["pid"] == proc.pid and manager.status()["adopted"] is True + # What: assert that router status activations equals 0; why: this assertion protects the native router binds and routes a real readopted child regression after the test's arranged inputs and exercised call. + assert router.status()["activations"] == 0 + # What: act by calling manager.stop with the declared inputs; why: the native router binds and routes a real readopted child scenario observes the manager.stop return value during proc wait timeout. + manager.stop() + # What: arrange timeout to proc.wait; why: the native router binds and routes a real readopted child scenario binds this 3 value to proc.wait's timeout input. + proc.wait(timeout=3) + # What: assert that store load is group delimiter; why: this assertion protects the native router binds and routes a real readopted child regression after the test's arranged inputs and exercised call. + assert store.load() is None + # What: run if proc poll is on every exit path; why: test_native_router_binds_and_routes_a_real_readopted_child performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: act on poll and proc before process lookup error and killpg and pid and os and proc; why: the native router binds and routes a real readopted child scenario admits process lookup error and killpg and pid and os and proc only for this predicate and excludes the opposite state. + if proc.poll() is None: + # What: establish the handler boundary for the protected operation; why: test_native_router_binds_and_routes_a_real_readopted_child routes failures to process lookup error while preserving cleanup and success flow. + try: + # What: act by calling os.killpg with pid and proc and 9; why: the native router binds and routes a real readopted child scenario observes the os.killpg return value during except process lookup error. + os.killpg(proc.pid, 9) + # What: handle process lookup error by pass; why: test_native_router_binds_and_routes_a_real_readopted_child converts that failure into this concrete recovery, response, or cleanup behavior. + except ProcessLookupError: + # What: ignore the anticipated exception handled by this branch; why: test_native_router_binds_and_routes_a_real_readopted_child continues its retry or cleanup path instead of re-raising that transient failure. + pass + # What: arrange timeout to proc.wait; why: the native router binds and routes a real readopted child scenario binds this 3 value to proc.wait's timeout input. + proc.wait(timeout=3) + + +# What: define the test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation test around tmp path; why: this test groups the arrange, act, and assertions that protect the native router uses fresh dynamic ports for real child reactivation outcome. +def test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation(tmp_path): + # What: define the free_port test helper around captured fixture state; why: the native router uses fresh dynamic ports for real child reactivation scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def free_port(): + # What: enter the socket.socket managed context before reservation bind; why: free_port releases this resource or lock after reservation bind on both success and failure paths. + with socket.socket() as reservation: + # What: arrange the exact reservation bind fixture fragment; why: the native router uses fresh dynamic ports for real child reactivation scenario feeds this byte-preserved fragment through reservation.bind(("127.0.0.1", 0)) before asserting its protocol or parser result. + reservation.bind(("127.0.0.1", 0)) + # What: return getsockname and reservation and 1 from the free_port test helper; why: the native router uses fresh dynamic ports for real child reactivation scenario uses this helper result in its subsequent act or assertion. + return reservation.getsockname()[1] + + # What: act by calling free_port and capture first port and second port; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + first_port, second_port = free_port(), free_port() + # What: assert that first port differs from second port; why: this assertion protects the native router uses fresh dynamic ports for real child reactivation regression after the test's arranged inputs and exercised call. + assert first_port != second_port + # What: arrange children as the fixture input; why: the native router uses fresh dynamic ports for real child reactivation test consumes this named precondition before exercising the behavior. + children = [] + + # What: define the spawn test helper around model and actual port and args; why: the native router uses fresh dynamic ports for real child reactivation scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def spawn(model, actual_port, args): + # What: act by calling subprocess.Popen and capture proc; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + proc = subprocess.Popen( + # What: act by calling str with actual port; why: the native router uses fresh dynamic ports for real child reactivation scenario observes the str return value during start new session stdin subprocess devnull stdout subprocess devnull. + [sys.executable, "-u", "-c", SERVER, model, str(actual_port), "no"], + # What: arrange start new session to subprocess.Popen; why: the native router uses fresh dynamic ports for real child reactivation scenario binds this true value to subprocess.Popen's start new session input. + start_new_session=True, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + # What: arrange stderr to subprocess.Popen; why: the native router uses fresh dynamic ports for real child reactivation scenario binds this devnull and subprocess value to subprocess.Popen's stderr input. + stderr=subprocess.DEVNULL, + # What: arrange the grouped source fragment for the scenario; why: test real process recovery test native router uses fresh dynamic ports for real child reactivation requires this concrete input or helper state before exercising the behavior under test. + ) + # What: act by calling PopenChild and capture child; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + child = PopenChild(proc, None) + # What: act by calling children.append with child; why: the native router uses fresh dynamic ports for real child reactivation scenario observes the children.append return value during return child. + children.append(child) + # What: return child from the spawn test helper; why: the native router uses fresh dynamic ports for real child reactivation scenario uses this helper result in its subsequent act or assertion. + return child + + # What: act by calling ServeStateStore and capture store; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + store = ServeStateStore(str(tmp_path / "serve.json")) + # What: act by calling ServeManager and capture manager; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + manager = ServeManager(LogRing(), store, spawn_fn=spawn, apply_oom=False, + # What: arrange grace s to ServeManager; why: the native router uses fresh dynamic ports for real child reactivation scenario binds this 0 2 value to ServeManager's grace s input. + grace_s=0.2, reap_wait_s=3, + # What: arrange the p input for test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation; why: test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation consumes p during signature binding, so callers must bind it with the other signature inputs. + read_stats=lambda p: json_get(p, "/v1/stats")) + # What: act by calling ServeProbe and capture probe; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + probe = ServeProbe() + # What: act by calling ModelCatalog and capture catalog; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + catalog = ModelCatalog({"good": ModelProfile("good", "good", (), port=0)}) + # What: act by calling iter and capture ports; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + ports = iter((first_port, second_port)) + # What: act by calling RoutingCoordinator and capture router; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog, probe, port_allocator=lambda: next(ports)) + # What: establish the handler boundary for the protected operation; why: test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(2) as proxy: + # What: act by calling build_app and capture app; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation; why: test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=probe, footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the native router uses fresh dynamic ports for real child reactivation scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling client.post and capture first; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + first = client.post("/v1/chat/completions", json={"model": "good", "stream": True}) + # What: assert that first status code equals 200; why: this assertion protects the native router uses fresh dynamic ports for real child reactivation regression after the test's arranged inputs and exercised call. + assert first.status_code == 200 + # What: assert that manager status port equals first port; why: this assertion protects the native router uses fresh dynamic ports for real child reactivation regression after the test's arranged inputs and exercised call. + assert manager.status()["port"] == first_port + # What: assert that router evict idle good is true; why: this assertion protects the native router uses fresh dynamic ports for real child reactivation regression after the test's arranged inputs and exercised call. + assert router.evict_idle("good") is True + # What: act by calling client.post and capture second; why: the native router uses fresh dynamic ports for real child reactivation test asserts the response, state, or failure produced by this call. + second = client.post("/v1/chat/completions", json={"model": "good", "stream": True}) + # What: assert that second status code equals 200; why: this assertion protects the native router uses fresh dynamic ports for real child reactivation regression after the test's arranged inputs and exercised call. + assert second.status_code == 200 + # What: assert that manager status port equals second port; why: this assertion protects the native router uses fresh dynamic ports for real child reactivation regression after the test's arranged inputs and exercised call. + assert manager.status()["port"] == second_port + # What: assert that router status activations equals 2; why: this assertion protects the native router uses fresh dynamic ports for real child reactivation regression after the test's arranged inputs and exercised call. + assert router.status()["activations"] == 2 + # What: assert that len children equals 2 and children 0 reaped is set; why: this assertion protects the native router uses fresh dynamic ports for real child reactivation regression after the test's arranged inputs and exercised call. + assert len(children) == 2 and children[0].reaped.is_set() + # What: act by calling manager.stop with the declared inputs; why: the native router uses fresh dynamic ports for real child reactivation scenario observes the manager.stop return value during assert children reaped is set and store load is. + manager.stop() + # What: assert that children 1 reaped is set and store load is; why: this assertion protects the native router uses fresh dynamic ports for real child reactivation regression after the test's arranged inputs and exercised call. + assert children[1].reaped.is_set() and store.load() is None + # What: run for child in children on every exit path; why: test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: act across children to perform process lookup error and killpg and pid and os and child; why: the native router uses fresh dynamic ports for real child reactivation scenario repeats the body only while or for the loop header admits an iteration. + for child in children: + # What: establish the handler boundary for the protected operation; why: test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation routes failures to process lookup error while preserving cleanup and success flow. + try: + # What: act by calling os.killpg with pid and child and 9; why: the native router uses fresh dynamic ports for real child reactivation scenario observes the os.killpg return value during except process lookup error. + os.killpg(child.pid, 9) + # What: handle process lookup error by pass; why: test_native_router_uses_fresh_dynamic_ports_for_real_child_reactivation converts that failure into this concrete recovery, response, or cleanup behavior. + except ProcessLookupError: + # What: ignore the anticipated exception handled by this branch; why: spawn continues its retry or cleanup path instead of re-raising that transient failure. + pass + # What: act on poll and proc and child before wait and proc and child; why: the native router uses fresh dynamic ports for real child reactivation scenario admits wait and proc and child only for this predicate and excludes the opposite state. + if child.proc.poll() is None: + # What: arrange timeout to child.proc.wait; why: the native router uses fresh dynamic ports for real child reactivation scenario binds this 3 value to child.proc.wait's timeout input. + child.proc.wait(timeout=3) diff --git a/tests/daemon/test_router.py b/tests/daemon/test_router.py new file mode 100644 index 0000000000..2fccfc39a6 --- /dev/null +++ b/tests/daemon/test_router.py @@ -0,0 +1,7215 @@ +# What: enable postponed evaluation of annotations; why: type hints in test_router can reference runtime types without eager imports or forward-reference failures. +from __future__ import annotations + +# What: import asyncio for test cancelled queued http request cannot trigger a later swap using asyncio; why: test_cancelled_queued_http_request_cannot_trigger_a_later_swap uses asyncio run, making that imported dependency available to its named operation. +import asyncio + +# What: import base64 for test activity and opt in capture apis are authenticated redacted and durable using base64; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable uses base64 b64decode, making that imported dependency available to its named operation. +import base64 + +# What: import json for test request filter applies nested drop global and requested id fields in order using json; why: test_request_filter_applies_nested_drop_global_and_requested_id_fields_in_order uses json loads, making that imported dependency available to its named operation. +import json + +# What: import socket for adjacent dynamic-port allocation verification; why: the regression binds both FreeToken listeners using production IPv4 stream semantics. +import socket + +# What: import threading for test switch waits until an active lease finishes using threading; why: test_switch_waits_until_an_active_lease_finishes uses threading event, making that imported dependency available to its named operation. +import threading + +# What: import time for test router reload cannot race atomic profile lookup and dynamic port binding using time; why: test_router_reload_cannot_race_atomic_profile_lookup_and_dynamic_port_binding uses time sleep, making that imported dependency available to its named operation. +import time + +# What: import thread pool executor for test cancelled queued http request cannot trigger a later swap using concurrent and futures and thread pool executor; why: test_cancelled_queued_http_request_cannot_trigger_a_later_swap uses thread pool executor, making that imported dependency available to its named operation. +from concurrent.futures import ThreadPoolExecutor + +# What: arrange from http server import BaseHTTPRequestHandler ThreadingHTTPServer for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +# What: import bytes io for test explicit cancel while upstream connects closes result and releases lease using io and bytes io; why: test_explicit_cancel_while_upstream_connects_closes_result_and_releases_lease uses bytes io, making that imported dependency available to its named operation. +from io import BytesIO + +# What: import httpx for scenario using httpx; why: scenario uses httpx asgitransport, making that imported dependency available to its named operation. +import httpx + +# What: import pytest for module initialization using pytest; why: module initialization uses pytest mark parametrize, making that imported dependency available to its named operation. +import pytest + +# What: import test client for test http concurrency rejection returns retry after and releases request id using fastapi and testclient and test client; why: test_http_concurrency_rejection_returns_retry_after_and_releases_request_id uses test client, making that imported dependency available to its named operation. +from fastapi.testclient import TestClient + +# What: import build app for test cancelled queued http request cannot trigger a later swap using freetoken and daemon and app and build app; why: test_cancelled_queued_http_request_cannot_trigger_a_later_swap uses build app, making that imported dependency available to its named operation. +from freetoken.daemon.app import build_app + +# What: arrange from freetoken daemon catalog import for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.catalog import ( + # What: arrange CatalogError for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + CatalogError, + # What: arrange ModelCapabilities for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + ModelCapabilities, + # What: arrange ModelCatalog for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + ModelCatalog, + # What: arrange ModelProfile for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + ModelProfile, + # What: arrange ModelSelector for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + ModelSelector, + # What: arrange RequestField for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + RequestField, + # What: arrange RouterSettings for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + RouterSettings, + # What: arrange RoutingGroup for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + RoutingGroup, + # What: arrange RoutingProfile for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + RoutingProfile, + # What: arrange the enclosing predicate with from freetoken daemon catalog import catalog error model capabilities model catalog model profile model selector request; why: test_router groups the supplied clauses as one test_router expression before its value is consumed. +) + +# What: arrange from freetoken daemon inference proxy import for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.inference_proxy import ( + # What: arrange UpstreamResponse for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + UpstreamResponse, + # What: arrange filter request body for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + filter_request_body, + # What: arrange forward headers for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + forward_headers, + # What: arrange open upstream for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + open_upstream, + # What: arrange response headers for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. + response_headers, + # What: arrange the enclosing predicate with from freetoken daemon inference proxy import upstream response filter request body forward headers open upstream respons; why: test_router groups the supplied clauses as one test_router expression before its value is consumed. +) + +# What: arrange from freetoken daemon logring import LogRing for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.logring import LogRing + +# What: import serve probe for test custom readiness path accepts real http success without json using freetoken and daemon and proxy and serve probe; why: test_custom_readiness_path_accepts_real_http_success_without_json uses serve probe, making that imported dependency available to its named operation. +from freetoken.daemon.proxy import ServeProbe + +# What: import wait for ready for test custom readiness path accepts real http success without json using freetoken and daemon and readiness and wait for ready; why: test_custom_readiness_path_accepts_real_http_success_without_json uses wait for ready, making that imported dependency available to its named operation. +from freetoken.daemon.readiness import wait_for_ready + +# What: arrange from freetoken daemon router import RoutingCoordinator RoutingError for the scenario; why: test router requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.router import ( + RoutingCoordinator, + RoutingError, + allocate_loopback_port, +) + + +# What: define Manager as the owner of __init__ and status and serve_args and start and switch_for_readiness; why: daemon callers use this class boundary so those methods share one manager state invariant. +class Manager: + # What: define the __init__ test helper around captured fixture state; why: the init scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def __init__(self): + # What: arrange model as the fixture input; why: the router test consumes this named precondition before exercising the behavior. + self.model = None + # What: arrange port as the fixture input; why: the router test consumes this named precondition before exercising the behavior. + self.port = None + # What: arrange args as the fixture input; why: the router test consumes this named precondition before exercising the behavior. + self.args = [] + # What: arrange pid as 100; why: the router test consumes this named precondition before exercising the behavior. + self.pid = 100 + # What: arrange calls as the fixture input; why: the router test consumes this named precondition before exercising the behavior. + self.calls = [] + + # What: define the status test helper around captured fixture state; why: the status scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def status(self): + # What: arrange the running field as model; why: Manager.status carries running into return {"running": self.model is not None, "model": self.model, "port":. + return {"running": self.model is not None, "model": self.model, "port": self.port, "pid": self.pid} + + # What: define the serve_args test helper around captured fixture state; why: the serve args scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def serve_args(self): + # What: return list and args from the serve_args test helper; why: the serve args scenario uses this helper result in its subsequent act or assertion. + return list(self.args) + + # What: define the start test helper around model and port and args; why: the start scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def start(self, model, port, args): + # What: arrange the exact self calls append start model fixture fragment; why: the start scenario feeds this byte-preserved fragment through self.calls.append(("start", model)) before asserting its protocol or parser result. + self.calls.append(("start", model)) + # What: act by calling list and capture model and port and args; why: the router test asserts the response, state, or failure produced by this call. + self.model, self.port, self.args = model, port, list(args) + # What: arrange pid from 1; why: the start scenario uses pid during return pid self pid before checking the protected result. + self.pid += 1 + # What: arrange the pid field as pid; why: Manager.start carries pid into return {"pid": self.pid}. + return {"pid": self.pid} + + # What: define the switch_for_readiness test helper around model and port and args; why: the switch for readiness scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def switch_for_readiness(self, model, port, args): + # What: arrange the exact self calls append switch model fixture fragment; why: the switch for readiness scenario feeds this byte-preserved fragment through self.calls.append(("switch", model)) before asserting its protocol or parser result. + self.calls.append(("switch", model)) + # What: act by calling list and capture previous; why: the router test asserts the response, state, or failure produced by this call. + previous = self.model, self.port, list(self.args) + # What: act by calling list and capture model and port and args; why: the router test asserts the response, state, or failure produced by this call. + self.model, self.port, self.args = model, port, list(args) + # What: arrange pid from 1; why: the switch for readiness scenario uses pid during return pid self pid previous before checking the protected result. + self.pid += 1 + # What: arrange the pid field as pid; why: Manager.switch_for_readiness carries pid into return {"pid": self.pid}, previous. + return {"pid": self.pid}, previous + + # What: define the recover_switch test helper around ticket; why: the recover switch scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def recover_switch(self, ticket): + # What: arrange model and port and args as ticket; why: the router test consumes this named precondition before exercising the behavior. + self.model, self.port, self.args = ticket + # What: arrange pid from 1; why: the recover switch scenario uses pid during return launched pid self pid before checking the protected result. + self.pid += 1 + # What: arrange the launched field as true; why: Manager.recover_switch carries launched into return {"launched": True, "pid": self.pid}. + return {"launched": True, "pid": self.pid} + + # What: define the stop test helper around timeout; why: the stop scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def stop(self, timeout): + # What: arrange the exact self calls append stop timeout fixture fragment; why: the stop scenario feeds this byte-preserved fragment through self.calls.append(("stop", timeout)) before asserting its protocol or parser result. + self.calls.append(("stop", timeout)) + # What: arrange model as the fixture input; why: the router test consumes this named precondition before exercising the behavior. + self.model = None + # What: arrange the stopped field as true; why: Manager.stop carries stopped into return {"stopped": True}. + return {"stopped": True} + + +# What: define the catalog test helper around captured fixture state; why: the catalog scenario calls this helper to produce or observe the exact behavior checked by its assertions. +def catalog(): + # What: return model catalog and model profile and low and high and low from the catalog test helper; why: the catalog scenario uses this helper result in its subsequent act or assertion. + return ModelCatalog({ + # What: arrange the low field as model profile and low and low and gguf and 0; why: catalog carries low into "low": ModelProfile("low", "low.gguf", (), priority=0). + "low": ModelProfile("low", "low.gguf", (), priority=0), + # What: arrange the high field as model profile and high and high and gguf and 10; why: catalog carries high into "high": ModelProfile("high", "high.gguf", (), priority=10). + "high": ModelProfile("high", "high.gguf", (), priority=10), + # What: arrange the ModelCatalog call with model profile; why: catalog groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + + +# What: define the ready test helper around manager and probe and pid and port and timeout s; why: the ready scenario calls this helper to produce or observe the exact behavior checked by its assertions. +def ready(manager, probe, *, pid, port, timeout_s): + # What: return HTTP 200 when accepting and 503 otherwise; why: supervisors use this status and ready boolean to decide whether the daemon control plane may receive traffic. + return {"ready": True, "health": {"status": "ok"}} + + +# What: define the test_routes_to_ready_engine_then_shares_its_lease test around local fixtures; why: this test groups the arrange, act, and assertions that protect the routes to ready engine then shares its lease outcome. +def test_routes_to_ready_engine_then_shares_its_lease(): + # What: act by calling Manager and capture manager; why: the routes to ready engine then shares its lease test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the routes to ready engine then shares its lease test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: act by calling router.acquire and capture first; why: the routes to ready engine then shares its lease test asserts the response, state, or failure produced by this call. + first = router.acquire("low") + # What: act by calling router.acquire and capture second; why: the routes to ready engine then shares its lease test asserts the response, state, or failure produced by this call. + second = router.acquire("low") + # What: assert that manager calls equals start low gguf; why: this assertion protects the routes to ready engine then shares its lease regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf")] + # What: assert that router status active requests equals 2; why: this assertion protects the routes to ready engine then shares its lease regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 2 + # What: act by calling second.release with the declared inputs; why: the routes to ready engine then shares its lease scenario observes the second.release return value during first release. + second.release() + # What: act by calling first.release with the declared inputs; why: the routes to ready engine then shares its lease scenario observes the first.release return value during assert router status active requests. + first.release() + # What: assert that router status active requests equals 0; why: this assertion protects the routes to ready engine then shares its lease regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + + +# What: define the test_unknown_model_is_a_stable_404_router_error test around local fixtures; why: this test groups the arrange, act, and assertions that protect the unknown model is a stable 404 router error outcome. +def test_unknown_model_is_a_stable_404_router_error(): + # What: assert the pytest.raises failure context; why: the unknown model is a stable 404 router error scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RoutingError, match="unknown model") as exc: + # What: arrange the exact routing coordinator manager catalog object ready fn ready fixture fragment; why: the unknown model is a stable 404 router error scenario feeds this byte-preserved fragment through RoutingCoordinator(Manager(), catalog(), object(), ready_fn=ready).acqui before asserting its protocol or. + RoutingCoordinator(Manager(), catalog(), object(), ready_fn=ready).acquire("missing") + # What: assert that exc value code equals unknown model; why: this assertion protects the unknown model is a stable 404 router error regression after the test's arranged inputs and exercised call. + assert exc.value.code == "unknown_model" + # What: assert that exc value status code equals 404; why: this assertion protects the unknown model is a stable 404 router error regression after the test's arranged inputs and exercised call. + assert exc.value.status_code == 404 + + +# What: define the test_dynamic_profile_port_is_stable_while_resident_and_fresh_after_a_swap test around local fixtures; why: this test groups the arrange, act, and assertions that protect the dynamic profile port is stable while resident and fresh after a swap outcome. +def test_dynamic_profile_port_is_stable_while_resident_and_fresh_after_a_swap(): + # What: act by calling Manager and capture manager; why: the dynamic profile port is stable while resident and fresh after a swap test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the dynamic profile port is stable while resident and fresh after a swap test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({ + # What: arrange the dynamic field as model profile and dynamic and dynamic and gguf and 0; why: test_dynamic_profile_port_is_stable_while_resident_and_fresh_after_a_swap carries dynamic through catalog doc into manager catalog doc object ready fn ready port allocator lambda. + "dynamic": ModelProfile("dynamic", "dynamic.gguf", (), port=0), + # What: arrange the other field as model profile and other and other and gguf and 19555; why: test_dynamic_profile_port_is_stable_while_resident_and_fresh_after_a_swap carries other through catalog doc into manager catalog doc object ready fn ready port allocator lambda. + "other": ModelProfile("other", "other.gguf", (), port=19555), + # What: arrange the ModelCatalog call with model profile; why: test_dynamic_profile_port_is_stable_while_resident_and_fresh_after_a_swap groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + # What: act by calling iter and capture allocated; why: the dynamic profile port is stable while resident and fresh after a swap test asserts the response, state, or failure produced by this call. + allocated = iter([20101, 20102]) + # What: act by calling RoutingCoordinator and capture router; why: the dynamic profile port is stable while resident and fresh after a swap test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator( + # What: arrange ready fn to object; why: the dynamic profile port is stable while resident and fresh after a swap scenario binds this ready value to object's ready fn input. + manager, catalog_doc, object(), ready_fn=ready, port_allocator=lambda: next(allocated), + # What: arrange the RoutingCoordinator call with ready fn and port allocator; why: test_dynamic_profile_port_is_stable_while_resident_and_fresh_after_a_swap groups the supplied clauses as one RoutingCoordinator call before its value is consumed. + ) + + # What: act by calling router.acquire and capture first; why: the dynamic profile port is stable while resident and fresh after a swap test asserts the response, state, or failure produced by this call. + first = router.acquire("dynamic") + # What: act by calling first.release with the declared inputs; why: the dynamic profile port is stable while resident and fresh after a swap scenario observes the first.release return value during warm router acquire dynamic. + first.release() + # What: act by calling router.acquire and capture warm; why: the dynamic profile port is stable while resident and fresh after a swap test asserts the response, state, or failure produced by this call. + warm = router.acquire("dynamic") + # What: assert that first port warm port equals 20101 20101; why: this assertion protects the dynamic profile port is stable while resident and fresh after a swap regression after the test's arranged inputs and exercised call. + assert (first.port, warm.port) == (20101, 20101) + # What: act by calling warm.release with the declared inputs; why: the dynamic profile port is stable while resident and fresh after a swap scenario observes the warm.release return value during router acquire other release. + warm.release() + # What: arrange the exact router acquire other release fixture fragment; why: the dynamic profile port is stable while resident and fresh after a swap scenario feeds this byte-preserved fragment through router.acquire("other").release() before asserting its protocol or parser result. + router.acquire("other").release() + # What: act by calling router.acquire and capture cold again; why: the dynamic profile port is stable while resident and fresh after a swap test asserts the response, state, or failure produced by this call. + cold_again = router.acquire("dynamic") + # What: assert that cold again port equals 20102; why: this assertion protects the dynamic profile port is stable while resident and fresh after a swap regression after the test's arranged inputs and exercised call. + assert cold_again.port == 20102 + # What: act by calling cold_again.release with the declared inputs; why: the dynamic profile port is stable while resident and fresh after a swap scenario observes the cold_again.release return value during assert manager calls. + cold_again.release() + # What: assert the expected manager calls == outcome; why: test router test dynamic profile port is stable while resident and fresh after a swap protects its regression by requiring this observable result after the exercised behavior. + assert manager.calls == [ + # What: arrange start dynamic gguf for the scenario; why: test router test dynamic profile port is stable while resident and fresh after a swap requires this concrete input or helper state before exercising the behavior under test. + ("start", "dynamic.gguf"), + # What: arrange switch other gguf for the scenario; why: test router test dynamic profile port is stable while resident and fresh after a swap requires this concrete input or helper state before exercising the behavior under test. + ("switch", "other.gguf"), + # What: arrange switch dynamic gguf for the scenario; why: test router test dynamic profile port is stable while resident and fresh after a swap requires this concrete input or helper state before exercising the behavior under test. + ("switch", "dynamic.gguf"), + # What: arrange the grouped source fragment for the scenario; why: test router test dynamic profile port is stable while resident and fresh after a swap requires this concrete input or helper state before exercising the behavior under test. + ] + + +# What: parameterize test_router_binds_unambiguous_exact_manager_re_adoption with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test router binds unambiguous exact manager re adoption. +@pytest.mark.parametrize("configured_port", [1919, 0, None]) +# What: define the test_router_binds_unambiguous_exact_manager_re_adoption test around configured port; why: this test groups the arrange, act, and assertions that protect the router binds unambiguous exact manager re adoption outcome. +def test_router_binds_unambiguous_exact_manager_re_adoption(configured_port): + # What: act by calling Manager and capture manager; why: the router binds unambiguous exact manager re adoption test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: arrange model as adopted and gguf; why: the router binds unambiguous exact manager re adoption test consumes this named precondition before exercising the behavior. + manager.model = "adopted.gguf" + # What: arrange port as 1919; why: the router binds unambiguous exact manager re adoption test consumes this named precondition before exercising the behavior. + manager.port = 1919 + # What: arrange args as served model name and adopted; why: the router binds unambiguous exact manager re adoption test consumes this named precondition before exercising the behavior. + manager.args = ["--served-model-name", "adopted"] + # What: act by calling ModelCatalog and capture catalog doc; why: the router binds unambiguous exact manager re adoption test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({ + # What: arrange the adopted field as model profile and tuple and args and configured port; why: test_router_binds_unambiguous_exact_manager_re_adoption carries adopted through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "adopted": ModelProfile( + # What: arrange port to tuple; why: the router binds unambiguous exact manager re adoption scenario binds this configured port value to tuple's port input. + "adopted", "adopted.gguf", tuple(manager.args), port=configured_port + # What: arrange the ModelProfile call with port; why: test_router_binds_unambiguous_exact_manager_re_adoption groups the supplied clauses as one ModelProfile call before its value is consumed. + ), + # What: arrange the ModelCatalog call with model profile; why: test_router_binds_unambiguous_exact_manager_re_adoption groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + + # What: act by calling RoutingCoordinator and capture router; why: the router binds unambiguous exact manager re adoption test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: assert that router status active profile equals adopted; why: this assertion protects the router binds unambiguous exact manager re adoption regression after the test's arranged inputs and exercised call. + assert router.status()["activeProfile"] == "adopted" + # What: assert that router status active identity matches engine is true; why: this assertion protects the router binds unambiguous exact manager re adoption regression after the test's arranged inputs and exercised call. + assert router.status()["activeIdentityMatchesEngine"] is True + # What: act by calling router.acquire and capture lease; why: the router binds unambiguous exact manager re adoption test asserts the response, state, or failure produced by this call. + lease = router.acquire("adopted") + # What: act by calling lease.release with the declared inputs; why: the router binds unambiguous exact manager re adoption scenario observes the lease.release return value during assert manager calls. + lease.release() + # What: assert that manager calls equals group delimiter; why: this assertion protects the router binds unambiguous exact manager re adoption regression after the test's arranged inputs and exercised call. + assert manager.calls == [] + # What: assert that router status activations equals 0; why: this assertion protects the router binds unambiguous exact manager re adoption regression after the test's arranged inputs and exercised call. + assert router.status()["activations"] == 0 + + +# What: define the test_router_refuses_ambiguous_or_argument_mismatched_re_adoption test around local fixtures; why: this test groups the arrange, act, and assertions that protect the router refuses ambiguous or argument mismatched re adoption outcome. +def test_router_refuses_ambiguous_or_argument_mismatched_re_adoption(): + # What: act by calling Manager and capture manager; why: the router refuses ambiguous or argument mismatched re adoption test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: arrange model as shared and gguf; why: the router refuses ambiguous or argument mismatched re adoption test consumes this named precondition before exercising the behavior. + manager.model = "shared.gguf" + # What: arrange port as 1919; why: the router refuses ambiguous or argument mismatched re adoption test consumes this named precondition before exercising the behavior. + manager.port = 1919 + # What: arrange args as actual; why: the router refuses ambiguous or argument mismatched re adoption test consumes this named precondition before exercising the behavior. + manager.args = ["--actual"] + # What: act by calling ModelCatalog and capture ambiguous; why: the router refuses ambiguous or argument mismatched re adoption test asserts the response, state, or failure produced by this call. + ambiguous = ModelCatalog({ + # What: arrange the one field as model profile and tuple and args and manager and one; why: test_router_refuses_ambiguous_or_argument_mismatched_re_adoption carries one through ambiguous into assert routing coordinator manager ambiguous object ready fn ready. + "one": ModelProfile("one", "shared.gguf", tuple(manager.args), port=0), + # What: arrange the two field as model profile and tuple and args and manager and two; why: test_router_refuses_ambiguous_or_argument_mismatched_re_adoption carries two through ambiguous into assert routing coordinator manager ambiguous object ready fn ready. + "two": ModelProfile("two", "shared.gguf", tuple(manager.args), port=0), + # What: arrange the ModelCatalog call with model profile; why: test_router_refuses_ambiguous_or_argument_mismatched_re_adoption groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + # What: act by calling ModelCatalog and capture mismatched; why: the router refuses ambiguous or argument mismatched re adoption test asserts the response, state, or failure produced by this call. + mismatched = ModelCatalog({ + # What: arrange the one field as model profile and one and shared and gguf and different; why: test_router_refuses_ambiguous_or_argument_mismatched_re_adoption carries one through mismatched into assert routing coordinator manager mismatched object ready fn ready. + "one": ModelProfile("one", "shared.gguf", ("--different",), port=1919), + # What: arrange the ModelCatalog call with model profile; why: test_router_refuses_ambiguous_or_argument_mismatched_re_adoption groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + + # What: assert that routing coordinator manager ambiguous object ready fn ready is group delimiter; why: this assertion protects the router refuses ambiguous or argument mismatched re adoption regression after the test's arranged inputs and exercised call. + assert RoutingCoordinator(manager, ambiguous, object(), ready_fn=ready).status()["activeProfile"] is None + # What: assert that routing coordinator manager mismatched object ready fn ready is group delimiter; why: this assertion protects the router refuses ambiguous or argument mismatched re adoption regression after the test's arranged inputs and exercised call. + assert RoutingCoordinator(manager, mismatched, object(), ready_fn=ready).status()["activeProfile"] is None + + +# What: define the test_switch_waits_until_an_active_lease_finishes test around local fixtures; why: this test groups the arrange, act, and assertions that protect the switch waits until an active lease finishes outcome. +def test_switch_waits_until_an_active_lease_finishes(): + # What: act by calling Manager and capture manager; why: the switch waits until an active lease finishes test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the switch waits until an active lease finishes test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: act by calling router.acquire and capture lease; why: the switch waits until an active lease finishes test asserts the response, state, or failure produced by this call. + lease = router.acquire("low") + # What: act by calling threading.Event and capture entered; why: the switch waits until an active lease finishes test asserts the response, state, or failure produced by this call. + entered = threading.Event() + # What: act by calling threading.Event and capture released; why: the switch waits until an active lease finishes test asserts the response, state, or failure produced by this call. + released = threading.Event() + # What: arrange result as the fixture input; why: the switch waits until an active lease finishes test consumes this named precondition before exercising the behavior. + result = [] + + # What: define the acquire_high test helper around captured fixture state; why: the switch waits until an active lease finishes scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def acquire_high(): + # What: act by calling entered.set with the declared inputs; why: the switch waits until an active lease finishes scenario observes the entered.set return value during held router acquire high. + entered.set() + # What: act by calling router.acquire and capture held; why: the switch waits until an active lease finishes test asserts the response, state, or failure produced by this call. + held = router.acquire("high") + # What: act by calling result.append with held; why: the switch waits until an active lease finishes scenario observes the result.append return value during released set. + result.append(held) + # What: act by calling released.set with the declared inputs; why: the switch waits until an active lease finishes scenario observes the released.set return value during the enclosing return. + released.set() + + # What: act by calling threading.Thread and capture thread; why: the switch waits until an active lease finishes test asserts the response, state, or failure produced by this call. + thread = threading.Thread(target=acquire_high) + # What: act by calling thread.start with the declared inputs; why: the switch waits until an active lease finishes scenario observes the thread.start return value during assert entered wait. + thread.start() + # What: assert that entered wait 1; why: this assertion protects the switch waits until an active lease finishes regression after the test's arranged inputs and exercised call. + assert entered.wait(1) + # What: assert that released wait 0 05 is false; why: this assertion protects the switch waits until an active lease finishes regression after the test's arranged inputs and exercised call. + assert not released.wait(0.05) + # What: act by calling lease.release with the declared inputs; why: the switch waits until an active lease finishes scenario observes the lease.release return value during assert released wait. + lease.release() + # What: assert that released wait 1; why: this assertion protects the switch waits until an active lease finishes regression after the test's arranged inputs and exercised call. + assert released.wait(1) + # What: act by calling operation.release with the declared inputs; why: the switch waits until an active lease finishes scenario observes the operation.release return value during thread join. + result.pop().release() + # What: act by calling thread.join with 1; why: the switch waits until an active lease finishes scenario observes the thread.join return value during assert manager calls start low gguf switch high gguf. + thread.join(1) + # What: assert that manager calls equals start low gguf switch high gguf; why: this assertion protects the switch waits until an active lease finishes regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf"), ("switch", "high.gguf")] + + +# What: define the test_cancelled_queued_http_request_cannot_trigger_a_later_swap test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the cancelled queued http request cannot trigger a later swap outcome. +def test_cancelled_queued_http_request_cannot_trigger_a_later_swap(monkeypatch): + # What: act by calling Manager and capture manager; why: the cancelled queued http request cannot trigger a later swap test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the cancelled queued http request cannot trigger a later swap test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: act by calling router.acquire and capture active; why: the cancelled queued http request cannot trigger a later swap test asserts the response, state, or failure produced by this call. + active = router.acquire("low") + # What: arrange monkeypatch setattr for the scenario; why: test cancelled queued http request cannot trigger a later swap requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the cancelled queued http request cannot trigger a later swap scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the exact lambda kwargs pytest fail cancelled queued request fixture fragment; why: the cancelled queued http request cannot trigger a later swap scenario feeds this byte-preserved fragment through lambda **kwargs: pytest.fail("cancelled queued request reached upstream" before asserting its pro. + lambda **kwargs: pytest.fail("cancelled queued request reached upstream"), + # What: arrange the monkeypatch.setattr call with fail; why: test_cancelled_queued_http_request_cannot_trigger_a_later_swap groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + + # What: define the scenario test helper around app; why: the cancelled queued http request cannot trigger a later swap scenario calls this helper to produce or observe the exact behavior checked by its assertions. + async def scenario(app): + # What: act by calling httpx.ASGITransport and capture transport; why: the cancelled queued http request cannot trigger a later swap test asserts the response, state, or failure produced by this call. + transport = httpx.ASGITransport(app=app) + # What: arrange async with httpx AsyncClient transport transport base url http test as client for the scenario; why: test cancelled queued http request cannot trigger requires this concrete input or helper state before exercising the behavior under test. + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + # What: act by calling asyncio.create_task and capture request; why: the cancelled queued http request cannot trigger a later swap test asserts the response, state, or failure produced by this call. + request = asyncio.create_task(client.post( + # What: arrange the model field as high; why: scenario sends this field through request so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "high"}, + # What: arrange the x ft request id field as cancelled while queued; why: scenario carries x ft request id through request into request cancel. + headers={"X-FT-Request-ID": "cancelled-while-queued"}, + # What: arrange the asyncio.create_task call with post; why: scenario groups the supplied clauses as one asyncio.create_task call before its value is consumed. + )) + # What: act across range to perform status and router; why: the cancelled queued http request cannot trigger a later swap scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: arrange if router status queuedRequests == 1 for the scenario; why: test router test cancelled queued http request cannot trigger a later swap requires this concrete input or helper state before exercising the behavior under test. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the cancelled queued http request cannot trigger a later swap scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the cancelled queued http request cannot trigger a later swap scenario observes the asyncio.sleep return value during assert router status queued requests. + await asyncio.sleep(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the cancelled queued http request cannot trigger a later swap regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + # What: act by calling request.cancel with the declared inputs; why: the cancelled queued http request cannot trigger a later swap scenario observes the request.cancel return value during with pytest raises asyncio cancelled error. + request.cancel() + # What: assert the pytest.raises failure context; why: the cancelled queued http request cannot trigger a later swap scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(asyncio.CancelledError): + # What: arrange the await request portion of the enclosing predicate; why: this clause remains in the cancelled queued http request cannot trigger a later swap scenario\'s enclosing expression so its grouping and evaluation order stay intact. + await request + # What: act across range to perform status and router; why: the cancelled queued http request cannot trigger a later swap scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: arrange if router status queuedRequests == 0 for the scenario; why: test router test cancelled queued http request cannot trigger a later swap requires this concrete input or helper state before exercising the behavior under test. + if router.status()["queuedRequests"] == 0: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the cancelled queued http request cannot trigger a later swap scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the cancelled queued http request cannot trigger a later swap scenario observes the asyncio.sleep return value during assert router status queued requests. + await asyncio.sleep(0.01) + # What: assert that router status queued requests equals 0; why: this assertion protects the cancelled queued http request cannot trigger a later swap regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 0 + # What: assert that await client get router requests json data equals group delimiter; why: this assertion protects the cancelled queued http request cannot trigger a later swap regression after the test's arranged inputs and exercised call. + assert (await client.get("/router/requests")).json()["data"] == [] + # What: act by calling client.post and capture retry; why: the cancelled queued http request cannot trigger a later swap test asserts the response, state, or failure produced by this call. + retry = await client.post( + # What: arrange the model field as missing; why: scenario sends this field through retry so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "missing"}, + # What: arrange the x ft request id field as cancelled while queued; why: scenario carries x ft request id through retry into assert retry status code equals 404. + headers={"X-FT-Request-ID": "cancelled-while-queued"}, + # What: arrange the client.post call with json and headers; why: scenario groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: assert that retry status code equals 404; why: this assertion protects the cancelled queued http request cannot trigger a later swap regression after the test's arranged inputs and exercised call. + assert retry.status_code == 404 + # What: assert that retry json error type equals unknown model; why: this assertion protects the cancelled queued http request cannot trigger a later swap regression after the test's arranged inputs and exercised call. + assert retry.json()["error"]["type"] == "unknown_model" + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_cancelled_queued_http_request_cannot_trigger_a_later_swap releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(2) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the cancelled queued http request cannot trigger a later swap test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_cancelled_queued_http_request_cannot_trigger_a_later_swap; why: test_cancelled_queued_http_request_cannot_trigger_a_later_swap consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to catalog; why: the cancelled queued http request cannot trigger a later swap scenario binds this lifecycle value to catalog's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog(), router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_cancelled_queued_http_request_cannot_trigger_a_later_swap groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling asyncio.run with scenario and app; why: the cancelled queued http request cannot trigger a later swap scenario observes the asyncio.run return value during active release. + asyncio.run(scenario(app)) + + # What: act by calling active.release with the declared inputs; why: the cancelled queued http request cannot trigger a later swap scenario observes the active.release return value during assert manager calls start low gguf. + active.release() + # What: assert that manager calls equals start low gguf; why: this assertion protects the cancelled queued http request cannot trigger a later swap regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf")] + # What: assert that router status cancellations equals 1; why: this assertion protects the cancelled queued http request cannot trigger a later swap regression after the test's arranged inputs and exercised call. + assert router.status()["cancellations"] == 1 + # What: assert that router status active requests equals 0; why: this assertion protects the cancelled queued http request cannot trigger a later swap regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + + +# What: define the test_default_profile_concurrency_limit_is_shared_by_alternate_ids test around local fixtures; why: this test groups the arrange, act, and assertions that protect the default profile concurrency limit is shared by alternate ids outcome. +def test_default_profile_concurrency_limit_is_shared_by_alternate_ids(): + # What: act by calling Manager and capture manager; why: the default profile concurrency limit is shared by alternate ids test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelProfile and capture profile; why: the default profile concurrency limit is shared by alternate ids test asserts the response, state, or failure produced by this call. + profile = ModelProfile("low", "low.gguf", (), aliases=("alternate",)) + # What: act by calling RoutingCoordinator and capture router; why: the default profile concurrency limit is shared by alternate ids test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator( + # What: arrange the low field as profile; why: test_default_profile_concurrency_limit_is_shared_by_alternate_ids carries low through router into leases router acquire low for value in range. + manager, ModelCatalog({"low": profile}), object(), ready_fn=ready + # What: arrange the RoutingCoordinator call with ready fn; why: test_default_profile_concurrency_limit_is_shared_by_alternate_ids groups the supplied clauses as one RoutingCoordinator call before its value is consumed. + ) + # What: act by calling router.acquire and capture leases; why: the default profile concurrency limit is shared by alternate ids test asserts the response, state, or failure produced by this call. + leases = [router.acquire("low") for _ in range(10)] + + # What: arrange with pytest raises RoutingError match concurrency limit as exc for the scenario; why: test raises routing error match concurrency limit as exc requires this concrete input or helper state before exercising the behavior under test. + with pytest.raises(RoutingError, match="concurrency limit") as exc: + # What: arrange the exact router acquire alternate fixture fragment; why: the default profile concurrency limit is shared by alternate ids scenario feeds this byte-preserved fragment through router.acquire("alternate") before asserting its protocol or parser result. + router.acquire("alternate") + # What: assert that exc value status code equals 429; why: this assertion protects the default profile concurrency limit is shared by alternate ids regression after the test's arranged inputs and exercised call. + assert exc.value.status_code == 429 + # What: assert that exc value code equals concurrency limit; why: this assertion protects the default profile concurrency limit is shared by alternate ids regression after the test's arranged inputs and exercised call. + assert exc.value.code == "concurrency_limit" + # What: assert that router status reserved requests equals 10; why: this assertion protects the default profile concurrency limit is shared by alternate ids regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 10 + # What: assert that router status queued requests equals 0; why: this assertion protects the default profile concurrency limit is shared by alternate ids regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 0 + + # What: act by calling leases.release with the declared inputs; why: the default profile concurrency limit is shared by alternate ids scenario observes the leases.release return value during replacement router acquire alternate. + leases[0].release() + # What: act by calling router.acquire and capture replacement; why: the default profile concurrency limit is shared by alternate ids test asserts the response, state, or failure produced by this call. + replacement = router.acquire("alternate") + # What: arrange with pytest raises ValueError match already released for the scenario; why: test raises value error match already released requires this concrete input or helper state before exercising the behavior under test. + with pytest.raises(ValueError, match="already released"): + # What: act by calling leases.release with the declared inputs; why: the default profile concurrency limit is shared by alternate ids scenario observes the leases.release return value during assert router status reserved requests. + leases[0].release() + # What: assert that router status reserved requests equals 10; why: this assertion protects the default profile concurrency limit is shared by alternate ids regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 10 + # What: act by calling replacement.release with the declared inputs; why: the default profile concurrency limit is shared by alternate ids scenario observes the replacement.release return value during for lease in leases. + replacement.release() + # What: act across leases to perform release and lease; why: the default profile concurrency limit is shared by alternate ids scenario repeats the body only while or for the loop header admits an iteration. + for lease in leases[1:]: + # What: act by calling lease.release with the declared inputs; why: the default profile concurrency limit is shared by alternate ids scenario observes the lease.release return value during assert router status reserved requests. + lease.release() + # What: assert that router status reserved requests equals 0; why: this assertion protects the default profile concurrency limit is shared by alternate ids regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 0 + # What: assert that manager calls equals start low gguf; why: this assertion protects the default profile concurrency limit is shared by alternate ids regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf")] + + +# What: define the test_global_concurrency_limit_rejects_conflicting_model_before_it_queues test around local fixtures; why: this test groups the arrange, act, and assertions that protect the global concurrency limit rejects conflicting model before it queues outcome. +def test_global_concurrency_limit_rejects_conflicting_model_before_it_queues(): + # What: act by calling Manager and capture manager; why: the global concurrency limit rejects conflicting model before it queues test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the global concurrency limit rejects conflicting model before it queues test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with low and high; why: test_global_concurrency_limit_rejects_conflicting_model_before_it_queues groups the supplied clauses as one catalog_doc mapping before its value. + { + # What: arrange the low field as model profile and low and low and gguf; why: test_global_concurrency_limit_rejects_conflicting_model_before_it_queues carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "low": ModelProfile("low", "low.gguf", ()), + # What: arrange the high field as model profile and high and high and gguf; why: test_global_concurrency_limit_rejects_conflicting_model_before_it_queues carries high through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "high": ModelProfile("high", "high.gguf", ()), + # What: arrange the catalog_doc mapping with low and high; why: test_global_concurrency_limit_rejects_conflicting_model_before_it_queues groups the supplied clauses as one catalog_doc mapping before its value. + }, + # What: arrange settings to RouterSettings; why: the global concurrency limit rejects conflicting model before it queues scenario binds this router settings and 1 value to RouterSettings's settings input. + settings=RouterSettings(global_concurrency_limit=1), + # What: arrange the ModelCatalog call with settings; why: test_global_concurrency_limit_rejects_conflicting_model_before_it_queues groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the global concurrency limit rejects conflicting model before it queues test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: act by calling router.acquire and capture lease; why: the global concurrency limit rejects conflicting model before it queues test asserts the response, state, or failure produced by this call. + lease = router.acquire("low") + + # What: assert the pytest.raises failure context; why: the global concurrency limit rejects conflicting model before it queues scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RoutingError) as exc: + # What: arrange the exact router acquire high fixture fragment; why: the global concurrency limit rejects conflicting model before it queues scenario feeds this byte-preserved fragment through router.acquire("high") before asserting its protocol or parser result. + router.acquire("high") + # What: assert that exc value code exc value status code equals concurrency limit 429; why: this assertion protects the global concurrency limit rejects conflicting model before it queues regression after the test's arranged inputs and exercised call. + assert (exc.value.code, exc.value.status_code) == ("concurrency_limit", 429) + # What: assert that router status queued requests equals 0; why: this assertion protects the global concurrency limit rejects conflicting model before it queues regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 0 + # What: assert that router status reserved requests equals 1; why: this assertion protects the global concurrency limit rejects conflicting model before it queues regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 1 + # What: act by calling lease.release with the declared inputs; why: the global concurrency limit rejects conflicting model before it queues scenario observes the lease.release return value during the enclosing return. + lease.release() + + +# What: define the test_admission_reservation_reports_cold_queue_position_and_cleans_up_on_cancel test around local fixtures; why: this test groups the arrange, act, and assertions that protect the admission reservation reports cold queue position and cleans up on cancel outcome. +def test_admission_reservation_reports_cold_queue_position_and_cleans_up_on_cancel(): + # What: act by calling Manager and capture manager; why: the admission reservation reports cold queue position and cleans up on cancel test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the admission reservation reports cold queue position and cleans up on cancel test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: act by calling router.acquire and capture active; why: the admission reservation reports cold queue position and cleans up on cancel test asserts the response, state, or failure produced by this call. + active = router.acquire("low") + # What: act by calling threading.Event and capture cancellation; why: the admission reservation reports cold queue position and cleans up on cancel test asserts the response, state, or failure produced by this call. + cancellation = threading.Event() + # What: act by calling threading.Event and capture reserved; why: the admission reservation reports cold queue position and cleans up on cancel test asserts the response, state, or failure produced by this call. + reserved = threading.Event() + # What: arrange cold as the fixture input; why: the admission reservation reports cold queue position and cleans up on cancel test consumes this named precondition before exercising the behavior. + cold = [] + # What: arrange errors as the fixture input; why: the admission reservation reports cold queue position and cleans up on cancel test consumes this named precondition before exercising the behavior. + errors = [] + + # What: define the acquire_high test helper around captured fixture state; why: the admission reservation reports cold queue position and cleans up on cancel scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def acquire_high(): + # What: establish the handler boundary for the protected operation; why: acquire_high routes failures to routing error while preserving cleanup and success flow. + try: + # What: act by calling router.acquire with high and cancellation and append and set and cold and loading required; why: the admission reservation reports cold queue position and cleans up on cancel scenario observes the router.acquire return value during high cancellation lambda loading required position. + router.acquire( + # What: arrange the exact high cancellation lambda loading required position fixture fragment; why: the admission reservation reports cold queue position and cleans up on cancel scenario feeds this byte-preserved fragment through "high", cancellation, lambda loading_required, position: ( before asserti. + "high", cancellation, lambda loading_required, position: ( + # What: act by calling cold.append with loading required and position; why: the admission reservation reports cold queue position and cleans up on cancel scenario observes the cold.append return value while evaluating cold.append((loading_required, position)), reserved.set(). + cold.append((loading_required, position)), reserved.set() + # What: arrange the enclosing predicate collection with append and cold and loading required and position and set and reserved; why: acquire_high groups the supplied clauses as one acquire_high expression collection before its value is consumed. + ), + # What: arrange the router.acquire call with cancellation and append; why: acquire_high groups the supplied clauses as one router.acquire call before its value is consumed. + ) + # What: handle routing error by errors append exc; why: acquire_high converts that failure into this concrete recovery, response, or cleanup behavior. + except RoutingError as exc: + # What: act by calling errors.append with exc; why: the admission reservation reports cold queue position and cleans up on cancel scenario observes the errors.append return value during the enclosing return. + errors.append(exc) + + # What: act by calling threading.Thread and capture thread; why: the admission reservation reports cold queue position and cleans up on cancel test asserts the response, state, or failure produced by this call. + thread = threading.Thread(target=acquire_high) + # What: act by calling thread.start with the declared inputs; why: the admission reservation reports cold queue position and cleans up on cancel scenario observes the thread.start return value during assert reserved wait. + thread.start() + # What: assert that reserved wait 1; why: this assertion protects the admission reservation reports cold queue position and cleans up on cancel regression after the test's arranged inputs and exercised call. + assert reserved.wait(1) + # What: assert that cold equals true 1; why: this assertion protects the admission reservation reports cold queue position and cleans up on cancel regression after the test's arranged inputs and exercised call. + assert cold == [(True, 1)] + # What: assert that router queue position cancellation equals 1; why: this assertion protects the admission reservation reports cold queue position and cleans up on cancel regression after the test's arranged inputs and exercised call. + assert router.queue_position(cancellation) == 1 + # What: act by calling router.cancel_acquire with cancellation; why: the admission reservation reports cold queue position and cleans up on cancel scenario observes the router.cancel_acquire return value during assert router queue position cancellation is. + router.cancel_acquire(cancellation) + # What: assert that router queue position cancellation is group delimiter; why: this assertion protects the admission reservation reports cold queue position and cleans up on cancel regression after the test's arranged inputs and exercised call. + assert router.queue_position(cancellation) is None + # What: assert that router status queued requests equals 0; why: this assertion protects the admission reservation reports cold queue position and cleans up on cancel regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 0 + # What: assert the expected router status reservedRequests == 1 only the active low lease remains outcome; why: test admission reservation reports cold protects its regression by requiring this observable result after the exercised behavior. + assert router.status()["reservedRequests"] == 1 # only the active low lease remains + # What: act by calling thread.join with 1; why: the admission reservation reports cold queue position and cleans up on cancel scenario observes the thread.join return value during assert not thread is alive. + thread.join(1) + # What: assert that thread is alive is false; why: this assertion protects the admission reservation reports cold queue position and cleans up on cancel regression after the test's arranged inputs and exercised call. + assert not thread.is_alive() + # What: assert that error code error status code for error in errors equals request cancelled 409; why: this assertion protects the admission reservation reports cold queue position and cleans up on cancel regression after the test's arranged inputs and exercised call. + assert [(error.code, error.status_code) for error in errors] == [("request_cancelled", 409)] + # What: assert the expected router status reservedRequests == 1 outcome; why: test router test admission reservation reports cold queue position and cleans up on cancel protects its regression by requiring this observable result after the exercised behavior. + assert router.status()["reservedRequests"] == 1 + # What: act by calling active.release with the declared inputs; why: the admission reservation reports cold queue position and cleans up on cancel scenario observes the active.release return value during the enclosing return. + active.release() + + +# What: define the test_admission_reservation_reports_warm_and_callback_failure_releases_capacity test around local fixtures; why: this test groups the arrange, act, and assertions that protect the admission reservation reports warm and callback failure releases capacity outcome. +def test_admission_reservation_reports_warm_and_callback_failure_releases_capacity(): + # What: act by calling Manager and capture manager; why: the admission reservation reports warm and callback failure releases capacity test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the admission reservation reports warm and callback failure releases capacity test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: arrange the exact router acquire low release fixture fragment; why: the admission reservation reports warm and callback failure releases capacity scenario feeds this byte-preserved fragment through router.acquire("low").release() before asserting its protocol or parser result. + router.acquire("low").release() + # What: arrange observed as the fixture input; why: the admission reservation reports warm and callback failure releases capacity test consumes this named precondition before exercising the behavior. + observed = [] + # What: act by calling router.acquire and capture warm; why: the admission reservation reports warm and callback failure releases capacity test asserts the response, state, or failure produced by this call. + warm = router.acquire( + # What: arrange low threading Event lambda loading required position observed append for the scenario; why: test admission reservation reports warm and callback failure releases capacity requires this concrete input or helper state before exercising the behavior under test. + "low", threading.Event(), lambda loading_required, position: observed.append( + # What: arrange the loading required position portion of warm; why: the admission reservation reports warm and callback failure releases capacity scenario uses this clause to evaluate warm as one grouped value. + (loading_required, position) + # What: arrange the observed.append call with loading required; why: test_admission_reservation_reports_warm_and_callback_failure_releases_capacity groups the supplied clauses as one observed.append call before its value is consumed. + ) + # What: arrange the router.acquire call with event and append; why: test_admission_reservation_reports_warm_and_callback_failure_releases_capacity groups the supplied clauses as one router.acquire call before its value is consumed. + ) + # What: assert that observed equals false 1; why: this assertion protects the admission reservation reports warm and callback failure releases capacity regression after the test's arranged inputs and exercised call. + assert observed == [(False, 1)] + # What: act by calling warm.release with the declared inputs; why: the admission reservation reports warm and callback failure releases capacity scenario observes the warm.release return value during def fail loading required position. + warm.release() + + # What: define the fail test helper around loading required and position; why: the admission reservation reports warm and callback failure releases capacity scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def fail(_loading_required, _position): + # What: raise RuntimeError for the caller; why: fail stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("observer failed") + + # What: assert the pytest.raises failure context; why: the admission reservation reports warm and callback failure releases capacity scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError, match="observer failed"): + # What: arrange the exact router acquire low threading event fail fixture fragment; why: the admission reservation reports warm and callback failure releases capacity scenario feeds this byte-preserved fragment through router.acquire("low", threading.Event(), fail) before asserting its protocol or parser resul. + router.acquire("low", threading.Event(), fail) + # What: assert that router status reserved requests equals 0; why: this assertion protects the admission reservation reports warm and callback failure releases capacity regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 0 + # What: assert that router status queued requests equals 0; why: this assertion protects the admission reservation reports warm and callback failure releases capacity regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 0 + + +# What: define the test_http_concurrency_rejection_returns_retry_after_and_releases_request_id test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the http concurrency rejection returns retry after and releases request id outcome. +def test_http_concurrency_rejection_returns_retry_after_and_releases_request_id(monkeypatch): + # What: act by calling Manager and capture manager; why: the http concurrency rejection returns retry after and releases request id test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelProfile and capture profile; why: the http concurrency rejection returns retry after and releases request id test asserts the response, state, or failure produced by this call. + profile = ModelProfile("low", "low.gguf", (), concurrency_limit=1) + # What: act by calling ModelCatalog and capture catalog doc; why: the http concurrency rejection returns retry after and releases request id test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({"low": profile}) + # What: act by calling RoutingCoordinator and capture router; why: the http concurrency rejection returns retry after and releases request id test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: act by calling router.acquire and capture lease; why: the http concurrency rejection returns retry after and releases request id test asserts the response, state, or failure produced by this call. + lease = router.acquire("low") + # What: arrange monkeypatch setattr for the scenario; why: test http concurrency rejection returns retry after and releases request id requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the http concurrency rejection returns retry after and releases request id scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the exact lambda kwargs pytest fail over limit request reached fixture fragment; why: the http concurrency rejection returns retry after and releases request id scenario feeds this byte-preserved fragment through lambda **kwargs: pytest.fail("over-limit request reached upstream") before asserti. + lambda **kwargs: pytest.fail("over-limit request reached upstream"), + # What: arrange the monkeypatch.setattr call with fail; why: test_http_concurrency_rejection_returns_retry_after_and_releases_request_id groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_http_concurrency_rejection_returns_retry_after_and_releases_request_id releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the http concurrency rejection returns retry after and releases request id test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_http_concurrency_rejection_returns_retry_after_and_releases_request_id; why: test_http_concurrency_rejection_returns_retry_after_and_releases_request_id consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the http concurrency rejection returns retry after and releases request id scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_http_concurrency_rejection_returns_retry_after_and_releases_request_id groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the http concurrency rejection returns retry after and releases request id test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling client.post and capture rejected; why: the http concurrency rejection returns retry after and releases request id test asserts the response, state, or failure produced by this call. + rejected = client.post( + # What: arrange the v1 messages portion of rejected; why: the http concurrency rejection returns retry after and releases request id scenario uses this clause to evaluate rejected as one grouped value. + "/v1/messages", + # What: arrange the model field as low; why: test_http_concurrency_rejection_returns_retry_after_and_releases_request_id sends this field through rejected so the router selects the canonical model or alias for upstream dispatch. + json={"model": "low", "messages": []}, + # What: arrange the x ft request id field as over limit; why: test_http_concurrency_rejection_returns_retry_after_and_releases_request_id carries x ft request id through rejected into assert rejected status code equals 429. + headers={"X-FT-Request-ID": "over-limit"}, + # What: arrange the client.post call with json and headers; why: test_http_concurrency_rejection_returns_retry_after_and_releases_request_id groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: assert that client get router requests json data equals group delimiter; why: this assertion protects the http concurrency rejection returns retry after and releases request id regression after the test's arranged inputs and exercised call. + assert client.get("/router/requests").json()["data"] == [] + + # What: assert that rejected status code equals 429; why: this assertion protects the http concurrency rejection returns retry after and releases request id regression after the test's arranged inputs and exercised call. + assert rejected.status_code == 429 + # What: assert that rejected headers retry after equals 1; why: this assertion protects the http concurrency rejection returns retry after and releases request id regression after the test's arranged inputs and exercised call. + assert rejected.headers["retry-after"] == "1" + # What: assert that rejected json error type equals concurrency limit; why: this assertion protects the http concurrency rejection returns retry after and releases request id regression after the test's arranged inputs and exercised call. + assert rejected.json()["error"]["type"] == "concurrency_limit" + # What: assert that router status reserved requests equals 1; why: this assertion protects the http concurrency rejection returns retry after and releases request id regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 1 + # What: act by calling lease.release with the declared inputs; why: the http concurrency rejection returns retry after and releases request id scenario observes the lease.release return value during the enclosing return. + lease.release() + + +# What: define the test_dynamic_port_failure_releases_concurrency_reservation test around local fixtures; why: this test groups the arrange, act, and assertions that protect the dynamic port failure releases concurrency reservation outcome. +def test_dynamic_port_failure_releases_concurrency_reservation(): + # What: act by calling RoutingCoordinator and capture router; why: the dynamic port failure releases concurrency reservation test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator( + # What: act by calling Manager with the declared inputs; why: the dynamic port failure releases concurrency reservation scenario observes the Manager return value during model catalog dynamic model profile dynamic dynamic gguf port. + Manager(), + # What: arrange the dynamic field as model profile and dynamic and dynamic and gguf and 0; why: test_dynamic_port_failure_releases_concurrency_reservation carries dynamic through router into router acquire dynamic. + ModelCatalog({"dynamic": ModelProfile("dynamic", "dynamic.gguf", (), port=0)}), + # What: act by calling object with the declared inputs; why: the dynamic port failure releases concurrency reservation scenario observes the object return value during ready fn ready. + object(), + # What: arrange ready fn to RoutingCoordinator; why: the dynamic port failure releases concurrency reservation scenario binds this ready value to RoutingCoordinator's ready fn input. + ready_fn=ready, + # What: arrange port allocator to operation.throw; why: the dynamic port failure releases concurrency reservation scenario binds this throw and oserror and value and no and port value to operation.throw's port allocator input. + port_allocator=lambda: (_ for _ in ()).throw(OSError("no port")), + # What: arrange the RoutingCoordinator call with ready fn and port allocator; why: test_dynamic_port_failure_releases_concurrency_reservation groups the supplied clauses as one RoutingCoordinator call before its value is consumed. + ) + + # What: assert the pytest.raises failure context; why: the dynamic port failure releases concurrency reservation scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(OSError, match="no port"): + # What: arrange the exact router acquire dynamic fixture fragment; why: the dynamic port failure releases concurrency reservation scenario feeds this byte-preserved fragment through router.acquire("dynamic") before asserting its protocol or parser result. + router.acquire("dynamic") + # What: assert that router status reserved requests equals 0; why: this assertion protects the dynamic port failure releases concurrency reservation regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 0 + # What: assert that router status queued requests equals 0; why: this assertion protects the dynamic port failure releases concurrency reservation regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 0 + + +# What: define the test_concurrent_cold_dynamic_requests_share_one_head_ticket_port test around local fixtures; why: this test groups the arrange, act, and assertions that protect the concurrent cold dynamic requests share one head ticket port outcome. +def test_concurrent_cold_dynamic_requests_share_one_head_ticket_port(): + # What: act by calling Manager and capture manager; why: the concurrent cold dynamic requests share one head ticket port test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling threading.Event and capture activation started; why: the concurrent cold dynamic requests share one head ticket port test asserts the response, state, or failure produced by this call. + activation_started = threading.Event() + # What: act by calling threading.Event and capture finish activation; why: the concurrent cold dynamic requests share one head ticket port test asserts the response, state, or failure produced by this call. + finish_activation = threading.Event() + # What: arrange allocated as the fixture input; why: the concurrent cold dynamic requests share one head ticket port test consumes this named precondition before exercising the behavior. + allocated = [] + + # What: define the allocate test helper around captured fixture state; why: the concurrent cold dynamic requests share one head ticket port scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def allocate(): + # What: act by calling len and capture port; why: the concurrent cold dynamic requests share one head ticket port test asserts the response, state, or failure produced by this call. + port = 21000 + len(allocated) + # What: act by calling allocated.append with port; why: the concurrent cold dynamic requests share one head ticket port scenario observes the allocated.append return value during return port. + allocated.append(port) + # What: return port from the allocate test helper; why: the concurrent cold dynamic requests share one head ticket port scenario uses this helper result in its subsequent act or assertion. + return port + + # What: define the blocking_ready test helper around manager and probe and pid and port and timeout s; why: the concurrent cold dynamic requests share one head ticket port scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def blocking_ready(manager, probe, *, pid, port, timeout_s): + # What: act by calling activation_started.set with the declared inputs; why: the concurrent cold dynamic requests share one head ticket port scenario observes the activation_started.set return value during assert finish activation wait. + activation_started.set() + # What: assert that finish activation wait 2; why: this assertion protects the concurrent cold dynamic requests share one head ticket port regression after the test's arranged inputs and exercised call. + assert finish_activation.wait(2) + # What: arrange the ready field as true; why: blocking_ready carries ready into return {"ready": True}. + return {"ready": True} + + # What: act by calling RoutingCoordinator and capture router; why: the concurrent cold dynamic requests share one head ticket port test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator( + # What: arrange the manager portion of router; why: the concurrent cold dynamic requests share one head ticket port scenario uses this clause to evaluate router as one grouped value. + manager, + # What: arrange the dynamic field as model profile and dynamic and dynamic and gguf and 0; why: test_concurrent_cold_dynamic_requests_share_one_head_ticket_port carries dynamic through router into first threading thread target lambda leases append router acquire dynamic. + ModelCatalog({"dynamic": ModelProfile("dynamic", "dynamic.gguf", (), port=0)}), + # What: act by calling object with the declared inputs; why: the concurrent cold dynamic requests share one head ticket port scenario observes the object return value during ready fn blocking ready. + object(), + # What: arrange ready fn to RoutingCoordinator; why: the concurrent cold dynamic requests share one head ticket port scenario binds this blocking ready value to RoutingCoordinator's ready fn input. + ready_fn=blocking_ready, + # What: arrange port allocator to RoutingCoordinator; why: the concurrent cold dynamic requests share one head ticket port scenario binds this allocate value to RoutingCoordinator's port allocator input. + port_allocator=allocate, + # What: arrange the RoutingCoordinator call with ready fn and port allocator; why: test_concurrent_cold_dynamic_requests_share_one_head_ticket_port groups the supplied clauses as one RoutingCoordinator call before its value is consumed. + ) + # What: arrange leases as the fixture input; why: the concurrent cold dynamic requests share one head ticket port test consumes this named precondition before exercising the behavior. + leases = [] + # What: act by calling threading.Thread and capture first; why: the concurrent cold dynamic requests share one head ticket port test asserts the response, state, or failure produced by this call. + first = threading.Thread(target=lambda: leases.append(router.acquire("dynamic"))) + # What: act by calling threading.Thread and capture second; why: the concurrent cold dynamic requests share one head ticket port test asserts the response, state, or failure produced by this call. + second = threading.Thread(target=lambda: leases.append(router.acquire("dynamic"))) + # What: act by calling first.start with the declared inputs; why: the concurrent cold dynamic requests share one head ticket port scenario observes the first.start return value during assert activation started wait. + first.start() + # What: assert that activation started wait 1; why: this assertion protects the concurrent cold dynamic requests share one head ticket port regression after the test's arranged inputs and exercised call. + assert activation_started.wait(1) + # What: act by calling second.start with the declared inputs; why: the concurrent cold dynamic requests share one head ticket port scenario observes the second.start return value during for value in range. + second.start() + # What: act across range to perform status and router; why: the concurrent cold dynamic requests share one head ticket port scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on status and router before the computed value; why: the concurrent cold dynamic requests share one head ticket port scenario admits the computed value only for this predicate and excludes the opposite state. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the concurrent cold dynamic requests share one head ticket port scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling time.sleep with 0 01; why: the concurrent cold dynamic requests share one head ticket port scenario observes the time.sleep return value during assert router status queued requests. + time.sleep(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the concurrent cold dynamic requests share one head ticket port regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + # What: act by calling finish_activation.set with the declared inputs; why: the concurrent cold dynamic requests share one head ticket port scenario observes the finish_activation.set return value during first join. + finish_activation.set() + # What: act by calling first.join with 2; why: the concurrent cold dynamic requests share one head ticket port scenario observes the first.join return value during second join. + first.join(2) + # What: act by calling second.join with 2; why: the concurrent cold dynamic requests share one head ticket port scenario observes the second.join return value during assert not first is alive and not second is alive. + second.join(2) + + # What: assert that not first is alive and not second is alive; why: this assertion protects the concurrent cold dynamic requests share one head ticket port regression after the test's arranged inputs and exercised call. + assert not first.is_alive() and not second.is_alive() + # What: assert that allocated equals 21000; why: this assertion protects the concurrent cold dynamic requests share one head ticket port regression after the test's arranged inputs and exercised call. + assert allocated == [21000] + # What: assert that lease port for lease in leases equals 21000 21000; why: this assertion protects the concurrent cold dynamic requests share one head ticket port regression after the test's arranged inputs and exercised call. + assert [lease.port for lease in leases] == [21000, 21000] + # What: assert that manager calls equals start dynamic gguf; why: this assertion protects the concurrent cold dynamic requests share one head ticket port regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "dynamic.gguf")] + # What: act across leases to perform release and lease; why: the concurrent cold dynamic requests share one head ticket port scenario repeats the body only while or for the loop header admits an iteration. + for lease in leases: + # What: act by calling lease.release with the declared inputs; why: the concurrent cold dynamic requests share one head ticket port scenario observes the lease.release return value during the enclosing return. + lease.release() + + +# What: define the test_explicit_cancel_removes_a_queued_request_before_it_can_swap test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the explicit cancel removes a queued request before it can swap outcome. +def test_explicit_cancel_removes_a_queued_request_before_it_can_swap(monkeypatch): + # What: act by calling Manager and capture manager; why: the explicit cancel removes a queued request before it can swap test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the explicit cancel removes a queued request before it can swap test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: act by calling router.acquire and capture active; why: the explicit cancel removes a queued request before it can swap test asserts the response, state, or failure produced by this call. + active = router.acquire("low") + # What: arrange monkeypatch setattr for the scenario; why: test explicit cancel removes a queued request before it can swap requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the explicit cancel removes a queued request before it can swap scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the exact lambda kwargs pytest fail cancelled queued request fixture fragment; why: the explicit cancel removes a queued request before it can swap scenario feeds this byte-preserved fragment through lambda **kwargs: pytest.fail("cancelled queued request reached upstream" before asserting its p. + lambda **kwargs: pytest.fail("cancelled queued request reached upstream"), + # What: arrange the monkeypatch.setattr call with fail; why: test_explicit_cancel_removes_a_queued_request_before_it_can_swap groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + + # What: define the scenario test helper around app; why: the explicit cancel removes a queued request before it can swap scenario calls this helper to produce or observe the exact behavior checked by its assertions. + async def scenario(app): + # What: act by calling httpx.ASGITransport and capture transport; why: the explicit cancel removes a queued request before it can swap test asserts the response, state, or failure produced by this call. + transport = httpx.ASGITransport(app=app) + # What: arrange async with httpx AsyncClient transport transport base url http test as client for the scenario; why: test explicit cancel removes a queued request requires this concrete input or helper state before exercising the behavior under test. + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + # What: act by calling asyncio.create_task and capture request; why: the explicit cancel removes a queued request before it can swap test asserts the response, state, or failure produced by this call. + request = asyncio.create_task(client.post( + # What: arrange the model field as high; why: scenario sends this field through request so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "high"}, + # What: arrange the x ft request id field as operator cancelled queue; why: scenario carries x ft request id through request into response await asyncio wait for request 1. + headers={"X-FT-Request-ID": "operator-cancelled-queue"}, + # What: arrange the grouped source fragment for the scenario; why: test router test explicit cancel removes a queued request before it can swap requires this concrete input. + )) + # What: act across range to perform status and router; why: the explicit cancel removes a queued request before it can swap scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on status and router before the computed value; why: the explicit cancel removes a queued request before it can swap scenario admits the computed value only for this predicate and excludes the opposite state. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the explicit cancel removes a queued request before it can swap scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the explicit cancel removes a queued request before it can swap scenario observes the asyncio.sleep return value during assert router status queued requests. + await asyncio.sleep(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the explicit cancel removes a queued request before it can swap regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + # What: assert the expected await client get router requests json data == outcome; why: test router test explicit cancel removes a queued request before it can swap protects its regression by requiring this observable result after the exercised behavior. + assert (await client.get("/router/requests")).json()["data"] == [ + # What: arrange id operator cancelled queue profile high for the scenario; why: test router test explicit cancel removes a queued request before it can swap requires this concrete input or helper state before exercising the behavior under test. + {"id": "operator-cancelled-queue", "profile": "high"} + # What: arrange the grouped source fragment for the scenario; why: test router test explicit cancel removes a queued request before it can swap requires this concrete. + ] + # What: act by calling client.post and capture cancelled; why: the explicit cancel removes a queued request before it can swap test asserts the response, state, or failure produced by this call. + cancelled = await client.post( + # What: arrange the router requests operator cancelled queue cancel portion of cancelled; why: the explicit cancel removes a queued request before it can swap scenario uses this clause to evaluate cancelled as one grouped value. + "/router/requests/operator-cancelled-queue/cancel" + # What: arrange the client.post call with ordered positional inputs; why: scenario groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: assert the expected cancelled json == outcome; why: test router test explicit cancel removes a queued request before it can swap protects its regression by requiring this observable result after the exercised behavior. + assert cancelled.json() == { + # What: arrange cancelled True id operator cancelled queue for the scenario; why: test router test explicit cancel removes a queued request before it can swap requires this concrete input or helper state before exercising the behavior under test. + "cancelled": True, "id": "operator-cancelled-queue" + # What: arrange the grouped source fragment for the scenario; why: test router test explicit cancel removes a queued request before it can swap requires this concrete input. + } + # What: act by calling client.post and capture repeated; why: the explicit cancel removes a queued request before it can swap test asserts the response, state, or failure produced by this call. + repeated = await client.post( + # What: arrange the router requests operator cancelled queue cancel portion of repeated; why: the explicit cancel removes a queued request before it can swap scenario uses this clause to evaluate repeated as one grouped value. + "/router/requests/operator-cancelled-queue/cancel" + # What: arrange the client.post call with ordered positional inputs; why: scenario groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: assert that repeated json equals cancelled false reason not found; why: this assertion protects the explicit cancel removes a queued request before it can swap regression after the test's arranged inputs and exercised call. + assert repeated.json() == {"cancelled": False, "reason": "not_found"} + # What: act by calling asyncio.wait_for and capture response; why: the explicit cancel removes a queued request before it can swap test asserts the response, state, or failure produced by this call. + response = await asyncio.wait_for(request, 1) + # What: assert that response status code equals 409; why: this assertion protects the explicit cancel removes a queued request before it can swap regression after the test's arranged inputs and exercised call. + assert response.status_code == 409 + # What: assert that response json error type equals request cancelled; why: this assertion protects the explicit cancel removes a queued request before it can swap regression after the test's arranged inputs and exercised call. + assert response.json()["error"]["type"] == "request_cancelled" + # What: assert that await client get router requests json data equals group delimiter; why: this assertion protects the explicit cancel removes a queued request before it can swap regression after the test's arranged inputs and exercised call. + assert (await client.get("/router/requests")).json()["data"] == [] + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_explicit_cancel_removes_a_queued_request_before_it_can_swap releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(2) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the explicit cancel removes a queued request before it can swap test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_explicit_cancel_removes_a_queued_request_before_it_can_swap; why: test_explicit_cancel_removes_a_queued_request_before_it_can_swap consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to catalog; why: the explicit cancel removes a queued request before it can swap scenario binds this lifecycle value to catalog's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog(), router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_explicit_cancel_removes_a_queued_request_before_it_can_swap groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling asyncio.run with scenario and app; why: the explicit cancel removes a queued request before it can swap scenario observes the asyncio.run return value during active release. + asyncio.run(scenario(app)) + + # What: act by calling active.release with the declared inputs; why: the explicit cancel removes a queued request before it can swap scenario observes the active.release return value during assert manager calls start low gguf. + active.release() + # What: assert that manager calls equals start low gguf; why: this assertion protects the explicit cancel removes a queued request before it can swap regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf")] + # What: assert that router status queued requests equals 0; why: this assertion protects the explicit cancel removes a queued request before it can swap regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 0 + # What: assert that router status active requests equals 0; why: this assertion protects the explicit cancel removes a queued request before it can swap regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + # What: assert that router status cancellations equals 1; why: this assertion protects the explicit cancel removes a queued request before it can swap regression after the test's arranged inputs and exercised call. + assert router.status()["cancellations"] == 1 + + +# What: define the test_queued_higher_priority_profile_runs_before_an_earlier_lower_priority_request test around local fixtures; why: this test groups the arrange, act, and assertions that protect the queued higher priority profile runs before an earlier lower priority request outcome. +def test_queued_higher_priority_profile_runs_before_an_earlier_lower_priority_request(): + # What: act by calling Manager and capture manager; why: the queued higher priority profile runs before an earlier lower priority request test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the queued higher priority profile runs before an earlier lower priority request test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({ + # What: arrange the active field as model profile and active and active and gguf and 0; why: test_queued_higher_priority_profile_runs_before_an_earlier_lower_priority_request carries active through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "active": ModelProfile("active", "active.gguf", (), priority=0), + # What: arrange the low field as model profile and low and low and gguf and 0; why: test_queued_higher_priority_profile_runs_before_an_earlier_lower_priority_request carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "low": ModelProfile("low", "low.gguf", (), priority=0), + # What: arrange the high field as model profile and high and high and gguf and 10; why: test_queued_higher_priority_profile_runs_before_an_earlier_lower_priority_request carries high through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "high": ModelProfile("high", "high.gguf", (), priority=10), + # What: arrange the ModelCatalog call with model profile; why: test_queued_higher_priority_profile_runs_before_an_earlier_lower_priority_request groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + # What: act by calling RoutingCoordinator and capture router; why: the queued higher priority profile runs before an earlier lower priority request test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: act by calling router.acquire and capture active; why: the queued higher priority profile runs before an earlier lower priority request test asserts the response, state, or failure produced by this call. + active = router.acquire("active") + # What: arrange completed as the fixture input; why: the queued higher priority profile runs before an earlier lower priority request test consumes this named precondition before exercising the behavior. + completed = [] + + # What: define the acquire_then_release test helper around name; why: the queued higher priority profile runs before an earlier lower priority request scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def acquire_then_release(name): + # What: act by calling router.acquire and capture lease; why: the queued higher priority profile runs before an earlier lower priority request test asserts the response, state, or failure produced by this call. + lease = router.acquire(name) + # What: act by calling completed.append with name; why: the queued higher priority profile runs before an earlier lower priority request scenario observes the completed.append return value during lease release. + completed.append(name) + # What: act by calling lease.release with the declared inputs; why: the queued higher priority profile runs before an earlier lower priority request scenario observes the lease.release return value during the enclosing return. + lease.release() + + # What: act by calling threading.Thread and capture low thread; why: the queued higher priority profile runs before an earlier lower priority request test asserts the response, state, or failure produced by this call. + low_thread = threading.Thread(target=acquire_then_release, args=("low",)) + # What: act by calling threading.Thread and capture high thread; why: the queued higher priority profile runs before an earlier lower priority request test asserts the response, state, or failure produced by this call. + high_thread = threading.Thread(target=acquire_then_release, args=("high",)) + # What: act by calling low_thread.start with the declared inputs; why: the queued higher priority profile runs before an earlier lower priority request scenario observes the low_thread.start return value during for value in range. + low_thread.start() + # What: act across range to perform status and router; why: the queued higher priority profile runs before an earlier lower priority request scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: arrange if router status queuedRequests == 1 for the scenario; why: test router test queued higher priority profile runs before an earlier lower priority request requires this concrete input or helper state before exercising the behavior under test. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the queued higher priority profile runs before an earlier lower priority request scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling operation.wait with 0 01; why: the queued higher priority profile runs before an earlier lower priority request scenario observes the operation.wait return value during assert router status queued requests. + threading.Event().wait(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the queued higher priority profile runs before an earlier lower priority request regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + # What: act by calling high_thread.start with the declared inputs; why: the queued higher priority profile runs before an earlier lower priority request scenario observes the high_thread.start return value during for value in range. + high_thread.start() + # What: act across range to perform status and router; why: the queued higher priority profile runs before an earlier lower priority request scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: arrange if router status queuedRequests == 2 for the scenario; why: test router test queued higher priority profile runs before an earlier lower priority request requires this concrete input or helper state before exercising the behavior under test. + if router.status()["queuedRequests"] == 2: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the queued higher priority profile runs before an earlier lower priority request scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling operation.wait with 0 01; why: the queued higher priority profile runs before an earlier lower priority request scenario observes the operation.wait return value during assert router status queued requests. + threading.Event().wait(0.01) + # What: assert that router status queued requests equals 2; why: this assertion protects the queued higher priority profile runs before an earlier lower priority request regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 2 + # What: act by calling active.release with the declared inputs; why: the queued higher priority profile runs before an earlier lower priority request scenario observes the active.release return value during low thread join. + active.release() + # What: act by calling low_thread.join with 1; why: the queued higher priority profile runs before an earlier lower priority request scenario observes the low_thread.join return value during high thread join. + low_thread.join(1) + # What: act by calling high_thread.join with 1; why: the queued higher priority profile runs before an earlier lower priority request scenario observes the high_thread.join return value during assert not low thread is alive and not high thread is alive. + high_thread.join(1) + # What: assert that not low thread is alive and not high thread is alive; why: this assertion protects the queued higher priority profile runs before an earlier lower priority request regression after the test's arranged inputs and exercised call. + assert not low_thread.is_alive() and not high_thread.is_alive() + # What: assert the expected manager calls == outcome; why: test router test queued higher priority profile runs before an earlier lower priority request protects its regression by requiring this observable result after the exercised behavior. + assert manager.calls == [ + # What: arrange start active gguf for the scenario; why: test router test queued higher priority profile runs before an earlier lower priority request requires this concrete input or helper state before exercising the behavior under test. + ("start", "active.gguf"), + # What: arrange switch high gguf for the scenario; why: test router test queued higher priority profile runs before an earlier lower priority request requires this concrete input or helper state before exercising the behavior under test. + ("switch", "high.gguf"), + # What: arrange switch low gguf for the scenario; why: test router test queued higher priority profile runs before an earlier lower priority request requires this concrete input or helper state before exercising the behavior under test. + ("switch", "low.gguf"), + # What: arrange the grouped source fragment for the scenario; why: test router test queued higher priority profile runs before an earlier lower priority request requires this concrete input or helper state before exercising the behavior under test. + ] + # What: assert that completed equals high low; why: this assertion protects the queued higher priority profile runs before an earlier lower priority request regression after the test's arranged inputs and exercised call. + assert completed == ["high", "low"] + + +# What: define the test_failed_readiness_restores_previous_engine_before_reporting_error test around local fixtures; why: this test groups the arrange, act, and assertions that protect the failed readiness restores previous engine before reporting error outcome. +def test_failed_readiness_restores_previous_engine_before_reporting_error(): + # What: act by calling Manager and capture manager; why: the failed readiness restores previous engine before reporting error test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by evaluating router RoutingCoordinator manager catalog object ready fn ready; why: test router test failed readiness restores previous engine before reporting error captures the behavior or response that its following assertions inspect. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: arrange the exact router acquire low release fixture fragment; why: the failed readiness restores previous engine before reporting error scenario feeds this byte-preserved fragment through router.acquire("low").release() before asserting its protocol or parser result. + router.acquire("low").release() + + # What: define the not_ready test helper around manager and probe and pid and port and timeout s; why: the failed readiness restores previous engine before reporting error scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def not_ready(manager, probe, *, pid, port, timeout_s): + # What: arrange the ready field as false; why: not_ready carries ready into return {"ready": False, "reason": "engine-error"}. + return {"ready": False, "reason": "engine-error"} + + # What: act by evaluating router RoutingCoordinator manager catalog object ready fn not ready; why: test router test failed readiness restores previous engine before reporting error captures the behavior or response that its following assertions inspect. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=not_ready) + # What: assert the pytest.raises failure context; why: the failed readiness restores previous engine before reporting error scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RoutingError, match="not ready") as exc: + # What: arrange the exact router acquire high fixture fragment; why: the failed readiness restores previous engine before reporting error scenario feeds this byte-preserved fragment through router.acquire("high") before asserting its protocol or parser result. + router.acquire("high") + # What: assert that exc value code equals engine not ready; why: this assertion protects the failed readiness restores previous engine before reporting error regression after the test's arranged inputs and exercised call. + assert exc.value.code == "engine_not_ready" + # What: assert that exc value recovery launched is true; why: this assertion protects the failed readiness restores previous engine before reporting error regression after the test's arranged inputs and exercised call. + assert exc.value.recovery["launched"] is True + # What: assert that router status activating profile is group delimiter; why: this assertion protects the failed readiness restores previous engine before reporting error regression after the test's arranged inputs and exercised call. + assert router.status()["activatingProfile"] is None + # What: act by calling router.model_listing_snapshot and capture and loaded profiles; why: the failed readiness restores previous engine before reporting error test asserts the response, state, or failure produced by this call. + _, loaded_profiles = router.model_listing_snapshot() + # What: assert that loaded profiles equals frozenset low; why: this assertion protects the failed readiness restores previous engine before reporting error regression after the test's arranged inputs and exercised call. + assert loaded_profiles == frozenset({"low"}) + # What: assert that manager model equals low gguf; why: this assertion protects the failed readiness restores previous engine before reporting error regression after the test's arranged inputs and exercised call. + assert manager.model == "low.gguf" + + +# What: define the test_ttl_evicts_only_after_final_lease_and_uses_profile_timeout test around local fixtures; why: this test groups the arrange, act, and assertions that protect the ttl evicts only after final lease and uses profile timeout outcome. +def test_ttl_evicts_only_after_final_lease_and_uses_profile_timeout(): + # What: define Timer as the owner of __init__ and start and cancel; why: daemon callers use this class boundary so those methods share one timer state invariant. + class Timer: + # What: define the __init__ test helper around delay and callback; why: the ttl evicts only after final lease and uses profile timeout scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def __init__(self, delay, callback): + # What: arrange delay as delay; why: the ttl evicts only after final lease and uses profile timeout test consumes this named precondition before exercising the behavior. + self.delay = delay + # What: arrange callback as callback; why: the ttl evicts only after final lease and uses profile timeout test consumes this named precondition before exercising the behavior. + self.callback = callback + # What: arrange started as false; why: the ttl evicts only after final lease and uses profile timeout test consumes this named precondition before exercising the behavior. + self.started = False + # What: arrange cancelled as false; why: the ttl evicts only after final lease and uses profile timeout test consumes this named precondition before exercising the behavior. + self.cancelled = False + + # What: define the start test helper around captured fixture state; why: the ttl evicts only after final lease and uses profile timeout scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def start(self): + # What: arrange started as true; why: the ttl evicts only after final lease and uses profile timeout test consumes this named precondition before exercising the behavior. + self.started = True + + # What: define the cancel test helper around captured fixture state; why: the ttl evicts only after final lease and uses profile timeout scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def cancel(self): + # What: arrange cancelled as true; why: the ttl evicts only after final lease and uses profile timeout test consumes this named precondition before exercising the behavior. + self.cancelled = True + + # What: act by calling Manager and capture manager; why: the ttl evicts only after final lease and uses profile timeout test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the ttl evicts only after final lease and uses profile timeout test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({ + # What: arrange the low field as model profile and low and low and gguf and 12; why: test_ttl_evicts_only_after_final_lease_and_uses_profile_timeout carries low through catalog doc into manager catalog doc object ready fn ready. + "low": ModelProfile("low", "low.gguf", (), ttl_s=12, unload_timeout_s=7), + # What: arrange the ModelCatalog call with model profile; why: test_ttl_evicts_only_after_final_lease_and_uses_profile_timeout groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + # What: arrange timers as the fixture input; why: the ttl evicts only after final lease and uses profile timeout test consumes this named precondition before exercising the behavior. + timers = [] + # What: act by calling RoutingCoordinator and capture router; why: the ttl evicts only after final lease and uses profile timeout test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator( + # What: arrange ready fn to object; why: the ttl evicts only after final lease and uses profile timeout scenario binds this ready value to object's ready fn input. + manager, catalog_doc, object(), ready_fn=ready, + # What: arrange the delay input for test_ttl_evicts_only_after_final_lease_and_uses_profile_timeout; why: test_ttl_evicts_only_after_final_lease_and_uses_profile_timeout consumes delay during signature binding, so callers must bind it with the other signature inputs. + timer_factory=lambda delay, callback: timers.append(Timer(delay, callback)) or timers[-1], + # What: arrange the RoutingCoordinator call with ready fn and timer factory; why: test_ttl_evicts_only_after_final_lease_and_uses_profile_timeout groups the supplied clauses as one RoutingCoordinator call before its value is consumed. + ) + # What: act by calling router.acquire and capture first; why: the ttl evicts only after final lease and uses profile timeout test asserts the response, state, or failure produced by this call. + first = router.acquire("low") + # What: act by calling router.acquire and capture second; why: the ttl evicts only after final lease and uses profile timeout test asserts the response, state, or failure produced by this call. + second = router.acquire("low") + # What: act by calling first.release with the declared inputs; why: the ttl evicts only after final lease and uses profile timeout scenario observes the first.release return value during assert timers. + first.release() + # What: assert that timers equals group delimiter; why: this assertion protects the ttl evicts only after final lease and uses profile timeout regression after the test's arranged inputs and exercised call. + assert timers == [] + # What: act by calling second.release with the declared inputs; why: the ttl evicts only after final lease and uses profile timeout scenario observes the second.release return value during assert len timers. + second.release() + # What: assert that len timers equals 1; why: this assertion protects the ttl evicts only after final lease and uses profile timeout regression after the test's arranged inputs and exercised call. + assert len(timers) == 1 + # What: assert that timers 0 delay equals 12; why: this assertion protects the ttl evicts only after final lease and uses profile timeout regression after the test's arranged inputs and exercised call. + assert timers[0].delay == 12 + # What: assert that timers 0 started is true; why: this assertion protects the ttl evicts only after final lease and uses profile timeout regression after the test's arranged inputs and exercised call. + assert timers[0].started is True + # What: assert that router evict idle low is true; why: this assertion protects the ttl evicts only after final lease and uses profile timeout regression after the test's arranged inputs and exercised call. + assert router.evict_idle("low") is True + # What: assert that manager calls equals start low gguf stop 7; why: this assertion protects the ttl evicts only after final lease and uses profile timeout regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf"), ("stop", 7)] + # What: assert that router status evictions equals 1; why: this assertion protects the ttl evicts only after final lease and uses profile timeout regression after the test's arranged inputs and exercised call. + assert router.status()["evictions"] == 1 + + +# What: define the test_all_supported_openai_and_anthropic_requests_use_native_router_and_preserve_sse test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the all supported openai and anthropic requests use native router and preserve sse outcome. +def test_all_supported_openai_and_anthropic_requests_use_native_router_and_preserve_sse(monkeypatch): + # What: act by calling Manager and capture manager; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({ + # What: arrange the low field as model profile and low and low and gguf; why: test_all_supported_openai_and_anthropic_requests_use_native_router_and_preserve_sse carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "low": ModelProfile("low", "low.gguf", ()), + # What: arrange the ModelCatalog call with model profile; why: test_all_supported_openai_and_anthropic_requests_use_native_router_and_preserve_sse groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + # What: act by calling RoutingCoordinator and capture router; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange calls as the fixture input; why: the all supported openai and anthropic requests use native router and preserve sse test consumes this named precondition before exercising the behavior. + calls = [] + + # What: define the upstream test helper around captured fixture state; why: the all supported openai and anthropic requests use native router and preserve sse scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: act by calling calls.append with kwargs; why: the all supported openai and anthropic requests use native router and preserve sse scenario observes the calls.append return value during return upstream response. + calls.append(kwargs) + # What: return upstream response and bytes io and 200 and content type and x upstream from the upstream test helper; why: the all supported openai and anthropic requests use native router and preserve sse scenario uses this helper result in its subsequent act or assertion. + return UpstreamResponse( + # What: arrange status to UpstreamResponse; why: the all supported openai and anthropic requests use native router and preserve sse scenario binds this 200 value to UpstreamResponse's status input. + status=200, + # What: arrange headers to UpstreamResponse; why: the all supported openai and anthropic requests use native router and preserve sse scenario binds this content type and x upstream and connection and keep alive and transfer encoding value to UpstreamResponse's headers input. + headers={ + # What: arrange Content Type text event stream X Upstream yes for the scenario; why: test router test all supported openai and anthropic requests use native router and preserve sse requires this concrete input or helper state before exercising the behavior under test. + "Content-Type": "text/event-stream", "X-Upstream": "yes", + # What: arrange the connection field as keep alive; why: upstream carries connection into "Connection": "keep-alive", "Keep-Alive": "timeout=5". + "Connection": "keep-alive", "Keep-Alive": "timeout=5", + # What: arrange the transfer encoding field as chunked; why: upstream carries transfer encoding into "Transfer-Encoding": "chunked", "Content-Length": "999". + "Transfer-Encoding": "chunked", "Content-Length": "999", + # What: arrange the enclosing predicate mapping with content type and x upstream and connection and keep alive and transfer encoding; why: upstream groups the supplied clauses as one upstream expression mapping before its value is consumed. + }, + # What: arrange raw to BytesIO; why: the all supported openai and anthropic requests use native router and preserve sse scenario binds this bytes io value to BytesIO's raw input. + raw=BytesIO(b"data: first\\n\\ndata: [DONE]\\n\\n"), + # What: arrange the grouped source fragment for the scenario; why: test router test all supported openai and anthropic requests use native router and preserve sse. + ) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the all supported openai and anthropic requests use native router and preserve sse scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream). + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_all_supported_openai_and_anthropic_requests_use_native_router_and_preserve_sse releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_all_supported_openai_and_anthropic_requests_use_native_router_and_preserve_sse; why: test_all_supported_openai_and_anthropic_requests_use_native_router_and_preserve_sse consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the all supported openai and anthropic requests use native router and preserve sse scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_all_supported_openai_and_anthropic_requests_use_native_router_and_preserve_sse groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act across the computed value to perform response and post and path and client; why: the all supported openai and anthropic requests use native router and preserve sse scenario repeats the body only while or for the loop header admits an iteration. + for path in ( + # What: arrange the v1 chat completions portion of the enclosing predicate; why: this clause remains in the all supported openai and anthropic requests use native router and preserve sse scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "/v1/chat/completions", + # What: arrange the v1 completions portion of the enclosing predicate; why: this clause remains in the all supported openai and anthropic requests use native router and preserve sse scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "/v1/completions", + # What: arrange the v1 responses portion of the enclosing predicate; why: this clause remains in the all supported openai and anthropic requests use native router and preserve sse scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "/v1/responses", + # What: arrange the v1 messages portion of the enclosing predicate; why: this clause remains in the all supported openai and anthropic requests use native router and preserve sse scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "/v1/messages", + # What: arrange the v1 messages count tokens portion of the enclosing predicate; why: this clause remains in the all supported openai and anthropic requests use native router and preserve sse scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "/v1/messages/count_tokens", + # What: arrange the grouped source fragment for the scenario; why: test all supported openai and anthropic requests use native router and preserve sse requires this concrete input or helper state before exercising the behavior under test. + ): + # What: act by calling client.post and capture response; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + response = client.post(path, json={"model": "low", "stream": True}) + # What: assert that response status code equals 200; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + # What: assert that response content equals b data first n ndata done; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert response.content == b"data: first\\n\\ndata: [DONE]\\n\\n" + # What: assert that response headers x upstream equals yes; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert response.headers["x-upstream"] == "yes" + # What: assert that connection is absent from response headers; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert "connection" not in response.headers + # What: assert that keep alive is absent from response headers; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert "keep-alive" not in response.headers + # What: assert that transfer encoding is absent from response headers; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert "transfer-encoding" not in response.headers + # What: assert that content length is absent from response headers; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert "content-length" not in response.headers + # What: act by calling client.get and capture status; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + status = client.get("/router/status") + # What: assert that status status code equals 200; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert status.status_code == 200 + # What: assert that status json active requests equals 0; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert status.json()["activeRequests"] == 0 + # What: act by calling operation.json and capture routed models; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + routed_models = client.get("/router/models").json() + # What: assert that routed models data 0 resident is true; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert routed_models["data"][0]["resident"] is True + # What: assert that routed models capacity equals max resident models 1 available resident slots 0; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert routed_models["capacity"] == {"maxResidentModels": 1, "availableResidentSlots": 0} + # What: assert that client get router profiles json active profile equals low; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert client.get("/router/profiles").json()["activeProfile"] == "low" + # What: act by calling client.get and capture metrics; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + metrics = client.get("/metrics") + # What: assert that metrics status code equals 200; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert metrics.status_code == 200 + # What: assert that freetoken swap admissions total 5 is present in metrics text; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert "freetoken_swap_admissions_total 5" in metrics.text + # What: act by calling client.get and capture passthrough; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + passthrough = client.get("/upstream/low/v1/models?limit=3") + # What: assert that passthrough status code equals 200; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert passthrough.status_code == 200 + # What: arrange legacy body as the fixture input; why: the all supported openai and anthropic requests use native router and preserve sse test consumes this named precondition before exercising the behavior. + legacy_body = b'{"prompt":"fixture","max_tokens":2}' + # What: assert that client post generate content legacy body status code equals 404; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert client.post("/generate", content=legacy_body).status_code == 404 + # What: act by calling client.post and capture legacy; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + legacy = client.post( + # What: arrange content to client.post; why: the all supported openai and anthropic requests use native router and preserve sse scenario binds this legacy body value to client.post's content input. + "/upstream/low/generate", content=legacy_body, + # What: arrange the content type field as application and json; why: test_all_supported_openai_and_anthropic_requests_use_native_router_and_preserve_sse carries content type through legacy into assert legacy status code equals 200. + headers={"Content-Type": "application/json"}, + # What: arrange the client.post call with content and headers; why: test_all_supported_openai_and_anthropic_requests_use_native_router_and_preserve_sse groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: assert that legacy status code equals 200; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert legacy.status_code == 200 + # What: assert that legacy content equals response content; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert legacy.content == response.content + # What: act by calling client.post and capture blocked; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + blocked = client.post("/upstream/low/v1/admin/prepare-stop") + # What: assert that blocked status code equals 403; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert blocked.status_code == 403 + # What: act by calling operation.json and capture activity; why: the all supported openai and anthropic requests use native router and preserve sse test asserts the response, state, or failure produced by this call. + activity = client.get("/router/activity").json() + # What: assert that activity count equals 7; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert activity["count"] == 7 + # What: assert that all row has capture is false for row in activity data; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert all(row["hasCapture"] is False for row in activity["data"]) + # What: assert that client get router activity stats json count equals 7; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert client.get("/router/activity/stats").json()["count"] == 7 + # What: assert that client get f router captures activity data equals 404; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert client.get(f'/router/captures/{activity["data"][0]["id"]}').status_code == 404 + # What: assert that manager calls equals start low gguf; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf")] + # What: assert the expected item path and query for item in calls == outcome; why: test router test all supported openai and anthropic requests use native router and preserve sse protects its regression by requiring this observable result after the exercised behavior. + assert [item["path_and_query"] for item in calls] == [ + # What: arrange v1 chat completions v1 completions v1 responses for the scenario; why: test router test all supported openai and anthropic requests use native router and preserve sse requires this concrete input or helper state before exercising the behavior under test. + "/v1/chat/completions", "/v1/completions", "/v1/responses", + # What: arrange v1 messages v1 messages count tokens v1 models limit 3 generate for the scenario; why: test router test all supported openai and anthropic requests use native router and preserve sse requires this concrete input or helper state before exercising the behavior under test. + "/v1/messages", "/v1/messages/count_tokens", "/v1/models?limit=3", "/generate", + # What: arrange the grouped source fragment for the scenario; why: test router test all supported openai and anthropic requests use native router and preserve sse requires. + ] + # What: assert that calls 2 method equals get; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert calls[-2]["method"] == "GET" + # What: assert that calls 1 method equals post; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert calls[-1]["method"] == "POST" + # What: assert that calls 1 body equals legacy body; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert calls[-1]["body"] == legacy_body + # What: assert that calls 1 timeout s equals 900 0; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert calls[-1]["timeout_s"] == 900.0 + # What: assert that router status active requests equals 0; why: this assertion protects the all supported openai and anthropic requests use native router and preserve sse regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + + +# What: define the test_profile_readiness_path_and_proxy_prefix_target_the_owned_child test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the profile readiness path and proxy prefix target the owned child outcome. +def test_profile_readiness_path_and_proxy_prefix_target_the_owned_child(monkeypatch): + # What: act by calling Manager and capture manager; why: the profile readiness path and proxy prefix target the owned child test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelProfile and capture profile; why: the profile readiness path and proxy prefix target the owned child test asserts the response, state, or failure produced by this call. + profile = ModelProfile( + # What: arrange the low portion of profile; why: the profile readiness path and proxy prefix target the owned child scenario uses this clause to evaluate profile as one grouped value. + "low", + # What: arrange the low gguf portion of profile; why: the profile readiness path and proxy prefix target the owned child scenario uses this clause to evaluate profile as one grouped value. + "low.gguf", + # What: arrange the profile collection with ordered entries; why: test_profile_readiness_path_and_proxy_prefix_target_the_owned_child groups the supplied clauses as one profile collection before its value is consumed. + (), + # What: arrange port to ModelProfile; why: the profile readiness path and proxy prefix target the owned child scenario binds this 1922 value to ModelProfile's port input. + port=1922, + # What: arrange check endpoint to ModelProfile; why: the profile readiness path and proxy prefix target the owned child scenario binds this ready value to ModelProfile's check endpoint input. + check_endpoint="/ready", + # What: arrange proxy to ModelProfile; why: the profile readiness path and proxy prefix target the owned child scenario binds this http and port and gateway value to ModelProfile's proxy input. + proxy="http://127.0.0.1:${PORT}/gateway", + # What: arrange upstream timeout s to ModelProfile; why: the profile readiness path and proxy prefix target the owned child scenario binds this 37 value to ModelProfile's upstream timeout s input. + upstream_timeout_s=37, + # What: arrange the ModelProfile call with port and check endpoint and proxy and upstream timeout s; why: test_profile_readiness_path_and_proxy_prefix_target_the_owned_child groups the supplied clauses as one ModelProfile call before its value is consumed. + ) + # What: act by calling ModelCatalog and capture catalog doc; why: the profile readiness path and proxy prefix target the owned child test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({"low": profile}) + # What: arrange readiness calls as the fixture input; why: the profile readiness path and proxy prefix target the owned child test consumes this named precondition before exercising the behavior. + readiness_calls = [] + # What: arrange upstream calls as the fixture input; why: the profile readiness path and proxy prefix target the owned child test consumes this named precondition before exercising the behavior. + upstream_calls = [] + + # What: define the custom_ready test helper around manager and probe and pid and port and timeout s and path; why: the profile readiness path and proxy prefix target the owned child scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def custom_ready(manager, probe, *, pid, port, timeout_s, path): + # What: act by calling readiness_calls.append with pid and port and timeout s and path; why: the profile readiness path and proxy prefix target the owned child scenario observes the readiness_calls.append return value during return ready health reachable. + readiness_calls.append((pid, port, timeout_s, path)) + # What: arrange the ready field as true; why: custom_ready carries ready into return {"ready": True, "health": {"reachable": True}}. + return {"ready": True, "health": {"reachable": True}} + + # What: define the upstream test helper around captured fixture state; why: the profile readiness path and proxy prefix target the owned child scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: act by calling upstream_calls.append with kwargs; why: the profile readiness path and proxy prefix target the owned child scenario observes the upstream_calls.append return value during return upstream response content type application json bytes io. + upstream_calls.append(kwargs) + # What: arrange the helper response as UpstreamResponse 200 Content Type application json BytesIO b; why: test profile readiness path and proxy prefix feeds this result into the behavior whose outcome is asserted. + return UpstreamResponse(200, {"Content-Type": "application/json"}, BytesIO(b'{}')) + + # What: act by calling RoutingCoordinator and capture router; why: the profile readiness path and proxy prefix target the owned child test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=custom_ready) + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the profile readiness path and proxy prefix target the owned child scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) before assertin. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_profile_readiness_path_and_proxy_prefix_target_the_owned_child releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the profile readiness path and proxy prefix target the owned child test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_profile_readiness_path_and_proxy_prefix_target_the_owned_child; why: test_profile_readiness_path_and_proxy_prefix_target_the_owned_child consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the profile readiness path and proxy prefix target the owned child scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_profile_readiness_path_and_proxy_prefix_target_the_owned_child groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture response; why: the profile readiness path and proxy prefix target the owned child test asserts the response, state, or failure produced by this call. + response = TestClient(app).post( + # What: arrange the model field as low; why: test_profile_readiness_path_and_proxy_prefix_target_the_owned_child sends this field through response so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "low", "messages": []} + # What: arrange the operation.post call with json; why: test_profile_readiness_path_and_proxy_prefix_target_the_owned_child groups the supplied clauses as one operation.post call before its value is consumed. + ) + + # What: assert that response status code equals 200; why: this assertion protects the profile readiness path and proxy prefix target the owned child regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + # What: assert that readiness calls equals 101 1922 120 0 ready; why: this assertion protects the profile readiness path and proxy prefix target the owned child regression after the test's arranged inputs and exercised call. + assert readiness_calls == [(101, 1922, 120.0, "/ready")] + # What: assert that upstream calls 0 base url equals http 127 0 0 1 1922 gateway; why: this assertion protects the profile readiness path and proxy prefix target the owned child regression after the test's arranged inputs and exercised call. + assert upstream_calls[0]["base_url"] == "http://127.0.0.1:1922/gateway" + # What: assert that upstream calls 0 path and query equals v1 chat completions; why: this assertion protects the profile readiness path and proxy prefix target the owned child regression after the test's arranged inputs and exercised call. + assert upstream_calls[0]["path_and_query"] == "/v1/chat/completions" + # What: assert that upstream calls 0 timeout s equals 37; why: this assertion protects the profile readiness path and proxy prefix target the owned child regression after the test's arranged inputs and exercised call. + assert upstream_calls[0]["timeout_s"] == 37 + + # What: define Probe as the owner of fresh_readiness; why: daemon callers use this class boundary so those methods share one probe state invariant. + class Probe: + # What: define the fresh_readiness test helper around port and path; why: the profile readiness path and proxy prefix target the owned child scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def fresh_readiness(self, port, path): + # What: assert that port path equals 1922 ready; why: this assertion protects the profile readiness path and proxy prefix target the owned child regression after the test's arranged inputs and exercised call. + assert (port, path) == (1922, "/ready") + # What: arrange the reachable field as true; why: Probe.fresh_readiness carries reachable into return {"reachable": True, "ready": True}. + return {"reachable": True, "ready": True} + + # What: assert that router is ready probe is true; why: this assertion protects the profile readiness path and proxy prefix target the owned child regression after the test's arranged inputs and exercised call. + assert router.is_ready(Probe()) is True + + +# What: define the test_custom_readiness_path_accepts_real_http_success_without_json test around local fixtures; why: this test groups the arrange, act, and assertions that protect the custom readiness path accepts real http success without json outcome. +def test_custom_readiness_path_accepts_real_http_success_without_json(): + # What: define Handler as the owner of do_GET and log_message; why: daemon callers use this class boundary so those methods share one handler state invariant. + class Handler(BaseHTTPRequestHandler): + # What: define the do_GET test helper around captured fixture state; why: the custom readiness path accepts real http success without json scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def do_GET(self): + # What: assert that self path equals ready; why: this assertion protects the custom readiness path accepts real http success without json regression after the test's arranged inputs and exercised call. + assert self.path == "/ready" + # What: act by calling self.send_response with 204; why: the custom readiness path accepts real http success without json scenario observes the self.send_response return value during self end headers. + self.send_response(204) + # What: act by calling self.end_headers with the declared inputs; why: the custom readiness path accepts real http success without json scenario observes the self.end_headers return value during the enclosing return. + self.end_headers() + + # What: define the log_message test helper around format; why: the custom readiness path accepts real http success without json scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def log_message(self, format, *args): + # What: ignore the anticipated exception handled by this branch; why: log_message continues its retry or cleanup path instead of re-raising that transient failure. + pass + + # What: define RunningManager as the owner of status; why: daemon callers use this class boundary so those methods share one running manager state invariant. + class RunningManager: + # What: define the status test helper around captured fixture state; why: the custom readiness path accepts real http success without json scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def status(self): + # What: arrange the running field as true; why: RunningManager.status carries running into return {"running": True, "pid": 44}. + return {"running": True, "pid": 44} + + # What: act by calling ThreadingHTTPServer and capture server; why: the custom readiness path accepts real http success without json test asserts the response, state, or failure produced by this call. + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + # What: act by calling threading.Thread and capture worker; why: the custom readiness path accepts real http success without json test asserts the response, state, or failure produced by this call. + worker = threading.Thread(target=server.serve_forever, daemon=True) + # What: act by calling worker.start with the declared inputs; why: the custom readiness path accepts real http success without json scenario observes the worker.start return value during try. + worker.start() + # What: establish the handler boundary for the protected operation; why: test_custom_readiness_path_accepts_real_http_success_without_json routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: act by calling wait_for_ready and capture result; why: the custom readiness path accepts real http success without json test asserts the response, state, or failure produced by this call. + result = wait_for_ready( + # What: act by calling RunningManager with the declared inputs; why: the custom readiness path accepts real http success without json scenario observes the RunningManager return value during serve probe. + RunningManager(), + # What: act by calling ServeProbe with the declared inputs; why: the custom readiness path accepts real http success without json scenario observes the ServeProbe return value during pid. + ServeProbe(), + # What: arrange pid to wait_for_ready; why: the custom readiness path accepts real http success without json scenario binds this 44 value to wait_for_ready's pid input. + pid=44, + # What: arrange port to wait_for_ready; why: the custom readiness path accepts real http success without json scenario binds this server port and server value to wait_for_ready's port input. + port=server.server_port, + # What: arrange timeout s to wait_for_ready; why: the custom readiness path accepts real http success without json scenario binds this 1 value to wait_for_ready's timeout s input. + timeout_s=1, + # What: arrange path to wait_for_ready; why: the custom readiness path accepts real http success without json scenario binds this ready value to wait_for_ready's path input. + path="/ready", + # What: arrange the wait_for_ready call with pid and port and timeout s and path; why: test_custom_readiness_path_accepts_real_http_success_without_json groups the supplied clauses as one wait_for_ready call before its value is consumed. + ) + # What: run server shutdown on every exit path; why: test_custom_readiness_path_accepts_real_http_success_without_json performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: act by calling server.shutdown with the declared inputs; why: the custom readiness path accepts real http success without json scenario observes the server.shutdown return value during server server close. + server.shutdown() + # What: act by calling server.server_close with the declared inputs; why: the custom readiness path accepts real http success without json scenario observes the server.server_close return value during worker join. + server.server_close() + # What: act by calling worker.join with 2; why: the custom readiness path accepts real http success without json scenario observes the worker.join return value during assert result ready health reachable. + worker.join(2) + + # What: assert that result equals ready true health reachable true; why: this assertion protects the custom readiness path accepts real http success without json regression after the test's arranged inputs and exercised call. + assert result == {"ready": True, "health": {"reachable": True}} + + +# What: define custom readiness HTTP-error retry coverage; why: a bound server that returns 503 while loading must not be admitted as inference-ready. +def test_custom_readiness_path_retries_reachable_http_error(): + # What: define a stable running manager; why: the readiness loop should remain focused on endpoint state rather than process replacement. + class RunningManager: + # What: report the expected owned process; why: readiness may continue only while the launched generation remains current. + def status(self): + # What: return the stable running identity; why: neither probe result should be classified as superseded. + return {"running": True, "pid": 44} + + # What: define a two-state custom probe; why: the regression requires one reachable 503 followed by a successful readiness response. + class Probe: + # What: initialize the ordered endpoint states; why: the test must prove the first HTTP error is retried rather than accepted. + def __init__(self): + # What: retain loading then ready results; why: each fresh probe consumes the next authoritative state. + self.results = [{"reachable": True, "status": "error", "httpStatus": 503}, {"reachable": True}] + + # What: return the next custom readiness state; why: wait_for_ready must evaluate HTTP semantics on every uncached probe. + def fresh_readiness(self, port, path): + # What: require the configured dynamic endpoint inputs; why: the helper must probe the exact owned port and catalog path. + assert (port, path) == (1922, "/ready") + # What: consume and return the next state; why: success is valid only after the initial 503 has been rejected. + return self.results.pop(0) + + # What: execute readiness with a no-delay retry; why: deterministic unit coverage does not need wall-clock sleeping between the two states. + result = wait_for_ready(RunningManager(), Probe(), pid=44, port=1922, timeout_s=1, path="/ready", sleep=lambda _: None) + # What: require eventual readiness after the successful probe; why: the fix must retry loading errors without rejecting a later healthy endpoint. + assert result == {"ready": True, "health": {"reachable": True}} + + +# What: parameterize test_upstream_connector_rejects_non_owned_or_unsafe_base_before_network with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test upstream connector rejects non owned or unsafe base before network. +@pytest.mark.parametrize("base_url", [ + # What: arrange the http portion of the enclosing predicate; why: this clause remains in the upstream connector rejects non owned or unsafe base before network scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "http://127.0.0.1:1923", + # What: arrange the http localhost portion of the enclosing predicate; why: this clause remains in the upstream connector rejects non owned or unsafe base before network scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "http://localhost:1922", + # What: arrange the http admin portion of the enclosing predicate; why: this clause remains in the upstream connector rejects non owned or unsafe base before network scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "http://127.0.0.1:1922/../admin", + # What: arrange the http api token x portion of the enclosing predicate; why: this clause remains in the upstream connector rejects non owned or unsafe base before network scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "http://127.0.0.1:1922/api?token=x", +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_upstream_connector_rejects_non_owned_or_unsafe_base_before_network groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_upstream_connector_rejects_non_owned_or_unsafe_base_before_network test around base url; why: this test groups the arrange, act, and assertions that protect the upstream connector rejects non owned or unsafe base before network outcome. +def test_upstream_connector_rejects_non_owned_or_unsafe_base_before_network(base_url): + # What: assert the pytest.raises failure context; why: the upstream connector rejects non owned or unsafe base before network scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(ValueError, match="manager-owned loopback port"): + # What: act by calling open_upstream with the declared inputs; why: the upstream connector rejects non owned or unsafe base before network scenario observes the open_upstream return value during port. + open_upstream( + # What: arrange port to open_upstream; why: the upstream connector rejects non owned or unsafe base before network scenario binds this 1922 value to open_upstream's port input. + port=1922, + # What: arrange base url to open_upstream; why: the upstream connector rejects non owned or unsafe base before network scenario binds this base url value to open_upstream's base url input. + base_url=base_url, + # What: arrange the exact path and query v1 models fixture fragment; why: the upstream connector rejects non owned or unsafe base before network scenario feeds this byte-preserved fragment through path_and_query="/v1/models" before asserting its protocol or parser result. + path_and_query="/v1/models", + # What: arrange headers to open_upstream; why: the upstream connector rejects non owned or unsafe base before network scenario binds this the named fixture input value to open_upstream's headers input. + headers={}, + # What: arrange body to open_upstream; why: the upstream connector rejects non owned or unsafe base before network scenario binds this the named fixture input value to open_upstream's body input. + body=b"", + # What: arrange the exact method get fixture fragment; why: the upstream connector rejects non owned or unsafe base before network scenario feeds this byte-preserved fragment through method="GET" before asserting its protocol or parser result. + method="GET", + # What: arrange the open_upstream call with port and base url and path and query and headers and body; why: test_upstream_connector_rejects_non_owned_or_unsafe_base_before_network groups the supplied clauses as one open_upstream call before its value is consumed. + ) + + +# What: define the test_stateless_response_resource_routes_preserve_engine_error_without_activation test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the stateless response resource routes preserve engine error without activation outcome. +def test_stateless_response_resource_routes_preserve_engine_error_without_activation(monkeypatch): + # What: act by calling Manager and capture manager; why: the stateless response resource routes preserve engine error without activation test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the stateless response resource routes preserve engine error without activation test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf; why: test_stateless_response_resource_routes_preserve_engine_error_without_activation carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": ModelProfile("low", "low.gguf", ())}, + # What: arrange settings to RouterSettings; why: the stateless response resource routes preserve engine error without activation scenario binds this router settings and router test key value to RouterSettings's settings input. + settings=RouterSettings(api_keys=("router-test-key",)), + # What: arrange the ModelCatalog call with settings; why: test_stateless_response_resource_routes_preserve_engine_error_without_activation groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the stateless response resource routes preserve engine error without activation test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange monkeypatch setattr for the scenario; why: test stateless response resource routes preserve engine error without activation requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the stateless response resource routes preserve engine error without activation scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the exact lambda kwargs pytest fail stateless response lookup fixture fragment; why: the stateless response resource routes preserve engine error without activation scenario feeds this byte-preserved fragment through lambda **kwargs: pytest.fail("stateless response lookup reached upstream befor. + lambda **kwargs: pytest.fail("stateless response lookup reached upstream"), + # What: arrange the monkeypatch.setattr call with fail; why: test_stateless_response_resource_routes_preserve_engine_error_without_activation groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_stateless_response_resource_routes_preserve_engine_error_without_activation releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the stateless response resource routes preserve engine error without activation test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_stateless_response_resource_routes_preserve_engine_error_without_activation; why: test_stateless_response_resource_routes_preserve_engine_error_without_activation consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the stateless response resource routes preserve engine error without activation scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_stateless_response_resource_routes_preserve_engine_error_without_activation groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the stateless response resource routes preserve engine error without activation test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: assert that client get v1 responses resp abc status code equals 401; why: this assertion protects the stateless response resource routes preserve engine error without activation regression after the test's arranged inputs and exercised call. + assert client.get("/v1/responses/resp_abc").status_code == 401 + # What: assert that client post v1 responses resp abc cancel status code equals 401; why: this assertion protects the stateless response resource routes preserve engine error without activation regression after the test's arranged inputs and exercised call. + assert client.post("/v1/responses/resp_abc/cancel").status_code == 401 + # What: arrange headers as authorization and bearer and router test key; why: the stateless response resource routes preserve engine error without activation test consumes this named precondition before exercising the behavior. + headers = {"Authorization": "Bearer router-test-key"} + # What: act by calling client.get and capture lookup; why: the stateless response resource routes preserve engine error without activation test asserts the response, state, or failure produced by this call. + lookup = client.get("/v1/responses/resp_abc", headers=headers) + # What: act by calling client.post and capture cancel; why: the stateless response resource routes preserve engine error without activation test asserts the response, state, or failure produced by this call. + cancel = client.post("/v1/responses/resp_abc/cancel", headers=headers) + + # What: arrange expected as error and message and type and code and response; why: the stateless response resource routes preserve engine error without activation test consumes this named precondition before exercising the behavior. + expected = { + # What: arrange the error portion of expected; why: the stateless response resource routes preserve engine error without activation scenario uses this clause to evaluate expected as one grouped value. + "error": { + # What: arrange the message field as response and resp abc and not and found; why: test_stateless_response_resource_routes_preserve_engine_error_without_activation carries message through expected into assert lookup json equals cancel json equals expected. + "message": "response 'resp_abc' not found (stateless server)", + # What: arrange the type field as invalid request error; why: test_stateless_response_resource_routes_preserve_engine_error_without_activation carries type through expected into assert lookup json equals cancel json equals expected. + "type": "invalid_request_error", + # What: arrange the code field as the fixture input; why: test_stateless_response_resource_routes_preserve_engine_error_without_activation carries code through expected into assert lookup json equals cancel json equals expected. + "code": None, + # What: arrange the expected mapping with message and type and code; why: test_stateless_response_resource_routes_preserve_engine_error_without_activation groups the supplied clauses as one expected mapping before its value is consumed. + } + # What: arrange the expected mapping with error; why: test_stateless_response_resource_routes_preserve_engine_error_without_activation groups the supplied clauses as one expected mapping before its value is consumed. + } + # What: assert that lookup status code equals cancel status code equals 404; why: this assertion protects the stateless response resource routes preserve engine error without activation regression after the test's arranged inputs and exercised call. + assert lookup.status_code == cancel.status_code == 404 + # What: assert that lookup json equals cancel json equals expected; why: this assertion protects the stateless response resource routes preserve engine error without activation regression after the test's arranged inputs and exercised call. + assert lookup.json() == cancel.json() == expected + # What: assert that manager calls equals group delimiter; why: this assertion protects the stateless response resource routes preserve engine error without activation regression after the test's arranged inputs and exercised call. + assert manager.calls == [] + # What: assert that router status admissions equals 0; why: this assertion protects the stateless response resource routes preserve engine error without activation regression after the test's arranged inputs and exercised call. + assert router.status()["admissions"] == 0 + # What: assert that router status reserved requests equals 0; why: this assertion protects the stateless response resource routes preserve engine error without activation regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 0 + + +# What: define the test_namespaced_upstream_uses_longest_model_prefix_and_preserves_escaped_suffix test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the namespaced upstream uses longest model prefix and preserves escaped suffix outcome. +def test_namespaced_upstream_uses_longest_model_prefix_and_preserves_escaped_suffix(monkeypatch): + # What: act by calling Manager and capture manager; why: the namespaced upstream uses longest model prefix and preserves escaped suffix test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the namespaced upstream uses longest model prefix and preserves escaped suffix test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({ + # What: arrange the author field as model profile and author and parent and gguf; why: test_namespaced_upstream_uses_longest_model_prefix_and_preserves_escaped_suffix carries author through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "author": ModelProfile("author", "parent.gguf", ()), + # What: arrange the author model field as model profile and author and model and exact and gguf; why: test_namespaced_upstream_uses_longest_model_prefix_and_preserves_escaped_suffix carries author model through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "author/model": ModelProfile( + # What: arrange aliases to ModelProfile; why: the namespaced upstream uses longest model prefix and preserves escaped suffix scenario binds this org and compat value to ModelProfile's aliases input. + "author/model", "exact.gguf", (), aliases=("org/compat",) + # What: arrange the ModelProfile call with aliases; why: test_namespaced_upstream_uses_longest_model_prefix_and_preserves_escaped_suffix groups the supplied clauses as one ModelProfile call before its value is consumed. + ), + # What: arrange the ModelCatalog call with model profile; why: test_namespaced_upstream_uses_longest_model_prefix_and_preserves_escaped_suffix groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + # What: act by calling RoutingCoordinator and capture router; why: the namespaced upstream uses longest model prefix and preserves escaped suffix test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange calls as the fixture input; why: the namespaced upstream uses longest model prefix and preserves escaped suffix test consumes this named precondition before exercising the behavior. + calls = [] + + # What: define the upstream test helper around captured fixture state; why: the namespaced upstream uses longest model prefix and preserves escaped suffix scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: act by calling calls.append with kwargs; why: the namespaced upstream uses longest model prefix and preserves escaped suffix scenario observes the calls.append return value during return upstream response content type application json bytes io. + calls.append(kwargs) + # What: arrange the helper response as UpstreamResponse 200 Content Type application json BytesIO b; why: test router test feeds this result into the. + return UpstreamResponse(200, {"Content-Type": "application/json"}, BytesIO(b'{}')) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the namespaced upstream uses longest model prefix and preserves escaped suffix scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) bef. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_namespaced_upstream_uses_longest_model_prefix_and_preserves_escaped_suffix releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the namespaced upstream uses longest model prefix and preserves escaped suffix test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_namespaced_upstream_uses_longest_model_prefix_and_preserves_escaped_suffix; why: test_namespaced_upstream_uses_longest_model_prefix_and_preserves_escaped_suffix consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the namespaced upstream uses longest model prefix and preserves escaped suffix scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_namespaced_upstream_uses_longest_model_prefix_and_preserves_escaped_suffix groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the namespaced upstream uses longest model prefix and preserves escaped suffix test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling client.post and capture exact; why: the namespaced upstream uses longest model prefix and preserves escaped suffix test asserts the response, state, or failure produced by this call. + exact = client.post( + # What: arrange content to client.post; why: the namespaced upstream uses longest model prefix and preserves escaped suffix scenario binds this the named fixture input value to client.post's content input. + "/upstream/author/model/api/x%2Fy?preview=a%2Fb", content=b"exact" + # What: arrange the client.post call with content; why: test_namespaced_upstream_uses_longest_model_prefix_and_preserves_escaped_suffix groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling client.get and capture encoded alias; why: the namespaced upstream uses longest model prefix and preserves escaped suffix test asserts the response, state, or failure produced by this call. + encoded_alias = client.get("/upstream/org%2Fcompat/v1/chat") + # What: act by calling client.post and capture automatic; why: the namespaced upstream uses longest model prefix and preserves escaped suffix test asserts the response, state, or failure produced by this call. + automatic = client.post("/v1/chat/completions", json={"model": "org/compat"}) + # What: act by calling client.get and capture bare; why: the namespaced upstream uses longest model prefix and preserves escaped suffix test asserts the response, state, or failure produced by this call. + bare = client.get("/upstream/org/compat") + # What: act by calling client.post and capture blocked; why: the namespaced upstream uses longest model prefix and preserves escaped suffix test asserts the response, state, or failure produced by this call. + blocked = client.post("/upstream/author/model/v1/admin/prepare-stop") + # What: act by calling client.get and capture unknown; why: the namespaced upstream uses longest model prefix and preserves escaped suffix test asserts the response, state, or failure produced by this call. + unknown = client.get("/upstream/missing/model/v1/chat") + + # What: assert that exact status code equals encoded alias status code equals automatic status code equals 200; why: this assertion protects the namespaced upstream uses longest model prefix and preserves escaped suffix regression after the test's arranged inputs and exercised call. + assert exact.status_code == encoded_alias.status_code == automatic.status_code == 200 + # What: assert that bare status code equals 200; why: this assertion protects the namespaced upstream uses longest model prefix and preserves escaped suffix regression after the test's arranged inputs and exercised call. + assert bare.status_code == 200 + # What: assert that blocked status code equals 403; why: this assertion protects the namespaced upstream uses longest model prefix and preserves escaped suffix regression after the test's arranged inputs and exercised call. + assert blocked.status_code == 403 + # What: assert that unknown status code equals 404; why: this assertion protects the namespaced upstream uses longest model prefix and preserves escaped suffix regression after the test's arranged inputs and exercised call. + assert unknown.status_code == 404 + # What: assert that unknown json error type equals unknown model; why: this assertion protects the namespaced upstream uses longest model prefix and preserves escaped suffix regression after the test's arranged inputs and exercised call. + assert unknown.json()["error"]["type"] == "unknown_model" + # What: assert that manager calls equals start exact gguf; why: this assertion protects the namespaced upstream uses longest model prefix and preserves escaped suffix regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "exact.gguf")] + # What: assert the expected call path and query for call in calls == outcome; why: test router test namespaced upstream uses longest model prefix and preserves escaped suffix protects its regression by requiring this observable result after the exercised behavior. + assert [call["path_and_query"] for call in calls] == [ + # What: arrange api x 2 Fy preview a 2 Fb v1 chat v1 chat completions for the scenario; why: test router test namespaced upstream uses longest model prefix and preserves escaped suffix requires this concrete input or helper state before exercising the behavior under test. + "/api/x%2Fy?preview=a%2Fb", "/v1/chat", "/v1/chat/completions", "/", + # What: arrange the grouped source fragment for the scenario; why: test router test namespaced upstream uses longest model prefix and preserves escaped suffix requires this concrete input or helper state before exercising the behavior under test. + ] + # What: assert that calls 0 body equals b exact; why: this assertion protects the namespaced upstream uses longest model prefix and preserves escaped suffix regression after the test's arranged inputs and exercised call. + assert calls[0]["body"] == b"exact" + + +# What: define the test_upstream_static_suffix_refuses_cold_activation_and_allows_exact_resident test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the upstream static suffix refuses cold activation and allows exact resident outcome. +def test_upstream_static_suffix_refuses_cold_activation_and_allows_exact_resident(monkeypatch): + # What: act by calling Manager and capture manager; why: the upstream static suffix refuses cold activation and allows exact resident test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the upstream static suffix refuses cold activation and allows exact resident test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({ + # What: arrange the author model field as model profile and author and model and exact and gguf; why: test_upstream_static_suffix_refuses_cold_activation_and_allows_exact_resident carries author model through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "author/model": ModelProfile( + # What: arrange aliases to ModelProfile; why: the upstream static suffix refuses cold activation and allows exact resident scenario binds this org and compat value to ModelProfile's aliases input. + "author/model", "exact.gguf", (), aliases=("org/compat",) + # What: arrange the ModelProfile call with aliases; why: test_upstream_static_suffix_refuses_cold_activation_and_allows_exact_resident groups the supplied clauses as one ModelProfile call before its value is consumed. + ), + # What: arrange the ModelCatalog call with model profile; why: test_upstream_static_suffix_refuses_cold_activation_and_allows_exact_resident groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + # What: act by calling RoutingCoordinator and capture router; why: the upstream static suffix refuses cold activation and allows exact resident test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange calls as the fixture input; why: the upstream static suffix refuses cold activation and allows exact resident test consumes this named precondition before exercising the behavior. + calls = [] + + # What: define the upstream test helper around captured fixture state; why: the upstream static suffix refuses cold activation and allows exact resident scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: act by calling calls.append with kwargs; why: the upstream static suffix refuses cold activation and allows exact resident scenario observes the calls.append return value during return upstream response content type text plain bytes io. + calls.append(kwargs) + # What: arrange the content type field as text and plain; why: upstream carries content type into return UpstreamResponse(200, {"Content-Type": "text/plain"}, BytesIO(b"a. + return UpstreamResponse(200, {"Content-Type": "text/plain"}, BytesIO(b"asset")) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the upstream static suffix refuses cold activation and allows exact resident scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) befor. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_upstream_static_suffix_refuses_cold_activation_and_allows_exact_resident releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the upstream static suffix refuses cold activation and allows exact resident test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_upstream_static_suffix_refuses_cold_activation_and_allows_exact_resident; why: test_upstream_static_suffix_refuses_cold_activation_and_allows_exact_resident consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the upstream static suffix refuses cold activation and allows exact resident scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_upstream_static_suffix_refuses_cold_activation_and_allows_exact_resident groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the upstream static suffix refuses cold activation and allows exact resident test asserts the response, state, or failure produced by this call. + client = TestClient(app) + + # What: act by calling client.get and capture cold asset; why: the upstream static suffix refuses cold activation and allows exact resident test asserts the response, state, or failure produced by this call. + cold_asset = client.get("/upstream/org/compat/ui/app.js") + # What: assert that cold asset status code equals 409; why: this assertion protects the upstream static suffix refuses cold activation and allows exact resident regression after the test's arranged inputs and exercised call. + assert cold_asset.status_code == 409 + # What: assert that cold asset json error type equals model not loaded; why: this assertion protects the upstream static suffix refuses cold activation and allows exact resident regression after the test's arranged inputs and exercised call. + assert cold_asset.json()["error"]["type"] == "model_not_loaded" + # What: assert that manager calls equals group delimiter; why: this assertion protects the upstream static suffix refuses cold activation and allows exact resident regression after the test's arranged inputs and exercised call. + assert manager.calls == [] + # What: assert that calls equals group delimiter; why: this assertion protects the upstream static suffix refuses cold activation and allows exact resident regression after the test's arranged inputs and exercised call. + assert calls == [] + # What: assert that router status reserved requests equals 0; why: this assertion protects the upstream static suffix refuses cold activation and allows exact resident regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 0 + + # What: act by calling client.get and capture cold api; why: the upstream static suffix refuses cold activation and allows exact resident test asserts the response, state, or failure produced by this call. + cold_api = client.get("/upstream/org/compat/api/status") + # What: assert that cold api status code equals 200; why: this assertion protects the upstream static suffix refuses cold activation and allows exact resident regression after the test's arranged inputs and exercised call. + assert cold_api.status_code == 200 + # What: assert that manager calls equals start exact gguf; why: this assertion protects the upstream static suffix refuses cold activation and allows exact resident regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "exact.gguf")] + + # What: act by calling client.get and capture warm asset; why: the upstream static suffix refuses cold activation and allows exact resident test asserts the response, state, or failure produced by this call. + warm_asset = client.get("/upstream/org/compat/ui/app.js") + # What: assert that warm asset status code equals 200; why: this assertion protects the upstream static suffix refuses cold activation and allows exact resident regression after the test's arranged inputs and exercised call. + assert warm_asset.status_code == 200 + # What: assert that warm asset content equals b asset; why: this assertion protects the upstream static suffix refuses cold activation and allows exact resident regression after the test's arranged inputs and exercised call. + assert warm_asset.content == b"asset" + + # What: assert that call path and query for call in calls equals api status ui app js; why: this assertion protects the upstream static suffix refuses cold activation and allows exact resident regression after the test's arranged inputs and exercised call. + assert [call["path_and_query"] for call in calls] == ["/api/status", "/ui/app.js"] + # What: assert that manager calls equals start exact gguf; why: this assertion protects the upstream static suffix refuses cold activation and allows exact resident regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "exact.gguf")] + + +# What: define the test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable test around monkeypatch and tmp path; why: this test groups the arrange, act, and assertions that protect the activity and opt in capture apis are authenticated redacted and durable outcome. +def test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable( + # What: arrange monkeypatch tmp path for the scenario; why: test activity and opt in capture apis are authenticated redacted and durable requires this concrete input or helper state before exercising the behavior under test. + monkeypatch, tmp_path +# What: arrange the grouped source fragment for the scenario; why: test activity and opt in capture apis are authenticated redacted and durable requires this concrete input or helper state before exercising the behavior under test. +): + # What: act by calling Manager and capture manager; why: the activity and opt in capture apis are authenticated redacted and durable test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the activity and opt in capture apis are authenticated redacted and durable test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": ModelProfile("low", "low.gguf", ())}, + # What: arrange api keys to RouterSettings; why: the activity and opt in capture apis are authenticated redacted and durable scenario binds this secret value to RouterSettings's api keys input. + RouterSettings(api_keys=("secret",), activity_max_entries=2, capture_buffer_mb=1), + # What: arrange the ModelCatalog call with model profile and router settings; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the activity and opt in capture apis are authenticated redacted and durable test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange activity path as tmp path and activity and jsonl; why: the activity and opt in capture apis are authenticated redacted and durable test consumes this named precondition before exercising the behavior. + activity_path = tmp_path / "activity.jsonl" + + # What: define the upstream test helper around captured fixture state; why: the activity and opt in capture apis are authenticated redacted and durable scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: return upstream response and bytes io and 200 and content type and set cookie from the upstream test helper; why: the activity and opt in capture apis are authenticated redacted and durable scenario uses this helper result in its subsequent act or assertion. + return UpstreamResponse( + # What: arrange the grouped expression portion of the enclosing predicate; why: this clause remains in the activity and opt in capture apis are authenticated redacted and durable scenario\'s enclosing expression so its grouping and evaluation order stay intact. + 200, + # What: arrange the content type field as application and octet stream; why: upstream carries content type into {"Content-Type": "application/octet-stream", "Set-Cookie": "private"}. + {"Content-Type": "application/octet-stream", "Set-Cookie": "private"}, + # What: act by calling BytesIO with the named fixture input; why: the activity and opt in capture apis are authenticated redacted and durable scenario observes the BytesIO return value while evaluating BytesIO(b"\xffresult"). + BytesIO(b"\xffresult"), + # What: arrange the grouped source fragment for the scenario; why: test router test activity and opt in capture apis are authenticated redacted and durable requires this concrete input or helper state before exercising the behavior under test. + ) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the activity and opt in capture apis are authenticated redacted and durable scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) before. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the activity and opt in capture apis are authenticated redacted and durable test asserts the response, state, or failure produced by this call. + app = build_app( + # What: act by evaluating manager manager ring LogRing probe object footprint fn lambda pid; why: test router test activity and opt in capture apis are authenticated redacted and durable captures the behavior or response that its following assertions inspect. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the activity and opt in capture apis are authenticated redacted and durable scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange activity path to str; why: the activity and opt in capture apis are authenticated redacted and durable scenario binds this str and activity path value to str's activity path input. + activity_path=str(activity_path), + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the activity and opt in capture apis are authenticated redacted and durable test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: assert that client get router activity status code equals 401; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert client.get("/router/activity").status_code == 401 + # What: arrange headers as authorization and x trace and x session id and bearer and secret; why: the activity and opt in capture apis are authenticated redacted and durable test consumes this named precondition before exercising the behavior. + headers = { + # What: arrange the authorization field as bearer and secret; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable carries authorization through headers into headers headers content type application json. + "Authorization": "Bearer secret", "X-Trace": "visible", + # What: arrange the x session id field as private session value; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable carries x session id through headers into headers headers content type application json. + "X-Session-ID": "private-session-value", + # What: arrange the headers mapping with authorization and x trace and x session id; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable groups the supplied clauses as one headers mapping before its value is consumed. + } + # What: act by calling client.post and capture response; why: the activity and opt in capture apis are authenticated redacted and durable test asserts the response, state, or failure produced by this call. + response = client.post( + # What: arrange content to client.post; why: the activity and opt in capture apis are authenticated redacted and durable scenario binds this the named fixture input value to client.post's content input. + "/v1/chat/completions", content=b'{"model":"low","prompt":"private"}', + # What: arrange the content type field as application and json; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable carries content type through response into assert response content equals b xffresult. + headers={**headers, "Content-Type": "application/json"}, + # What: arrange the client.post call with content and headers; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: assert that response content equals b xffresult; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert response.content == b"\xffresult" + + # What: act by evaluating page client get router activity headers headers json; why: test router test activity and opt in capture apis are authenticated redacted and durable captures the behavior or response that its following assertions inspect. + page = client.get("/router/activity", headers=headers).json() + # What: assert that page count equals 1; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert page["count"] == 1 + # What: arrange row as page and 0 and data; why: the activity and opt in capture apis are authenticated redacted and durable test consumes this named precondition before exercising the behavior. + row = page["data"][0] + # What: assert that row model equals low; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert row["model"] == "low" + # What: assert that row route equals v1 chat completions; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert row["route"] == "/v1/chat/completions" + # What: assert that row has capture is true; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert row["hasCapture"] is True + # What: assert that len row session id equals 16; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert len(row["sessionId"]) == 16 + # What: assert that row session id differs from private session value; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert row["sessionId"] != "private-session-value" + # What: assert that client get router activity stats headers headers equals 1; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert client.get("/router/activity/stats", headers=headers).json()["count"] == 1 + # What: act by calling operation.json and capture capture; why: the activity and opt in capture apis are authenticated redacted and durable test asserts the response, state, or failure produced by this call. + capture = client.get(f'/router/captures/{row["id"]}', headers=headers).json() + # What: assert that capture request headers authorization equals redacted; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert capture["requestHeaders"]["authorization"] == "[REDACTED]" + # What: assert that capture response headers set cookie equals redacted; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert capture["responseHeaders"]["Set-Cookie"] == "[REDACTED]" + # What: assert that base64 b64decode capture request body base64 equals b model low prompt private; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert base64.b64decode(capture["requestBodyBase64"]) == b'{"model":"low","prompt":"private"}' + # What: assert that base64 b64decode capture response body base64 equals b xffresult; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert base64.b64decode(capture["responseBodyBase64"]) == b"\xffresult" + + # What: act by calling Manager and capture restarted manager; why: the activity and opt in capture apis are authenticated redacted and durable test asserts the response, state, or failure produced by this call. + restarted_manager = Manager() + # What: act by calling RoutingCoordinator and capture restarted router; why: the activity and opt in capture apis are authenticated redacted and durable test asserts the response, state, or failure produced by this call. + restarted_router = RoutingCoordinator( + # What: arrange ready fn to object; why: the activity and opt in capture apis are authenticated redacted and durable scenario binds this ready value to object's ready fn input. + restarted_manager, catalog_doc, object(), ready_fn=ready + # What: arrange the RoutingCoordinator call with ready fn; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable groups the supplied clauses as one RoutingCoordinator call before its value is consumed. + ) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before restarted app build app; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable releases this resource or lock after restarted app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture restarted app; why: the activity and opt in capture apis are authenticated redacted and durable test asserts the response, state, or failure produced by this call. + restarted_app = build_app( + # What: arrange manager to LogRing; why: the activity and opt in capture apis are authenticated redacted and durable scenario binds this restarted manager value to LogRing's manager input. + manager=restarted_manager, ring=LogRing(), probe=object(), + # What: arrange footprint fn lambda pid lifecycle pool lifecycle proxy pool proxy for the scenario; why: test router test activity and opt in capture apis are authenticated redacted and durable requires this concrete input or helper state before exercising the behavior under test. + footprint_fn=lambda pid: {}, lifecycle_pool=lifecycle, proxy_pool=proxy, + # What: arrange catalog to str; why: the activity and opt in capture apis are authenticated redacted and durable scenario binds this catalog doc value to str's catalog input. + catalog=catalog_doc, router=restarted_router, activity_path=str(activity_path), + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_activity_and_opt_in_capture_apis_are_authenticated_redacted_and_durable groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture restarted; why: the activity and opt in capture apis are authenticated redacted and durable test asserts the response, state, or failure produced by this call. + restarted = TestClient(restarted_app) + # What: act by evaluating page restarted get router activity headers headers json; why: test router test activity and opt in capture apis are authenticated redacted and durable captures the behavior or response that its following assertions inspect. + page = restarted.get("/router/activity", headers=headers).json() + # What: assert that page count equals 1; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert page["count"] == 1 + # What: assert that page data 0 has capture is false; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert page["data"][0]["hasCapture"] is False + # What: assert that page data 0 session id equals row session id; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert page["data"][0]["sessionId"] == row["sessionId"] + # What: assert that page persistence equals enabled true healthy true error; why: this assertion protects the activity and opt in capture apis are authenticated redacted and durable regression after the test's arranged inputs and exercised call. + assert page["persistence"] == {"enabled": True, "healthy": True, "error": None} + # What: assert the expected restarted get outcome; why: test router test activity and opt in capture apis are authenticated redacted and durable protects its regression by requiring this observable result after the exercised behavior. + assert restarted.get( + # What: arrange f router captures page data 0 id headers headers for the scenario; why: test router test activity and opt in capture apis are authenticated redacted and durable requires this concrete input or helper state before exercising the behavior under test. + f'/router/captures/{page["data"][0]["id"]}', headers=headers + # What: arrange status code == 404 for the scenario; why: test router test activity and opt in capture apis are authenticated redacted and durable requires this concrete input or helper state before exercising the behavior under test. + ).status_code == 404 + + +# What: parameterize test_all_routed_text_endpoints_share_stable_unknown_model_error with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test all routed text endpoints share stable unknown model error. +@pytest.mark.parametrize( + # What: arrange the path portion of the enclosing predicate; why: this clause remains in the all routed text endpoints share stable unknown model error scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "path", + # What: arrange the grouped source fragment for the scenario; why: test all routed text endpoints share stable unknown model error requires this concrete input or helper state before exercising the behavior under test. + ( + # What: arrange the v1 chat completions portion of the enclosing predicate; why: this clause remains in the all routed text endpoints share stable unknown model error scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "/v1/chat/completions", + # What: arrange the v1 completions portion of the enclosing predicate; why: this clause remains in the all routed text endpoints share stable unknown model error scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "/v1/completions", + # What: arrange the v1 responses portion of the enclosing predicate; why: this clause remains in the all routed text endpoints share stable unknown model error scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "/v1/responses", + # What: arrange the v1 messages portion of the enclosing predicate; why: this clause remains in the all routed text endpoints share stable unknown model error scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "/v1/messages", + # What: arrange the v1 messages count tokens portion of the enclosing predicate; why: this clause remains in the all routed text endpoints share stable unknown model error scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "/v1/messages/count_tokens", + # What: arrange the grouped source fragment for the scenario; why: test all routed text endpoints share stable unknown model error requires this concrete input or helper state before exercising the behavior under test. + ), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_all_routed_text_endpoints_share_stable_unknown_model_error groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +) +# What: define the test_all_routed_text_endpoints_share_stable_unknown_model_error test around path and monkeypatch; why: this test groups the arrange, act, and assertions that protect the all routed text endpoints share stable unknown model error outcome. +def test_all_routed_text_endpoints_share_stable_unknown_model_error(path, monkeypatch): + # What: act by calling Manager and capture manager; why: the all routed text endpoints share stable unknown model error test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the all routed text endpoints share stable unknown model error test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({"known": ModelProfile("known", "known.gguf", ())}) + # What: act by calling RoutingCoordinator and capture router; why: the all routed text endpoints share stable unknown model error test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange monkeypatch setattr for the scenario; why: test all routed text endpoints share stable unknown model error requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the all routed text endpoints share stable unknown model error scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the exact lambda kwargs value for value in fixture fragment; why: the all routed text endpoints share stable unknown model error scenario feeds this byte-preserved fragment through lambda **kwargs: (_ for _ in ()).throw(AssertionError("unknown model rea before asserting its protocol or parser r. + lambda **kwargs: (_ for _ in ()).throw(AssertionError("unknown model reached upstream")), + # What: arrange the monkeypatch.setattr call with throw; why: test_all_routed_text_endpoints_share_stable_unknown_model_error groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_all_routed_text_endpoints_share_stable_unknown_model_error releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the all routed text endpoints share stable unknown model error test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_all_routed_text_endpoints_share_stable_unknown_model_error; why: test_all_routed_text_endpoints_share_stable_unknown_model_error consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the all routed text endpoints share stable unknown model error scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_all_routed_text_endpoints_share_stable_unknown_model_error groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the all routed text endpoints share stable unknown model error test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling client.post and capture response; why: the all routed text endpoints share stable unknown model error test asserts the response, state, or failure produced by this call. + response = client.post( + # What: arrange the model field as missing; why: test_all_routed_text_endpoints_share_stable_unknown_model_error sends this field through response so the router selects the canonical model or alias for upstream dispatch. + path, json={"model": "missing"}, headers={"X-FT-Request-ID": "reusable-failure"} + # What: arrange the client.post call with json and headers; why: test_all_routed_text_endpoints_share_stable_unknown_model_error groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling client.post and capture repeated; why: the all routed text endpoints share stable unknown model error test asserts the response, state, or failure produced by this call. + repeated = client.post( + # What: arrange the model field as missing; why: test_all_routed_text_endpoints_share_stable_unknown_model_error sends this field through repeated so the router selects the canonical model or alias for upstream dispatch. + path, json={"model": "missing"}, headers={"X-FT-Request-ID": "reusable-failure"} + # What: arrange the client.post call with json and headers; why: test_all_routed_text_endpoints_share_stable_unknown_model_error groups the supplied clauses as one client.post call before its value is consumed. + ) + + # What: assert that response status code equals 404; why: this assertion protects the all routed text endpoints share stable unknown model error regression after the test's arranged inputs and exercised call. + assert response.status_code == 404 + # What: assert that response json error type equals unknown model; why: this assertion protects the all routed text endpoints share stable unknown model error regression after the test's arranged inputs and exercised call. + assert response.json()["error"]["type"] == "unknown_model" + # What: assert that missing is present in response json error message; why: this assertion protects the all routed text endpoints share stable unknown model error regression after the test's arranged inputs and exercised call. + assert "missing" in response.json()["error"]["message"] + # What: assert that repeated status code equals 404; why: this assertion protects the all routed text endpoints share stable unknown model error regression after the test's arranged inputs and exercised call. + assert repeated.status_code == 404 + # What: assert that repeated json error type equals unknown model; why: this assertion protects the all routed text endpoints share stable unknown model error regression after the test's arranged inputs and exercised call. + assert repeated.json()["error"]["type"] == "unknown_model" + # What: assert that manager calls equals group delimiter; why: this assertion protects the all routed text endpoints share stable unknown model error regression after the test's arranged inputs and exercised call. + assert manager.calls == [] + + +# What: define the test_router_preserves_upstream_error_status_headers_and_body test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the router preserves upstream error status headers and body outcome. +def test_router_preserves_upstream_error_status_headers_and_body(monkeypatch): + # What: act by calling Manager and capture manager; why: the router preserves upstream error status headers and body test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the router preserves upstream error status headers and body test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({"low": ModelProfile("low", "low.gguf", ())}) + # What: act by calling RoutingCoordinator and capture router; why: the router preserves upstream error status headers and body test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + + # What: define the upstream test helper around captured fixture state; why: the router preserves upstream error status headers and body scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: assert that kwargs path and query equals v1 responses; why: this assertion protects the router preserves upstream error status headers and body regression after the test's arranged inputs and exercised call. + assert kwargs["path_and_query"] == "/v1/responses" + # What: return upstream response and bytes io and 429 and content type and retry after from the upstream test helper; why: the router preserves upstream error status headers and body scenario uses this helper result in its subsequent act or assertion. + return UpstreamResponse( + # What: arrange status to UpstreamResponse; why: the router preserves upstream error status headers and body scenario binds this 429 value to UpstreamResponse's status input. + status=429, + # What: arrange headers Content Type application json Retry After 2 Content Length 999 for the scenario; why: test router preserves upstream error stat requires this concrete input or helper state before exercising the behavior under test. + headers={"Content-Type": "application/json", "Retry-After": "2", "Content-Length": "999"}, + # What: arrange raw to BytesIO; why: the router preserves upstream error status headers and body scenario binds this bytes io value to BytesIO's raw input. + raw=BytesIO(b'{"error":{"message":"busy"}}'), + # What: arrange the grouped source fragment for the scenario; why: test router test router preserves upstream error status headers and body requires this concrete input or helper state before exercising the behavior under test. + ) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the router preserves upstream error status headers and body scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) before asserting its p. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_router_preserves_upstream_error_status_headers_and_body releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the router preserves upstream error status headers and body test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_router_preserves_upstream_error_status_headers_and_body; why: test_router_preserves_upstream_error_status_headers_and_body consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the router preserves upstream error status headers and body scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_router_preserves_upstream_error_status_headers_and_body groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture response; why: the router preserves upstream error status headers and body test asserts the response, state, or failure produced by this call. + response = TestClient(app).post("/v1/responses", json={"model": "low", "input": "private"}) + + # What: assert that response status code equals 429; why: this assertion protects the router preserves upstream error status headers and body regression after the test's arranged inputs and exercised call. + assert response.status_code == 429 + # What: assert that response headers retry after equals 2; why: this assertion protects the router preserves upstream error status headers and body regression after the test's arranged inputs and exercised call. + assert response.headers["retry-after"] == "2" + # What: assert that response content equals b error message busy; why: this assertion protects the router preserves upstream error status headers and body regression after the test's arranged inputs and exercised call. + assert response.content == b'{"error":{"message":"busy"}}' + # What: assert that content length is absent from response headers; why: this assertion protects the router preserves upstream error status headers and body regression after the test's arranged inputs and exercised call. + assert "content-length" not in response.headers + # What: assert that router status active requests equals 0; why: this assertion protects the router preserves upstream error status headers and body regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + # What: assert that router status terminal streams equals 1; why: this assertion protects the router preserves upstream error status headers and body regression after the test's arranged inputs and exercised call. + assert router.status()["terminalStreams"] == 1 + + +# What: define the test_failed_upstream_connect_releases_lease_and_request_id_reservation test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the failed upstream connect releases lease and request id reservation outcome. +def test_failed_upstream_connect_releases_lease_and_request_id_reservation(monkeypatch): + # What: act by calling Manager and capture manager; why: the failed upstream connect releases lease and request id reservation test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the failed upstream connect releases lease and request id reservation test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({"low": ModelProfile("low", "low.gguf", ())}) + # What: act by calling RoutingCoordinator and capture router; why: the failed upstream connect releases lease and request id reservation test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange monkeypatch setattr for the scenario; why: test failed upstream connect releases lease and request id reservation requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the failed upstream connect releases lease and request id reservation scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the exact lambda kwargs value for value in fixture fragment; why: the failed upstream connect releases lease and request id reservation scenario feeds this byte-preserved fragment through lambda **kwargs: (_ for _ in ()).throw(OSError("fixture unavailable")) before asserting its protocol or par. + lambda **kwargs: (_ for _ in ()).throw(OSError("fixture unavailable")), + # What: arrange the monkeypatch.setattr call with throw; why: test_failed_upstream_connect_releases_lease_and_request_id_reservation groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_failed_upstream_connect_releases_lease_and_request_id_reservation releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the failed upstream connect releases lease and request id reservation test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_failed_upstream_connect_releases_lease_and_request_id_reservation; why: test_failed_upstream_connect_releases_lease_and_request_id_reservation consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the failed upstream connect releases lease and request id reservation scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_failed_upstream_connect_releases_lease_and_request_id_reservation groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the failed upstream connect releases lease and request id reservation test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling client.post and capture responses; why: the failed upstream connect releases lease and request id reservation test asserts the response, state, or failure produced by this call. + responses = [ + # What: act by calling client.post with v1 and chat and completions; why: the failed upstream connect releases lease and request id reservation scenario observes the client.post return value during v1 chat completions json model low. + client.post( + # What: arrange the model field as low; why: test_failed_upstream_connect_releases_lease_and_request_id_reservation sends this field through responses so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "low"}, + # What: arrange the x ft request id field as retry after connect failure; why: test_failed_upstream_connect_releases_lease_and_request_id_reservation carries x ft request id through responses into assert response status code for response in responses equals. + headers={"X-FT-Request-ID": "retry-after-connect-failure"}, + # What: arrange the client.post call with json and headers; why: test_failed_upstream_connect_releases_lease_and_request_id_reservation groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling range with 2; why: the failed upstream connect releases lease and request id reservation scenario observes the range return value while evaluating for _ in range(2). + for _ in range(2) + # What: arrange the responses expression with responses client post v1 chat completions json model low headers; why: test_failed_upstream_connect_releases_lease_and_request_id_reservation groups the supplied clauses as one responses expression before its value is consumed. + ] + + # What: assert that response status code for response in responses equals 502 502; why: this assertion protects the failed upstream connect releases lease and request id reservation regression after the test's arranged inputs and exercised call. + assert [response.status_code for response in responses] == [502, 502] + # What: assert that all response json error type equals upstream unavailable for response in responses; why: this assertion protects the failed upstream connect releases lease and request id reservation regression after the test's arranged inputs and exercised call. + assert all(response.json()["error"]["type"] == "upstream_unavailable" for response in responses) + # What: assert that router status active requests equals 0; why: this assertion protects the failed upstream connect releases lease and request id reservation regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + # What: assert that router status admissions equals 2; why: this assertion protects the failed upstream connect releases lease and request id reservation regression after the test's arranged inputs and exercised call. + assert router.status()["admissions"] == 2 + + +# What: define the test_alias_routes_to_canonical_residency_and_model_list_respects_visibility test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the alias routes to canonical residency and model list respects visibility outcome. +def test_alias_routes_to_canonical_residency_and_model_list_respects_visibility(monkeypatch): + # What: act by calling Manager and capture manager; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with canonical and hidden; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility groups the supplied clauses as one catalog_doc mapping before its value. + { + # What: arrange the canonical field as model profile and canonical and shared and gguf and compat id; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility carries canonical through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "canonical": ModelProfile( + # What: arrange aliases to ModelProfile; why: the alias routes to canonical residency and model list respects visibility scenario binds this compat id value to ModelProfile's aliases input. + "canonical", "shared.gguf", (), aliases=("compat-id",) + # What: arrange the ModelProfile call with aliases; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility groups the supplied clauses as one ModelProfile call before its value is consumed. + ), + # What: arrange the hidden field as model profile and hidden and hidden and gguf and true; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility carries hidden through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "hidden": ModelProfile( + # What: arrange aliases to ModelProfile; why: the alias routes to canonical residency and model list respects visibility scenario binds this private id value to ModelProfile's aliases input. + "hidden", "hidden.gguf", (), aliases=("private-id",), unlisted=True + # What: arrange the ModelProfile call with aliases and unlisted; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility groups the supplied clauses as one ModelProfile call before its value is consumed. + ), + # What: arrange the catalog_doc mapping with canonical and hidden; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility groups the supplied clauses as one catalog_doc mapping before its value. + }, + # What: arrange settings to RouterSettings; why: the alias routes to canonical residency and model list respects visibility scenario binds this router settings and true value to RouterSettings's settings input. + settings=RouterSettings(include_aliases_in_list=True), + # What: arrange the ModelCatalog call with settings; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange calls as the fixture input; why: the alias routes to canonical residency and model list respects visibility test consumes this named precondition before exercising the behavior. + calls = [] + + # What: define the upstream test helper around captured fixture state; why: the alias routes to canonical residency and model list respects visibility scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: act by calling calls.append with kwargs; why: the alias routes to canonical residency and model list respects visibility scenario observes the calls.append return value during return upstream response. + calls.append(kwargs) + # What: return upstream response and bytes io and 200 and content type and application from the upstream test helper; why: the alias routes to canonical residency and model list respects visibility scenario uses this helper result in its subsequent act or assertion. + return UpstreamResponse( + # What: arrange status to UpstreamResponse; why: the alias routes to canonical residency and model list respects visibility scenario binds this 200 value to UpstreamResponse's status input. + status=200, + # What: arrange headers Content Type application json for the scenario; why: test router test alias routes to canonical residency and model list respects visibility requires this concrete input or helper state before exercising the behavior under test. + headers={"Content-Type": "application/json"}, + # What: arrange raw to BytesIO; why: the alias routes to canonical residency and model list respects visibility scenario binds this bytes io value to BytesIO's raw input. + raw=BytesIO(b'{"ok":true}'), + # What: arrange the grouped source fragment for the scenario; why: test router test alias routes to canonical residency and model list respects visibility requires this concrete. + ) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the alias routes to canonical residency and model list respects visibility scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) before. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_alias_routes_to_canonical_residency_and_model_list_respects_visibility; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the alias routes to canonical residency and model list respects visibility scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling client.get and capture listed; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + listed = client.get("/v1/models") + # What: act by calling client.post and capture alias response; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + alias_response = client.post( + # What: arrange content to client.post; why: the alias routes to canonical residency and model list respects visibility scenario binds this the named fixture input value to client.post's content input. + "/v1/chat/completions", content=b'{"model":"compat-id","max_tokens":1}', + # What: arrange the content type field as application and json; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility carries content type through alias response into assert alias response status code equals canonical response status code equals 200. + headers={"Content-Type": "application/json"}, + # What: arrange the client.post call with content and headers; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling client.get and capture loaded; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + loaded = client.get("/v1/models") + # What: act by calling client.post and capture canonical response; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + canonical_response = client.post( + # What: arrange the model field as canonical; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility sends this field through canonical response so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "canonical", "max_tokens": 1} + # What: arrange the client.post call with json; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling client.post and capture hidden response; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + hidden_response = client.post( + # What: arrange the model field as private id; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility sends this field through hidden response so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "private-id", "max_tokens": 1} + # What: arrange the client.post call with json; why: test_alias_routes_to_canonical_residency_and_model_list_respects_visibility groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling client.post and capture unloaded; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + unloaded = client.post("/router/unload", json={"name": "private-id"}) + + # What: act by calling listed.json and capture listed data; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + listed_data = listed.json()["data"] + # What: assert that item id for item in listed data equals canonical compat id; why: this assertion protects the alias routes to canonical residency and model list respects visibility regression after the test's arranged inputs and exercised call. + assert [item["id"] for item in listed_data] == ["canonical", "compat-id"] + # What: assert that item status value for item in equals unloaded; why: this assertion protects the alias routes to canonical residency and model list respects visibility regression after the test's arranged inputs and exercised call. + assert {item["status"]["value"] for item in listed_data} == {"unloaded"} + # What: act by calling loaded.json and capture loaded data; why: the alias routes to canonical residency and model list respects visibility test asserts the response, state, or failure produced by this call. + loaded_data = loaded.json()["data"] + # What: assert the expected item id item status value for item in loaded data == outcome; why: test router test alias routes to canonical residency and model list respects visibility protects its regression by requiring this observable result after the exercised behavior. + assert {item["id"]: item["status"]["value"] for item in loaded_data} == { + # What: arrange canonical loaded compat id loaded for the scenario; why: test router test alias routes to canonical residency and model list respects visibility requires this concrete input or helper state before exercising the behavior under test. + "canonical": "loaded", "compat-id": "loaded", + # What: arrange the grouped source fragment for the scenario; why: test router test alias routes to canonical residency and model list respects visibility requires this concrete input or. + } + # What: assert that alias response status code equals canonical response status code equals 200; why: this assertion protects the alias routes to canonical residency and model list respects visibility regression after the test's arranged inputs and exercised call. + assert alias_response.status_code == canonical_response.status_code == 200 + # What: assert that hidden response status code equals 200; why: this assertion protects the alias routes to canonical residency and model list respects visibility regression after the test's arranged inputs and exercised call. + assert hidden_response.status_code == 200 + # What: assert that unloaded json unloaded is true; why: this assertion protects the alias routes to canonical residency and model list respects visibility regression after the test's arranged inputs and exercised call. + assert unloaded.json()["unloaded"] is True + # What: assert the expected manager calls == outcome; why: test router test alias routes to canonical residency and model list respects visibility protects its regression by requiring this observable result after the exercised behavior. + assert manager.calls == [ + # What: arrange start shared gguf for the scenario; why: test router test alias routes to canonical residency and model list respects visibility requires this concrete input or helper state before exercising the behavior under test. + ("start", "shared.gguf"), + # What: arrange switch hidden gguf for the scenario; why: test router test alias routes to canonical residency and model list respects visibility requires this concrete input or helper state before exercising the behavior under test. + ("switch", "hidden.gguf"), + # What: arrange stop 30.0 for the scenario; why: test router test alias routes to canonical residency and model list respects visibility requires this concrete input or helper state before exercising the behavior under test. + ("stop", 30.0), + # What: arrange the grouped source fragment for the scenario; why: test router test alias routes to canonical residency and model list respects visibility requires this concrete input. + ] + # What: assert that calls 0 body equals b model compat id max tokens 1; why: this assertion protects the alias routes to canonical residency and model list respects visibility regression after the test's arranged inputs and exercised call. + assert calls[0]["body"] == b'{"model":"compat-id","max_tokens":1}' + # What: assert that router status active profile is group delimiter; why: this assertion protects the alias routes to canonical residency and model list respects visibility regression after the test's arranged inputs and exercised call. + assert router.status()["activeProfile"] is None + + +# What: define the test_pin_and_warm_selectors_resolve_against_one_resident_slot test around local fixtures; why: this test groups the arrange, act, and assertions that protect the pin and warm selectors resolve against one resident slot outcome. +def test_pin_and_warm_selectors_resolve_against_one_resident_slot(): + # What: act by calling Manager and capture manager; why: the pin and warm selectors resolve against one resident slot test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the pin and warm selectors resolve against one resident slot test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with a and b; why: test_pin_and_warm_selectors_resolve_against_one_resident_slot groups the supplied clauses as one catalog_doc mapping before its value is consumed. + { + # What: arrange the a field as model profile and a and a and gguf; why: test_pin_and_warm_selectors_resolve_against_one_resident_slot carries a through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "a": ModelProfile("a", "a.gguf", ()), + # What: arrange the b field as model profile and b and b and gguf; why: test_pin_and_warm_selectors_resolve_against_one_resident_slot carries b through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "b": ModelProfile("b", "b.gguf", ()), + # What: arrange the catalog_doc mapping with a and b; why: test_pin_and_warm_selectors_resolve_against_one_resident_slot groups the supplied clauses as one catalog_doc mapping before its value. + }, + # What: arrange selectors to ModelCatalog; why: the pin and warm selectors resolve against one resident slot scenario binds this model selector and pinned and warm and pinned and pin value to ModelCatalog's selectors input. + selectors={ + # What: arrange the pinned field as model selector and pinned and pin and a and b; why: test_pin_and_warm_selectors_resolve_against_one_resident_slot carries pinned through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "pinned": ModelSelector("pinned", "pin", ("a", "b")), + # What: arrange the warm field as model selector and warm and warm and a and b; why: test_pin_and_warm_selectors_resolve_against_one_resident_slot carries warm through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "warm": ModelSelector("warm", "warm", ("a", "b")), + # What: arrange the catalog_doc mapping with pinned and warm; why: test_pin_and_warm_selectors_resolve_against_one_resident_slot groups the supplied clauses as one catalog_doc mapping before its value is consumed. + }, + # What: arrange the ModelCatalog call with selectors; why: test_pin_and_warm_selectors_resolve_against_one_resident_slot groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the pin and warm selectors resolve against one resident slot test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange the exact router acquire b release fixture fragment; why: the pin and warm selectors resolve against one resident slot scenario feeds this byte-preserved fragment through router.acquire("b").release() before asserting its protocol or parser result. + router.acquire("b").release() + + # What: act by calling router.acquire and capture warm; why: the pin and warm selectors resolve against one resident slot test asserts the response, state, or failure produced by this call. + warm = router.acquire("warm") + # What: assert that warm profile name warm model id warm selector id equals b b warm; why: this assertion protects the pin and warm selectors resolve against one resident slot regression after the test's arranged inputs and exercised call. + assert (warm.profile.name, warm.model_id, warm.selector_id) == ("b", "b", "warm") + # What: act by calling warm.release with the declared inputs; why: the pin and warm selectors resolve against one resident slot scenario observes the warm.release return value during pinned router acquire pinned. + warm.release() + + # What: act by calling router.acquire and capture pinned; why: the pin and warm selectors resolve against one resident slot test asserts the response, state, or failure produced by this call. + pinned = router.acquire("pinned") + # What: assert the expected pinned profile name pinned model id pinned selector id == outcome; why: test router test pin and warm selectors resolve against one resident slot protects its regression by requiring this observable result after the exercised behavior. + assert (pinned.profile.name, pinned.model_id, pinned.selector_id) == ( + # What: arrange a a pinned for the scenario; why: test router test pin and warm selectors resolve against one resident slot requires this concrete input or helper state before exercising the behavior under test. + "a", "a", "pinned", + # What: arrange the grouped source fragment for the scenario; why: test router test pin and warm selectors resolve against one resident slot requires this concrete input or helper state before exercising the behavior under test. + ) + # What: act by calling pinned.release with the declared inputs; why: the pin and warm selectors resolve against one resident slot scenario observes the pinned.release return value during assert manager calls start b gguf switch a gguf. + pinned.release() + # What: assert that manager calls equals start b gguf switch a gguf; why: this assertion protects the pin and warm selectors resolve against one resident slot regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "b.gguf"), ("switch", "a.gguf")] + + +# What: define the test_warm_selector_cold_fallback_uses_first_target test around local fixtures; why: this test groups the arrange, act, and assertions that protect the warm selector cold fallback uses first target outcome. +def test_warm_selector_cold_fallback_uses_first_target(): + # What: act by calling ModelCatalog and capture catalog doc; why: the warm selector cold fallback uses first target test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with a and b; why: test_warm_selector_cold_fallback_uses_first_target groups the supplied clauses as one catalog_doc mapping before its value is consumed. + { + # What: arrange the a field as model profile and a and a and gguf; why: test_warm_selector_cold_fallback_uses_first_target carries a through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "a": ModelProfile("a", "a.gguf", ()), + # What: arrange the b field as model profile and b and b and gguf; why: test_warm_selector_cold_fallback_uses_first_target carries b through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "b": ModelProfile("b", "b.gguf", ()), + # What: arrange the catalog_doc mapping with a and b; why: test_warm_selector_cold_fallback_uses_first_target groups the supplied clauses as one catalog_doc mapping before its value is consumed. + }, + # What: arrange the warm field as model selector and warm and warm and a and b; why: test_warm_selector_cold_fallback_uses_first_target carries warm through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + selectors={"warm": ModelSelector("warm", "warm", ("a", "b"))}, + # What: arrange the ModelCatalog call with selectors; why: test_warm_selector_cold_fallback_uses_first_target groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the warm selector cold fallback uses first target test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(Manager(), catalog_doc, object(), ready_fn=ready) + # What: act by calling router.acquire and capture lease; why: the warm selector cold fallback uses first target test asserts the response, state, or failure produced by this call. + lease = router.acquire("warm") + # What: assert that lease profile name lease model id equals a a; why: this assertion protects the warm selector cold fallback uses first target regression after the test's arranged inputs and exercised call. + assert (lease.profile.name, lease.model_id) == ("a", "a") + # What: act by calling lease.release with the declared inputs; why: the warm selector cold fallback uses first target scenario observes the lease.release return value during the enclosing return. + lease.release() + + +# What: parameterize test_selector_reservation_uses_atomically_resolved_target_loading_policy with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test selector reservation uses atomically resolved target loading policy. +@pytest.mark.parametrize("target_setting,global_setting,expected", [ + # What: arrange the grouped expression portion of the enclosing predicate; why: this clause remains in the selector reservation uses atomically resolved target loading policy scenario\'s enclosing expression so its grouping and evaluation order stay intact. + (False, True, False), + # What: arrange the grouped expression portion of the enclosing predicate; why: this clause remains in the selector reservation uses atomically resolved target loading policy scenario\'s enclosing expression so its grouping and evaluation order stay intact. + (True, False, True), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_selector_reservation_uses_atomically_resolved_target_loading_policy groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_selector_reservation_uses_atomically_resolved_target_loading_policy test around target setting and global setting and expected; why: this test groups the arrange, act, and assertions that protect the selector reservation uses atomically resolved target loading policy outcome. +def test_selector_reservation_uses_atomically_resolved_target_loading_policy( + # What: arrange target setting global setting expected for the scenario; why: test selector reservation uses atomically resolved target loading policy requires this concrete input or helper state before exercising the behavior under test. + target_setting, global_setting, expected +# What: arrange the grouped source fragment for the scenario; why: test selector reservation uses atomically resolved target loading policy requires this concrete input or helper state before exercising the behavior under test. +): + # What: act by calling ModelCatalog and capture catalog doc; why: the selector reservation uses atomically resolved target loading policy test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the a field as model profile and target setting and a and a and gguf; why: test_selector_reservation_uses_atomically_resolved_target_loading_policy carries a through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"a": ModelProfile("a", "a.gguf", (), send_loading_state=target_setting)}, + # What: arrange settings to RouterSettings; why: the selector reservation uses atomically resolved target loading policy scenario binds this router settings and global setting value to RouterSettings's settings input. + settings=RouterSettings(send_loading_state=global_setting), + # What: arrange the public field as model selector and public and pin and a; why: test_selector_reservation_uses_atomically_resolved_target_loading_policy carries public through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + selectors={"public": ModelSelector("public", "pin", ("a",))}, + # What: arrange the ModelCatalog call with settings and selectors; why: test_selector_reservation_uses_atomically_resolved_target_loading_policy groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the selector reservation uses atomically resolved target loading policy test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(Manager(), catalog_doc, object(), ready_fn=ready) + # What: arrange reserved as the fixture input; why: the selector reservation uses atomically resolved target loading policy test consumes this named precondition before exercising the behavior. + reserved = [] + + # What: act by calling router.acquire and capture lease; why: the selector reservation uses atomically resolved target loading policy test asserts the response, state, or failure produced by this call. + lease = router.acquire( + # What: arrange the public portion of lease; why: the selector reservation uses atomically resolved target loading policy scenario uses this clause to evaluate lease as one grouped value. + "public", + # What: arrange the loading input for test_selector_reservation_uses_atomically_resolved_target_loading_policy; why: test_selector_reservation_uses_atomically_resolved_target_loading_policy consumes loading during signature binding, so callers must bind it with the other signature inputs. + on_reserved=lambda loading, position: reserved.append((loading, position)), + # What: arrange apply loading policy to router.acquire; why: the selector reservation uses atomically resolved target loading policy scenario binds this true value to router.acquire's apply loading policy input. + apply_loading_policy=True, + # What: arrange the router.acquire call with on reserved and apply loading policy; why: test_selector_reservation_uses_atomically_resolved_target_loading_policy groups the supplied clauses as one router.acquire call before its value is consumed. + ) + # What: act by calling lease.release with the declared inputs; why: the selector reservation uses atomically resolved target loading policy scenario observes the lease.release return value during assert reserved expected. + lease.release() + + # What: assert that reserved equals expected 1; why: this assertion protects the selector reservation uses atomically resolved target loading policy regression after the test's arranged inputs and exercised call. + assert reserved == [(expected, 1)] + + +# What: define the test_warm_selector_joins_the_first_starting_target test around local fixtures; why: this test groups the arrange, act, and assertions that protect the warm selector joins the first starting target outcome. +def test_warm_selector_joins_the_first_starting_target(): + # What: act by calling Manager and capture manager; why: the warm selector joins the first starting target test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling threading.Event and capture activation started; why: the warm selector joins the first starting target test asserts the response, state, or failure produced by this call. + activation_started = threading.Event() + # What: act by calling threading.Event and capture finish activation; why: the warm selector joins the first starting target test asserts the response, state, or failure produced by this call. + finish_activation = threading.Event() + # What: act by calling ModelCatalog and capture catalog doc; why: the warm selector joins the first starting target test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with a and b; why: test_warm_selector_joins_the_first_starting_target groups the supplied clauses as one catalog_doc mapping before its value is consumed. + { + # What: arrange the a field as model profile and a and a and gguf; why: test_warm_selector_joins_the_first_starting_target carries a through catalog doc into router routing coordinator manager catalog doc object ready fn blocking ready. + "a": ModelProfile("a", "a.gguf", ()), + # What: arrange the b field as model profile and b and b and gguf; why: test_warm_selector_joins_the_first_starting_target carries b through catalog doc into router routing coordinator manager catalog doc object ready fn blocking ready. + "b": ModelProfile("b", "b.gguf", ()), + # What: arrange the catalog_doc mapping with a and b; why: test_warm_selector_joins_the_first_starting_target groups the supplied clauses as one catalog_doc mapping before its value is consumed. + }, + # What: arrange the warm field as model selector and warm and warm and a and b; why: test_warm_selector_joins_the_first_starting_target carries warm through catalog doc into router routing coordinator manager catalog doc object ready fn blocking ready. + selectors={"warm": ModelSelector("warm", "warm", ("a", "b"))}, + # What: arrange the ModelCatalog call with selectors; why: test_warm_selector_joins_the_first_starting_target groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + + # What: define the blocking_ready test helper around manager and probe and pid and port and timeout s; why: the warm selector joins the first starting target scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def blocking_ready(manager, probe, *, pid, port, timeout_s): + # What: act by calling activation_started.set with the declared inputs; why: the warm selector joins the first starting target scenario observes the activation_started.set return value during assert finish activation wait. + activation_started.set() + # What: assert that finish activation wait 2; why: this assertion protects the warm selector joins the first starting target regression after the test's arranged inputs and exercised call. + assert finish_activation.wait(2) + # What: arrange the ready field as true; why: blocking_ready carries ready into return {"ready": True, "health": {"status": "ok"}}. + return {"ready": True, "health": {"status": "ok"}} + + # What: act by calling RoutingCoordinator and capture router; why: the warm selector joins the first starting target test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=blocking_ready) + # What: arrange leases as the fixture input; why: the warm selector joins the first starting target test consumes this named precondition before exercising the behavior. + leases = [] + # What: act by calling threading.Thread and capture first; why: the warm selector joins the first starting target test asserts the response, state, or failure produced by this call. + first = threading.Thread(target=lambda: leases.append(router.acquire("b"))) + # What: act by calling threading.Thread and capture second; why: the warm selector joins the first starting target test asserts the response, state, or failure produced by this call. + second = threading.Thread(target=lambda: leases.append(router.acquire("warm"))) + # What: act by calling first.start with the declared inputs; why: the warm selector joins the first starting target scenario observes the first.start return value during assert activation started wait. + first.start() + # What: assert that activation started wait 1; why: this assertion protects the warm selector joins the first starting target regression after the test's arranged inputs and exercised call. + assert activation_started.wait(1) + # What: act by calling second.start with the declared inputs; why: the warm selector joins the first starting target scenario observes the second.start return value during for value in range. + second.start() + # What: act across range to perform status and router; why: the warm selector joins the first starting target scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on status and router before the computed value; why: the warm selector joins the first starting target scenario admits the computed value only for this predicate and excludes the opposite state. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the warm selector joins the first starting target scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling time.sleep with 0 01; why: the warm selector joins the first starting target scenario observes the time.sleep return value during assert router status queued requests. + time.sleep(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the warm selector joins the first starting target regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + # What: act by calling finish_activation.set with the declared inputs; why: the warm selector joins the first starting target scenario observes the finish_activation.set return value during first join. + finish_activation.set() + # What: act by calling first.join with 2; why: the warm selector joins the first starting target scenario observes the first.join return value during second join. + first.join(2) + # What: act by calling second.join with 2; why: the warm selector joins the first starting target scenario observes the second.join return value during assert not first is alive and not second is alive. + second.join(2) + + # What: assert that not first is alive and not second is alive; why: this assertion protects the warm selector joins the first starting target regression after the test's arranged inputs and exercised call. + assert not first.is_alive() and not second.is_alive() + # What: act by calling next and capture selector lease; why: the warm selector joins the first starting target test asserts the response, state, or failure produced by this call. + selector_lease = next(lease for lease in leases if lease.selector_id == "warm") + # What: assert that selector lease profile name selector lease model id equals b b; why: this assertion protects the warm selector joins the first starting target regression after the test's arranged inputs and exercised call. + assert (selector_lease.profile.name, selector_lease.model_id) == ("b", "b") + # What: assert that manager calls equals start b gguf; why: this assertion protects the warm selector joins the first starting target regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "b.gguf")] + # What: act across leases to perform release and lease; why: the warm selector joins the first starting target scenario repeats the body only while or for the loop header admits an iteration. + for lease in leases: + # What: act by calling lease.release with the declared inputs; why: the warm selector joins the first starting target scenario observes the lease.release return value during the enclosing return. + lease.release() + + +# What: define the test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the selector rewrites before target alias filters and is not an upstream id outcome. +def test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id( + # What: arrange monkeypatch for the scenario; why: test selector rewrites before target alias filters and is not an upstream id requires this concrete input or helper state before exercising the behavior under test. + monkeypatch +# What: arrange the grouped source fragment for the scenario; why: test selector rewrites before target alias filters and is not an upstream id requires this concrete input or helper state before exercising the behavior under test. +): + # What: act by calling RequestField and capture alias fields; why: the selector rewrites before target alias filters and is not an upstream id test asserts the response, state, or failure produced by this call. + alias_fields = (("a:high", ( + # What: act by calling RequestField with temperature and 0 1; why: the selector rewrites before target alias filters and is not an upstream id scenario observes the RequestField return value while evaluating RequestField(("temperature",), "0.1"). + RequestField(("temperature",), "0.1"), + # What: arrange the grouped expression portion of alias fields; why: the selector rewrites before target alias filters and is not an upstream id scenario uses this clause to evaluate alias fields as one grouped value. + )),) + # What: act by calling ModelCatalog and capture catalog doc; why: the selector rewrites before target alias filters and is not an upstream id test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the a field as model profile and alias fields and a and private and gguf; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id carries a through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"a": ModelProfile( + # What: arrange aliases to ModelProfile; why: the selector rewrites before target alias filters and is not an upstream id scenario binds this a and high value to ModelProfile's aliases input. + "a", "private.gguf", (), aliases=("a:high",), + # What: arrange set fields by id to ModelProfile; why: the selector rewrites before target alias filters and is not an upstream id scenario binds this alias fields value to ModelProfile's set fields by id input. + set_fields_by_id=alias_fields, + # What: arrange the catalog_doc mapping with a; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id groups the supplied clauses as one catalog_doc mapping before its value is consumed. + )}, + # What: arrange selectors to ModelCatalog; why: the selector rewrites before target alias filters and is not an upstream id scenario binds this model selector and public and public and pin and public value to ModelCatalog's selectors input. + selectors={ + # What: arrange the public field as model selector and public and pin and public and model; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id carries public through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "public": ModelSelector( + # What: arrange the public pin a high public model portion of catalog doc; why: the selector rewrites before target alias filters and is not an upstream id scenario uses this clause to evaluate catalog doc as one grouped value. + "public", "pin", ("a:high",), "Public Model", "Stable target" + # What: arrange the ModelSelector call with ordered positional inputs; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id groups the supplied clauses as one ModelSelector call before its value is consumed. + ), + # What: arrange the catalog_doc mapping with public; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id groups the supplied clauses as one catalog_doc mapping before its value is consumed. + }, + # What: arrange the ModelCatalog call with selectors; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling Manager and capture manager; why: the selector rewrites before target alias filters and is not an upstream id test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the selector rewrites before target alias filters and is not an upstream id test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange seen as the fixture input; why: the selector rewrites before target alias filters and is not an upstream id test consumes this named precondition before exercising the behavior. + seen = {} + # What: act by calling LogRing and capture router ring; why: the selector rewrites before target alias filters and is not an upstream id test asserts the response, state, or failure produced by this call. + router_ring = LogRing() + + # What: define the upstream test helper around captured fixture state; why: the selector rewrites before target alias filters and is not an upstream id scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: act by calling seen.update with kwargs; why: the selector rewrites before target alias filters and is not an upstream id scenario observes the seen.update return value during return upstream response content type application json bytes io. + seen.update(kwargs) + # What: arrange the helper response as UpstreamResponse 200 Content Type application json BytesIO b; why: test router test feeds this result into the. + return UpstreamResponse(200, {"Content-Type": "application/json"}, BytesIO(b'{}')) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the selector rewrites before target alias filters and is not an upstream id scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) before. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the selector rewrites before target alias filters and is not an upstream id test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the selector rewrites before target alias filters and is not an upstream id scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange router ring to build_app; why: the selector rewrites before target alias filters and is not an upstream id scenario binds this router ring value to build_app's router ring input. + router_ring=router_ring, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the selector rewrites before target alias filters and is not an upstream id test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling client.post and capture response; why: the selector rewrites before target alias filters and is not an upstream id test asserts the response, state, or failure produced by this call. + response = client.post( + # What: arrange the model field as public; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id sends this field through response so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "public", "messages": []} + # What: arrange the client.post call with json; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling client.post and capture direct; why: the selector rewrites before target alias filters and is not an upstream id test asserts the response, state, or failure produced by this call. + direct = client.post( + # What: arrange the upstream public v1 chat completions portion of direct; why: the selector rewrites before target alias filters and is not an upstream id scenario uses this clause to evaluate direct as one grouped value. + "/upstream/public/v1/chat/completions", + # What: arrange the model field as public; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id sends this field through direct so the router selects the canonical model or alias for upstream dispatch. + json={"model": "public", "messages": []}, + # What: arrange the client.post call with json; why: test_selector_rewrites_before_target_alias_filters_and_is_not_an_upstream_id groups the supplied clauses as one client.post call before its value is consumed. + ) + + # What: assert that response status code equals 200; why: this assertion protects the selector rewrites before target alias filters and is not an upstream id regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + # What: assert that json loads seen body model equals a high; why: this assertion protects the selector rewrites before target alias filters and is not an upstream id regression after the test's arranged inputs and exercised call. + assert json.loads(seen["body"])["model"] == "a:high" + # What: assert that json loads seen body temperature equals 0 1; why: this assertion protects the selector rewrites before target alias filters and is not an upstream id regression after the test's arranged inputs and exercised call. + assert json.loads(seen["body"])["temperature"] == 0.1 + # What: assert that direct status code equals 404; why: this assertion protects the selector rewrites before target alias filters and is not an upstream id regression after the test's arranged inputs and exercised call. + assert direct.status_code == 404 + # What: act by calling json.loads and capture events; why: the selector rewrites before target alias filters and is not an upstream id test asserts the response, state, or failure produced by this call. + events = [json.loads(item["text"]) for item in router_ring.since(0)[0]] + # What: act by calling next and capture admitted; why: the selector rewrites before target alias filters and is not an upstream id test asserts the response, state, or failure produced by this call. + admitted = next(event for event in events if event["event"] == "admitted") + # What: assert that admitted profile equals a; why: this assertion protects the selector rewrites before target alias filters and is not an upstream id regression after the test's arranged inputs and exercised call. + assert admitted["profile"] == "a" + # What: assert that admitted selector equals public; why: this assertion protects the selector rewrites before target alias filters and is not an upstream id regression after the test's arranged inputs and exercised call. + assert admitted["selector"] == "public" + # What: assert that admitted target equals a high; why: this assertion protects the selector rewrites before target alias filters and is not an upstream id regression after the test's arranged inputs and exercised call. + assert admitted["target"] == "a:high" + + +# What: define the test_selector_model_listing_uses_strategy_specific_loaded_status test around local fixtures; why: this test groups the arrange, act, and assertions that protect the selector model listing uses strategy specific loaded status outcome. +def test_selector_model_listing_uses_strategy_specific_loaded_status(): + # What: act by calling Manager and capture manager; why: the selector model listing uses strategy specific loaded status test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the selector model listing uses strategy specific loaded status test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with a and b; why: test_selector_model_listing_uses_strategy_specific_loaded_status groups the supplied clauses as one catalog_doc mapping before its value. + { + # What: arrange the a field as model profile and a and a and gguf; why: test_selector_model_listing_uses_strategy_specific_loaded_status carries a through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "a": ModelProfile("a", "a.gguf", ()), + # What: arrange the b field as model profile and b and b and gguf; why: test_selector_model_listing_uses_strategy_specific_loaded_status carries b through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "b": ModelProfile("b", "b.gguf", ()), + # What: arrange the catalog_doc mapping with a and b; why: test_selector_model_listing_uses_strategy_specific_loaded_status groups the supplied clauses as one catalog_doc mapping before its value. + }, + # What: arrange selectors to ModelCatalog; why: the selector model listing uses strategy specific loaded status scenario binds this model selector and pin and warm and hidden and pin value to ModelCatalog's selectors input. + selectors={ + # What: arrange the pin field as model selector and pin and pin and pinned and first; why: test_selector_model_listing_uses_strategy_specific_loaded_status carries pin through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "pin": ModelSelector( + # What: arrange the pin pin a b pinned first portion of catalog doc; why: the selector model listing uses strategy specific loaded status scenario uses this clause to evaluate catalog doc as one grouped value. + "pin", "pin", ("a", "b"), "Pinned", "First only", + # What: arrange metadata json to ModelSelector; why: the selector model listing uses strategy specific loaded status scenario binds this tier and stable and type and operator value value to ModelSelector's metadata json input. + metadata_json='{"tier":"stable","type":"operator-value"}', + # What: arrange the ModelSelector call with metadata json; why: test_selector_model_listing_uses_strategy_specific_loaded_status groups the supplied clauses as one ModelSelector call before its value is consumed. + ), + # What: arrange the warm field as model selector and warm and warm and a and b; why: test_selector_model_listing_uses_strategy_specific_loaded_status carries warm through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "warm": ModelSelector("warm", "warm", ("a", "b")), + # What: arrange the hidden field as model selector and hidden and pin and b and true; why: test_selector_model_listing_uses_strategy_specific_loaded_status carries hidden through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "hidden": ModelSelector("hidden", "pin", ("b",), unlisted=True), + # What: arrange the catalog_doc mapping with pin and warm and hidden; why: test_selector_model_listing_uses_strategy_specific_loaded_status groups the supplied clauses as one catalog_doc mapping before its value is consumed. + }, + # What: arrange the ModelCatalog call with selectors; why: test_selector_model_listing_uses_strategy_specific_loaded_status groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the selector model listing uses strategy specific loaded status test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange the exact router acquire b release fixture fragment; why: the selector model listing uses strategy specific loaded status scenario feeds this byte-preserved fragment through router.acquire("b").release() before asserting its protocol or parser result. + router.acquire("b").release() + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_selector_model_listing_uses_strategy_specific_loaded_status releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the selector model listing uses strategy specific loaded status test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_selector_model_listing_uses_strategy_specific_loaded_status; why: test_selector_model_listing_uses_strategy_specific_loaded_status consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the selector model listing uses strategy specific loaded status scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_selector_model_listing_uses_strategy_specific_loaded_status groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the selector model listing uses strategy specific loaded status test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling operation.json and capture records; why: the selector model listing uses strategy specific loaded status test asserts the response, state, or failure produced by this call. + records = {item["id"]: item for item in client.get("/v1/models").json()["data"]} + # What: act by calling operation.json and capture management; why: the selector model listing uses strategy specific loaded status test asserts the response, state, or failure produced by this call. + management = client.get("/router/profiles").json() + + # What: assert that hidden is absent from records; why: this assertion protects the selector model listing uses strategy specific loaded status regression after the test's arranged inputs and exercised call. + assert "hidden" not in records + # What: assert that records pin status value equals unloaded; why: this assertion protects the selector model listing uses strategy specific loaded status regression after the test's arranged inputs and exercised call. + assert records["pin"]["status"]["value"] == "unloaded" + # What: assert that records warm status value equals loaded; why: this assertion protects the selector model listing uses strategy specific loaded status regression after the test's arranged inputs and exercised call. + assert records["warm"]["status"]["value"] == "loaded" + # What: assert that records pin name equals pinned; why: this assertion protects the selector model listing uses strategy specific loaded status regression after the test's arranged inputs and exercised call. + assert records["pin"]["name"] == "Pinned" + # What: assert that records pin description equals first only; why: this assertion protects the selector model listing uses strategy specific loaded status regression after the test's arranged inputs and exercised call. + assert records["pin"]["description"] == "First only" + # What: assert the expected records pin meta == freetoken outcome; why: test router test selector model listing uses strategy specific loaded status protects its regression by requiring this observable result after the exercised behavior. + assert records["pin"]["meta"] == {"freetoken": { + # What: arrange tier stable type selector strategy pin for the scenario; why: test router test selector model listing uses strategy specific loaded status requires this concrete input or helper state before exercising the behavior under test. + "tier": "stable", "type": "selector", "strategy": "pin", + # What: arrange targets a b for the scenario; why: test router test selector model listing uses strategy specific loaded status requires this concrete input or helper state before exercising the behavior under test. + "targets": ["a", "b"], + # What: arrange the grouped source fragment for the scenario; why: test router test selector model listing uses strategy specific loaded status requires this concrete input or helper state before exercising. + }} + # What: assert the expected item name for item in management selectors == outcome; why: test router test selector model listing uses strategy specific loaded status protects its regression by requiring this observable result after the exercised behavior. + assert {item["name"] for item in management["selectors"]} == { + # What: arrange pin warm hidden for the scenario; why: test router test selector model listing uses strategy specific loaded status requires this concrete input or helper state before exercising the behavior under test. + "pin", "warm", "hidden", + # What: arrange the grouped source fragment for the scenario; why: test router test selector model listing uses strategy specific loaded status requires this concrete input or helper state before. + } + + +# What: define the test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models test around local fixtures; why: this test groups the arrange, act, and assertions that protect the runtime profile pins compose before warm selectors and can shadow models outcome. +def test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models(): + # What: act by calling ModelCatalog and capture catalog doc; why: the runtime profile pins compose before warm selectors and can shadow models test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with a and b; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models groups the supplied clauses as one catalog_doc mapping before its value. + { + # What: arrange the a field as model profile and a and a and gguf; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models carries a through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "a": ModelProfile("a", "a.gguf", ()), + # What: arrange the b field as model profile and b and b and gguf; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models carries b through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "b": ModelProfile("b", "b.gguf", ()), + # What: arrange the catalog_doc mapping with a and b; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models groups the supplied clauses as one catalog_doc mapping before its value. + }, + # What: arrange the warm field as model selector and warm and warm and a and b; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models carries warm through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + selectors={"warm": ModelSelector("warm", "warm", ("a", "b"))}, + # What: arrange the coding field as routing profile and coding and a and b and disabled; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models carries coding through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + routing_profiles={"coding": RoutingProfile( + # What: arrange the coding a b disabled public warm portion of catalog doc; why: the runtime profile pins compose before warm selectors and can shadow models scenario uses this clause to evaluate catalog doc as one grouped value. + "coding", (("a", "b"), ("disabled", None), ("public", "warm")) + # What: arrange the catalog_doc mapping with coding; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models groups the supplied clauses as one catalog_doc mapping before its value is consumed. + )}, + # What: arrange the ModelCatalog call with selectors and routing profiles; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling Manager and capture manager; why: the runtime profile pins compose before warm selectors and can shadow models test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the runtime profile pins compose before warm selectors and can shadow models test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange the exact router acquire b release fixture fragment; why: the runtime profile pins compose before warm selectors and can shadow models scenario feeds this byte-preserved fragment through router.acquire("b").release() before asserting its protocol or parser result. + router.acquire("b").release() + # What: assert that router set active routing profile coding equals coding; why: this assertion protects the runtime profile pins compose before warm selectors and can shadow models regression after the test's arranged inputs and exercised call. + assert router.set_active_routing_profile("coding") == "coding" + + # What: act by calling router.acquire and capture selected; why: the runtime profile pins compose before warm selectors and can shadow models test asserts the response, state, or failure produced by this call. + selected = router.acquire("public") + # What: Assert assert in test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models uses this assert to implement the named assert operation. + assert ( + # What: Assert selected profile name in test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models uses this assert to implement the named selected profile name operation. + selected.profile.name, + # What: Assert selected model id in test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models uses this assert to implement the named selected model id operation. + selected.model_id, + # What: Assert selected selector id in test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models uses this assert to implement the named selected selector id operation. + selected.selector_id, + # What: Assert selected routing profile id in test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models uses this assert to implement the named selected routing profile id operation. + selected.routing_profile_id, + # What: Assert selected pin id in test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models uses this assert to implement the named selected pin id operation. + selected.pin_id, + # What: Assert equals b b warm coding public in test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models; why: test_runtime_profile_pins_compose_before_warm_selectors_and_can_shadow_models uses this assert to implement the named equals b b warm coding public operation. + ) == ("b", "b", "warm", "coding", "public") + # What: act by calling selected.release with the declared inputs; why: the runtime profile pins compose before warm selectors and can shadow models scenario observes the selected.release return value during shadowed router acquire a. + selected.release() + # What: act by calling router.acquire and capture shadowed; why: the runtime profile pins compose before warm selectors and can shadow models test asserts the response, state, or failure produced by this call. + shadowed = router.acquire("a") + # What: assert that shadowed profile name shadowed model id shadowed pin id equals b b a; why: this assertion protects the runtime profile pins compose before warm selectors and can shadow models regression after the test's arranged inputs and exercised call. + assert (shadowed.profile.name, shadowed.model_id, shadowed.pin_id) == ("b", "b", "a") + # What: act by calling shadowed.release with the declared inputs; why: the runtime profile pins compose before warm selectors and can shadow models scenario observes the shadowed.release return value during with pytest raises routing error match disabled by. + shadowed.release() + # What: arrange with pytest raises RoutingError match disabled by routing profile as disabled for the scenario; why: test raises routing error match disabled by in test runtime profile pins compose before warm requires this concrete input or helper state before exercising the behavior under test. + with pytest.raises(RoutingError, match="disabled by routing profile") as disabled: + # What: arrange the exact router acquire disabled fixture fragment; why: the runtime profile pins compose before warm selectors and can shadow models scenario feeds this byte-preserved fragment through router.acquire("disabled") before asserting its protocol or parser result. + router.acquire("disabled") + # What: assert that disabled value code equals unknown model; why: this assertion protects the runtime profile pins compose before warm selectors and can shadow models regression after the test's arranged inputs and exercised call. + assert disabled.value.code == "unknown_model" + # What: assert that router has routable id disabled is false; why: this assertion protects the runtime profile pins compose before warm selectors and can shadow models regression after the test's arranged inputs and exercised call. + assert router.has_routable_id("disabled") is False + # What: assert that manager calls equals start b gguf; why: this assertion protects the runtime profile pins compose before warm selectors and can shadow models regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "b.gguf")] + + # What: assert that router set active routing profile is group delimiter; why: this assertion protects the runtime profile pins compose before warm selectors and can shadow models regression after the test's arranged inputs and exercised call. + assert router.set_active_routing_profile(None) is None + # What: arrange with pytest raises RoutingError as missing for the scenario; why: test raises routing error as missing requires this concrete input or helper state before exercising the behavior under test. + with pytest.raises(RoutingError) as missing: + # What: arrange the exact router acquire public fixture fragment; why: the runtime profile pins compose before warm selectors and can shadow models scenario feeds this byte-preserved fragment through router.acquire("public") before asserting its protocol or parser result. + router.acquire("public") + # What: assert that missing value code equals unknown model; why: this assertion protects the runtime profile pins compose before warm selectors and can shadow models regression after the test's arranged inputs and exercised call. + assert missing.value.code == "unknown_model" + # What: arrange with pytest raises RoutingError as unknown profile for the scenario; why: test raises routing error as unknown profile requires this concrete input or helper state before exercising the behavior under test. + with pytest.raises(RoutingError) as unknown_profile: + # What: arrange the exact router set active routing profile missing fixture fragment; why: the runtime profile pins compose before warm selectors and can shadow models scenario feeds this byte-preserved fragment through router.set_active_routing_profile("missing") before asserting its protocol or parser result. + router.set_active_routing_profile("missing") + # What: assert that unknown profile value code equals unknown profile; why: this assertion protects the runtime profile pins compose before warm selectors and can shadow models regression after the test's arranged inputs and exercised call. + assert unknown_profile.value.code == "unknown_profile" + + +# What: define the test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the runtime profile http rewrites before alias filters and lists virtual pins outcome. +def test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins( + # What: arrange monkeypatch for the scenario; why: test runtime profile http rewrites before alias filters and lists virtual pins requires this concrete input or helper state before exercising the behavior under test. + monkeypatch +# What: arrange the grouped source fragment for the scenario; why: test runtime profile http rewrites before alias filters and lists virtual pins requires this concrete input or helper state before exercising the behavior under test. +): + # What: act by calling ModelCatalog and capture catalog doc; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the a field as model profile and request field and a and private and gguf; why: test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins carries a through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"a": ModelProfile( + # What: arrange aliases to ModelProfile; why: the runtime profile http rewrites before alias filters and lists virtual pins scenario binds this a and high value to ModelProfile's aliases input. + "a", "private.gguf", (), aliases=("a:high",), + # What: arrange set fields by id to RequestField; why: the runtime profile http rewrites before alias filters and lists virtual pins scenario binds this request field and a and high and 0 1 and temperature value to RequestField's set fields by id input. + set_fields_by_id=(("a:high", (RequestField(("temperature",), "0.1"),)),), + # What: arrange the catalog_doc mapping with a; why: test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins groups the supplied clauses as one catalog_doc mapping before its value is consumed. + )}, + # What: arrange the coding field as routing profile and coding and coding and mode and disabled; why: test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins carries coding through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + routing_profiles={"coding": RoutingProfile( + # What: arrange the coding portion of catalog doc; why: the runtime profile http rewrites before alias filters and lists virtual pins scenario uses this clause to evaluate catalog doc as one grouped value. + "coding", + # What: arrange the disabled public a high portion of catalog doc; why: the runtime profile http rewrites before alias filters and lists virtual pins scenario uses this clause to evaluate catalog doc as one grouped value. + (("disabled", None), ("public", "a:high")), + # What: arrange the coding mode portion of catalog doc; why: the runtime profile http rewrites before alias filters and lists virtual pins scenario uses this clause to evaluate catalog doc as one grouped value. + "Coding mode", + # What: arrange the catalog_doc mapping with coding; why: test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins groups the supplied clauses as one catalog_doc mapping before its value is consumed. + )}, + # What: arrange the ModelCatalog call with routing profiles; why: test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling Manager and capture manager; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange seen as the fixture input; why: the runtime profile http rewrites before alias filters and lists virtual pins test consumes this named precondition before exercising the behavior. + seen = [] + # What: act by calling LogRing and capture ring; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + ring = LogRing() + + # What: define the upstream test helper around captured fixture state; why: the runtime profile http rewrites before alias filters and lists virtual pins scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: act by calling seen.append with kwargs; why: the runtime profile http rewrites before alias filters and lists virtual pins scenario observes the seen.append return value during return upstream response content type application json bytes io. + seen.append(kwargs) + # What: arrange the helper response as UpstreamResponse 200 Content Type application json BytesIO b; why: test runtime profile http rewrites feeds this result into the behavior whose outcome is asserted. + return UpstreamResponse(200, {"Content-Type": "application/json"}, BytesIO(b'{}')) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the runtime profile http rewrites before alias filters and lists virtual pins scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) befo. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins; why: test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the runtime profile http rewrites before alias filters and lists virtual pins scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange router ring to build_app; why: the runtime profile http rewrites before alias filters and lists virtual pins scenario binds this ring value to build_app's router ring input. + router_ring=ring, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling operation.json and capture initial; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + initial = client.get("/router/profiles").json() + # What: act by calling client.put and capture activated; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + activated = client.put("/router/profiles/active", json={"name": "coding"}) + # What: act by calling client.post and capture routed; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + routed = client.post( + # What: arrange the model field as public; why: test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins sends this field through routed so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "public", "messages": []} + # What: arrange the client.post call with json; why: test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling client.post and capture direct; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + direct = client.post( + # What: arrange the upstream public custom fpart opaque a portion of direct; why: the runtime profile http rewrites before alias filters and lists virtual pins scenario uses this clause to evaluate direct as one grouped value. + "/upstream/public/custom%2Fpart?opaque=a%2Fb", + # What: arrange content to client.post; why: the runtime profile http rewrites before alias filters and lists virtual pins scenario binds this the named fixture input value to client.post's content input. + content=b'{"model":"public"}', + # What: arrange the content type field as application and json; why: test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins carries content type through direct into assert routed status code equals direct status code equals 200. + headers={"Content-Type": "application/json"}, + # What: arrange the client.post call with content and headers; why: test_runtime_profile_http_rewrites_before_alias_filters_and_lists_virtual_pins groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling operation.json and capture listed; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + listed = {item["id"]: item for item in client.get("/v1/models").json()["data"]} + # What: act by calling client.post and capture disabled; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + disabled = client.post("/v1/chat/completions", json={"model": "disabled"}) + # What: act by calling client.put and capture cleared; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + cleared = client.put("/router/profiles/active", json={"name": None}) + # What: act by calling client.put and capture missing; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + missing = client.put("/router/profiles/active", json={"name": "missing"}) + + # What: assert that initial active routing profile is group delimiter; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert initial["activeRoutingProfile"] is None + # What: assert the expected initial routingProfiles == outcome; why: test router test runtime profile http rewrites before alias filters and lists virtual pins protects its regression by requiring this observable result after the exercised behavior. + assert initial["routingProfiles"] == [{ + # What: arrange name coding description Coding mode for the scenario; why: test router test runtime profile http rewrites before alias filters and lists virtual pins requires this concrete input or helper state before exercising the behavior under test. + "name": "coding", "description": "Coding mode", + # What: arrange pins disabled None public a high for the scenario; why: test router test runtime profile http rewrites before alias filters and lists virtual pins requires this concrete input or helper state before exercising the behavior under test. + "pins": {"disabled": None, "public": "a:high"}, + # What: arrange the grouped source fragment for the scenario; why: test router test runtime profile http rewrites before alias filters and lists virtual pins requires this concrete input. + }] + # What: assert that activated json equals active coding; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert activated.json() == {"active": "coding"} + # What: assert that routed status code equals direct status code equals 200; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert routed.status_code == direct.status_code == 200 + # What: assert the expected json loads seen 0 body == outcome; why: test router test runtime profile http rewrites before alias filters and lists virtual pins protects its regression by requiring this observable result after the exercised behavior. + assert json.loads(seen[0]["body"]) == { + # What: arrange model a high messages temperature 0.1 for the scenario; why: test router test runtime profile http rewrites before alias filters and lists virtual pins requires this concrete input or helper state before exercising the behavior under test. + "model": "a:high", "messages": [], "temperature": 0.1, + # What: arrange the grouped source fragment for the scenario; why: test router test runtime profile http rewrites before alias filters and lists virtual pins requires this concrete. + } + # What: assert that seen 1 path and query equals custom 2 fpart opaque a 2; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert seen[1]["path_and_query"] == "/custom%2Fpart?opaque=a%2Fb" + # What: assert that json loads seen 1 body equals model public temperature 0 1; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert json.loads(seen[1]["body"]) == {"model": "public", "temperature": 0.1} + # What: assert that listed public status value equals unloaded; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert listed["public"]["status"]["value"] == "unloaded" + # What: assert that listed public meta equals freetoken type profile; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert listed["public"]["meta"] == {"freetoken": {"type": "profile"}} + # What: assert that disabled is absent from listed; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert "disabled" not in listed + # What: assert that disabled status code equals 404; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert disabled.status_code == 404 + # What: assert that cleared json equals active; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert cleared.json() == {"active": None} + # What: assert that missing status code equals 404; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert missing.status_code == 404 + # What: act by calling json.loads and capture events; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + events = [json.loads(item["text"]) for item in ring.since(0)[0]] + # What: act by calling next and capture admitted; why: the runtime profile http rewrites before alias filters and lists virtual pins test asserts the response, state, or failure produced by this call. + admitted = next(event for event in events if event["event"] == "admitted") + # What: assert that admitted routing profile equals coding; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert admitted["routingProfile"] == "coding" + # What: assert that admitted pin equals public; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert admitted["pin"] == "public" + # What: assert that admitted target equals a high; why: this assertion protects the runtime profile http rewrites before alias filters and lists virtual pins regression after the test's arranged inputs and exercised call. + assert admitted["target"] == "a:high" + + +# What: define the test_runtime_profile_direct_upstream_uses_longest_pin_and_rejects_selector_target test around local fixtures; why: this test groups the arrange, act, and assertions that protect the runtime profile direct upstream uses longest pin and rejects selector target outcome. +def test_runtime_profile_direct_upstream_uses_longest_pin_and_rejects_selector_target(): + # What: act by calling ModelCatalog and capture catalog doc; why: the runtime profile direct upstream uses longest pin and rejects selector target test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with a and b; why: test_runtime_profile_direct_upstream_uses_longest_pin_and_rejects_selector_target groups the supplied clauses as one catalog_doc mapping before its value. + { + # What: arrange the a field as model profile and a and a and gguf; why: test_runtime_profile_direct_upstream_uses_longest_pin_and_rejects_selector_target carries a through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "a": ModelProfile("a", "a.gguf", ()), + # What: arrange the b field as model profile and b and b and gguf; why: test_runtime_profile_direct_upstream_uses_longest_pin_and_rejects_selector_target carries b through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "b": ModelProfile("b", "b.gguf", ()), + # What: arrange the catalog_doc mapping with a and b; why: test_runtime_profile_direct_upstream_uses_longest_pin_and_rejects_selector_target groups the supplied clauses as one catalog_doc mapping before its value. + }, + # What: arrange the virtual field as model selector and virtual and pin and a; why: test_runtime_profile_direct_upstream_uses_longest_pin_and_rejects_selector_target carries virtual through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + selectors={"virtual": ModelSelector("virtual", "pin", ("a",))}, + # What: arrange the coding field as routing profile and coding and author and a and author; why: test_runtime_profile_direct_upstream_uses_longest_pin_and_rejects_selector_target carries coding through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + routing_profiles={"coding": RoutingProfile( + # What: arrange the coding author a author public b portion of catalog doc; why: the runtime profile direct upstream uses longest pin and rejects selector target scenario uses this clause to evaluate catalog doc as one grouped value. + "coding", (("author", "a"), ("author/public", "b"), ("select", "virtual")) + # What: arrange the catalog_doc mapping with coding; why: test_runtime_profile_direct_upstream_uses_longest_pin_and_rejects_selector_target groups the supplied clauses as one catalog_doc mapping before its value is consumed. + )}, + # What: arrange the ModelCatalog call with selectors and routing profiles; why: test_runtime_profile_direct_upstream_uses_longest_pin_and_rejects_selector_target groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the runtime profile direct upstream uses longest pin and rejects selector target test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(Manager(), catalog_doc, object(), ready_fn=ready) + # What: arrange the exact router set active routing profile coding fixture fragment; why: the runtime profile direct upstream uses longest pin and rejects selector target scenario feeds this byte-preserved fragment through router.set_active_routing_profile("coding") before asserting its protocol or parser result. + router.set_active_routing_profile("coding") + + # What: assert the expected router resolve upstream path author public v1 stats 2 == outcome; why: test runtime profile direct upstream uses longest protects its regression by requiring this observable result after the exercised behavior. + assert router.resolve_upstream_path("author/public/v1/stats")[:2] == ( + # What: arrange author public b for the scenario; why: test router test runtime profile direct upstream uses longest pin and rejects selector target requires this concrete input or helper state before exercising the behavior under test. + "author/public", "b", + # What: arrange the grouped source fragment for the scenario; why: test router test runtime profile direct upstream uses longest pin and rejects selector target requires this concrete input or helper state before exercising the behavior under test. + ) + # What: assert that router resolve upstream path author public v1 stats 3 equals v1 stats; why: this assertion protects the runtime profile direct upstream uses longest pin and rejects selector target regression after the test's arranged inputs and exercised call. + assert router.resolve_upstream_path("author/public/v1/stats")[3] == "/v1/stats" + # What: assert the pytest.raises failure context; why: the runtime profile direct upstream uses longest pin and rejects selector target scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match="configured model ID"): + # What: arrange the exact router resolve upstream path select v1 stats fixture fragment; why: the runtime profile direct upstream uses longest pin and rejects selector target scenario feeds this byte-preserved fragment through router.resolve_upstream_path("select/v1/stats") before asserting its protocol or par. + router.resolve_upstream_path("select/v1/stats") + + +# What: define the test_catalog_reload_clears_active_runtime_profile test around local fixtures; why: this test groups the arrange, act, and assertions that protect the catalog reload clears active runtime profile outcome. +def test_catalog_reload_clears_active_runtime_profile(): + # What: act by calling RoutingProfile and capture profile; why: the catalog reload clears active runtime profile test asserts the response, state, or failure produced by this call. + profile = RoutingProfile("coding", (("public", "a"),)) + # What: act by calling ModelCatalog and capture current; why: the catalog reload clears active runtime profile test asserts the response, state, or failure produced by this call. + current = ModelCatalog( + # What: arrange the a field as model profile and a and a and gguf; why: test_catalog_reload_clears_active_runtime_profile carries a through current into router routing coordinator manager current object ready fn ready. + {"a": ModelProfile("a", "a.gguf", ())}, routing_profiles={"coding": profile} + # What: arrange the ModelCatalog call with routing profiles; why: test_catalog_reload_clears_active_runtime_profile groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the catalog reload clears active runtime profile test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(Manager(), current, object(), ready_fn=ready) + # What: arrange the exact router set active routing profile coding fixture fragment; why: the catalog reload clears active runtime profile scenario feeds this byte-preserved fragment through router.set_active_routing_profile("coding") before asserting its protocol or parser result. + router.set_active_routing_profile("coding") + + # What: act by calling router.replace_catalog with model catalog and model profile and profile and a and a; why: the catalog reload clears active runtime profile scenario observes the router.replace_catalog return value during a model profile a a gguf routing profiles coding. + router.replace_catalog(ModelCatalog( + # What: arrange the a field as model profile and a and a and gguf; why: test_catalog_reload_clears_active_runtime_profile carries a into {"a": ModelProfile("a", "a.gguf", ())}, routing_profiles={"coding": prof. + {"a": ModelProfile("a", "a.gguf", ())}, routing_profiles={"coding": profile} + # What: arrange the router.replace_catalog call with model catalog; why: test_catalog_reload_clears_active_runtime_profile groups the supplied clauses as one router.replace_catalog call before its value is consumed. + )) + + # What: act by calling router.control_plane_snapshot and capture catalog snapshot and route state; why: the catalog reload clears active runtime profile test asserts the response, state, or failure produced by this call. + catalog_snapshot, route_state = router.control_plane_snapshot() + # What: assert that catalog snapshot routing profile coding equals profile; why: this assertion protects the catalog reload clears active runtime profile regression after the test's arranged inputs and exercised call. + assert catalog_snapshot.routing_profile("coding") == profile + # What: assert that route state active routing profile is group delimiter; why: this assertion protects the catalog reload clears active runtime profile regression after the test's arranged inputs and exercised call. + assert route_state["activeRoutingProfile"] is None + # What: assert that router has routable id public is false; why: this assertion protects the catalog reload clears active runtime profile regression after the test's arranged inputs and exercised call. + assert router.has_routable_id("public") is False + + +# What: define the test_management_load_ignores_active_routing_profile_pin test around local fixtures; why: this test groups the arrange, act, and assertions that protect the management load ignores active routing profile pin outcome. +def test_management_load_ignores_active_routing_profile_pin(): + # What: act by calling ModelCatalog and capture catalog doc; why: the management load ignores active routing profile pin test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with a and b; why: test_management_load_ignores_active_routing_profile_pin groups the supplied clauses as one catalog_doc mapping before its value is consumed. + { + # What: arrange the a field as model profile and a and a and gguf; why: test_management_load_ignores_active_routing_profile_pin carries a through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "a": ModelProfile("a", "a.gguf", ()), + # What: arrange the b field as model profile and b and b and gguf; why: test_management_load_ignores_active_routing_profile_pin carries b through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "b": ModelProfile("b", "b.gguf", ()), + # What: arrange the catalog_doc mapping with a and b; why: test_management_load_ignores_active_routing_profile_pin groups the supplied clauses as one catalog_doc mapping before its value is consumed. + }, + # What: arrange the coding field as routing profile and coding and a and b; why: test_management_load_ignores_active_routing_profile_pin carries coding through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + routing_profiles={"coding": RoutingProfile("coding", (("a", "b"),))}, + # What: arrange the ModelCatalog call with routing profiles; why: test_management_load_ignores_active_routing_profile_pin groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling Manager and capture manager; why: the management load ignores active routing profile pin test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the management load ignores active routing profile pin test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange the exact router set active routing profile coding fixture fragment; why: the management load ignores active routing profile pin scenario feeds this byte-preserved fragment through router.set_active_routing_profile("coding") before asserting its protocol or parser result. + router.set_active_routing_profile("coding") + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_management_load_ignores_active_routing_profile_pin releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the management load ignores active routing profile pin test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_management_load_ignores_active_routing_profile_pin; why: test_management_load_ignores_active_routing_profile_pin consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the management load ignores active routing profile pin scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_management_load_ignores_active_routing_profile_pin groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture loaded; why: the management load ignores active routing profile pin test asserts the response, state, or failure produced by this call. + loaded = TestClient(app).post("/router/load", json={"name": "a"}) + + # What: assert that loaded status code equals 200; why: this assertion protects the management load ignores active routing profile pin regression after the test's arranged inputs and exercised call. + assert loaded.status_code == 200 + # What: assert that loaded json profile equals a; why: this assertion protects the management load ignores active routing profile pin regression after the test's arranged inputs and exercised call. + assert loaded.json()["profile"] == "a" + # What: assert that manager calls equals start a gguf; why: this assertion protects the management load ignores active routing profile pin regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "a.gguf")] + + +# What: define the test_queued_request_keeps_its_atomic_routing_profile_snapshot test around local fixtures; why: this test groups the arrange, act, and assertions that protect the queued request keeps its atomic routing profile snapshot outcome. +def test_queued_request_keeps_its_atomic_routing_profile_snapshot(): + # What: act by calling ModelCatalog and capture catalog doc; why: the queued request keeps its atomic routing profile snapshot test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with a and b; why: test_queued_request_keeps_its_atomic_routing_profile_snapshot groups the supplied clauses as one catalog_doc mapping before its value is consumed. + { + # What: arrange the a field as model profile and a and a and gguf; why: test_queued_request_keeps_its_atomic_routing_profile_snapshot carries a through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "a": ModelProfile("a", "a.gguf", ()), + # What: arrange the b field as model profile and b and b and gguf; why: test_queued_request_keeps_its_atomic_routing_profile_snapshot carries b through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "b": ModelProfile("b", "b.gguf", ()), + # What: arrange the catalog_doc mapping with a and b; why: test_queued_request_keeps_its_atomic_routing_profile_snapshot groups the supplied clauses as one catalog_doc mapping before its value. + }, + # What: arrange the coding field as routing profile and coding and public and a; why: test_queued_request_keeps_its_atomic_routing_profile_snapshot carries coding through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + routing_profiles={"coding": RoutingProfile("coding", (("public", "a"),))}, + # What: arrange the ModelCatalog call with routing profiles; why: test_queued_request_keeps_its_atomic_routing_profile_snapshot groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling Manager and capture manager; why: the queued request keeps its atomic routing profile snapshot test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the queued request keeps its atomic routing profile snapshot test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: act by calling router.acquire and capture active; why: the queued request keeps its atomic routing profile snapshot test asserts the response, state, or failure produced by this call. + active = router.acquire("b") + # What: arrange the exact router set active routing profile coding fixture fragment; why: the queued request keeps its atomic routing profile snapshot scenario feeds this byte-preserved fragment through router.set_active_routing_profile("coding") before asserting its protocol or parser result. + router.set_active_routing_profile("coding") + # What: arrange leases as the fixture input; why: the queued request keeps its atomic routing profile snapshot test consumes this named precondition before exercising the behavior. + leases = [] + # What: act by calling threading.Thread and capture waiting; why: the queued request keeps its atomic routing profile snapshot test asserts the response, state, or failure produced by this call. + waiting = threading.Thread(target=lambda: leases.append(router.acquire("public"))) + # What: act by calling waiting.start with the declared inputs; why: the queued request keeps its atomic routing profile snapshot scenario observes the waiting.start return value during for value in range. + waiting.start() + # What: act across range to perform status and router; why: the queued request keeps its atomic routing profile snapshot scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on status and router before the computed value; why: the queued request keeps its atomic routing profile snapshot scenario admits the computed value only for this predicate and excludes the opposite state. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the queued request keeps its atomic routing profile snapshot scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling time.sleep with 0 01; why: the queued request keeps its atomic routing profile snapshot scenario observes the time.sleep return value during assert router status queued requests. + time.sleep(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the queued request keeps its atomic routing profile snapshot regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + + # What: act by calling router.set_active_routing_profile with the named fixture input; why: the queued request keeps its atomic routing profile snapshot scenario observes the router.set_active_routing_profile return value during active release. + router.set_active_routing_profile(None) + # What: act by calling active.release with the declared inputs; why: the queued request keeps its atomic routing profile snapshot scenario observes the active.release return value during waiting join. + active.release() + # What: act by calling waiting.join with 2; why: the queued request keeps its atomic routing profile snapshot scenario observes the waiting.join return value during assert not waiting is alive. + waiting.join(2) + + # What: assert that waiting is alive is false; why: this assertion protects the queued request keeps its atomic routing profile snapshot regression after the test's arranged inputs and exercised call. + assert not waiting.is_alive() + # What: act by calling leases.pop and capture lease; why: the queued request keeps its atomic routing profile snapshot test asserts the response, state, or failure produced by this call. + lease = leases.pop() + # What: assert the expected lease profile name lease routing profile id lease pin id == outcome; why: test router test queued request keeps its atomic routing profile snapshot protects its regression by requiring this observable result after the exercised behavior. + assert (lease.profile.name, lease.routing_profile_id, lease.pin_id) == ( + # What: arrange a coding public for the scenario; why: test router test queued request keeps its atomic routing profile snapshot requires this concrete input or helper state before exercising the behavior under test. + "a", "coding", "public", + # What: arrange the grouped source fragment for the scenario; why: test router test queued request keeps its atomic routing profile snapshot requires this concrete input or helper state before exercising the behavior under test. + ) + # What: act by calling lease.release with the declared inputs; why: the queued request keeps its atomic routing profile snapshot scenario observes the lease.release return value during assert manager calls start b gguf switch a gguf. + lease.release() + # What: assert that manager calls equals start b gguf switch a gguf; why: this assertion protects the queued request keeps its atomic routing profile snapshot regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "b.gguf"), ("switch", "a.gguf")] + + +# What: define the test_model_list_renders_capability_metadata_for_canonical_and_alias test around local fixtures; why: this test groups the arrange, act, and assertions that protect the model list renders capability metadata for canonical and alias outcome. +def test_model_list_renders_capability_metadata_for_canonical_and_alias(): + # What: act by calling Manager and capture manager; why: the model list renders capability metadata for canonical and alias test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the model list renders capability metadata for canonical and alias test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with canonical; why: test_model_list_renders_capability_metadata_for_canonical_and_alias groups the supplied clauses as one catalog_doc mapping before its value. + { + # What: arrange the canonical field as model profile and model capabilities and canonical and private and gguf; why: test_model_list_renders_capability_metadata_for_canonical_and_alias carries canonical through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "canonical": ModelProfile( + # What: arrange the canonical portion of catalog doc; why: the model list renders capability metadata for canonical and alias scenario uses this clause to evaluate catalog doc as one grouped value. + "canonical", + # What: arrange the private gguf portion of catalog doc; why: the model list renders capability metadata for canonical and alias scenario uses this clause to evaluate catalog doc as one grouped value. + "private.gguf", + # What: arrange the catalog_doc collection with ordered entries; why: test_model_list_renders_capability_metadata_for_canonical_and_alias groups the supplied clauses as one catalog_doc collection before its value is consumed. + (), + # What: arrange aliases to ModelProfile; why: the model list renders capability metadata for canonical and alias scenario binds this compat id value to ModelProfile's aliases input. + aliases=("compat-id",), + # What: arrange capabilities to ModelCapabilities; why: the model list renders capability metadata for canonical and alias scenario binds this model capabilities and true and 32768 and text and text value to ModelCapabilities's capabilities input. + capabilities=ModelCapabilities(("text",), ("text",), True, 32768), + # What: arrange display name to ModelProfile; why: the model list renders capability metadata for canonical and alias scenario binds this canonical and model value to ModelProfile's display name input. + display_name=" Canonical Model ", + # What: arrange description to ModelProfile; why: the model list renders capability metadata for canonical and alias scenario binds this public and description value to ModelProfile's description input. + description=" Public description ", + # What: arrange metadata json to ModelProfile; why: the model list renders capability metadata for canonical and alias scenario binds this architecture and operator and context window and custom value to ModelProfile's metadata json input. + metadata_json=( + # What: arrange the exact architecture operator context window custom remain fixture fragment; why: the model list renders capability metadata for canonical and alias scenario feeds this byte-preserved fragment through catalog doc before asserting its protocol or parser result. + # What: arrange the exact type operator fixture fragment; why: the model list renders capability metadata for canonical and alias scenario feeds this byte-preserved fragment through catalog doc before asserting its protocol or parser result. + '{"architecture":"operator","context_window":1,"custom":"remain",' + '"type":"operator"}' + # What: arrange the ModelProfile call with aliases and capabilities and display name and description and metadata json; why: test_model_list_renders_capability_metadata_for_canonical_and_alias groups the supplied clauses as one ModelProfile call before its value is consumed. + ), + # What: arrange the ModelProfile call with aliases and capabilities and display name and description and metadata json; why: test_model_list_renders_capability_metadata_for_canonical_and_alias groups the supplied clauses as one ModelProfile call before its value is consumed. + ) + # What: arrange the catalog_doc mapping with canonical; why: test_model_list_renders_capability_metadata_for_canonical_and_alias groups the supplied clauses as one catalog_doc mapping before its value. + }, + # What: arrange settings to RouterSettings; why: the model list renders capability metadata for canonical and alias scenario binds this router settings and true value to RouterSettings's settings input. + settings=RouterSettings(include_aliases_in_list=True), + # What: arrange the ModelCatalog call with settings; why: test_model_list_renders_capability_metadata_for_canonical_and_alias groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the model list renders capability metadata for canonical and alias test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_model_list_renders_capability_metadata_for_canonical_and_alias releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the model list renders capability metadata for canonical and alias test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_model_list_renders_capability_metadata_for_canonical_and_alias; why: test_model_list_renders_capability_metadata_for_canonical_and_alias consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the model list renders capability metadata for canonical and alias scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_model_list_renders_capability_metadata_for_canonical_and_alias groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.json and capture data; why: the model list renders capability metadata for canonical and alias test asserts the response, state, or failure produced by this call. + data = TestClient(app).get("/v1/models").json()["data"] + + # What: assert that record id for record in data equals canonical compat id; why: this assertion protects the model list renders capability metadata for canonical and alias regression after the test's arranged inputs and exercised call. + assert [record["id"] for record in data] == ["canonical", "compat-id"] + # What: act across data to perform record; why: the model list renders capability metadata for canonical and alias scenario repeats the body only while or for the loop header admits an iteration. + for record in data: + # What: assert that record name equals canonical model; why: this assertion protects the model list renders capability metadata for canonical and alias regression after the test's arranged inputs and exercised call. + assert record["name"] == "Canonical Model" + # What: assert that record description equals public description; why: this assertion protects the model list renders capability metadata for canonical and alias regression after the test's arranged inputs and exercised call. + assert record["description"] == "Public description" + # What: assert the expected record architecture == outcome; why: test router test model list renders capability metadata for canonical and alias protects its regression by requiring this observable result after the exercised behavior. + assert record["architecture"] == { + # What: arrange input modalities text for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + "input_modalities": ["text"], + # What: arrange output modalities text for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + "output_modalities": ["text"], + # What: arrange modality text text for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + "modality": "text->text", + # What: arrange the grouped source fragment for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that record capabilities equals function calling true; why: this assertion protects the model list renders capability metadata for canonical and alias regression after the test's arranged inputs and exercised call. + assert record["capabilities"] == {"function_calling": True} + # What: assert that record supported parameters equals tools tool choice; why: this assertion protects the model list renders capability metadata for canonical and alias regression after the test's arranged inputs and exercised call. + assert record["supported_parameters"] == ["tools", "tool_choice"] + # What: assert that record context length equals 32768; why: this assertion protects the model list renders capability metadata for canonical and alias regression after the test's arranged inputs and exercised call. + assert record["context_length"] == 32768 + # What: assert that record context window equals 32768; why: this assertion protects the model list renders capability metadata for canonical and alias regression after the test's arranged inputs and exercised call. + assert record["context_window"] == 32768 + # What: assert the expected data 0 meta == outcome; why: test router test model list renders capability metadata for canonical and alias protects its regression by requiring this observable result after the exercised behavior. + assert data[0]["meta"] == { + # What: arrange n ctx 32768 for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + "n_ctx": 32768, + # What: arrange freetoken for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + "freetoken": { + # What: arrange aliases compat id custom remain type model for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + "aliases": ["compat-id"], "custom": "remain", "type": "model", + # What: arrange the grouped source fragment for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + }, + # What: arrange the grouped source fragment for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert the expected data 1 meta == outcome; why: test router test model list renders capability metadata for canonical and alias protects its regression by requiring this observable result after the exercised behavior. + assert data[1]["meta"] == { + # What: arrange n ctx 32768 for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + "n_ctx": 32768, + # What: arrange freetoken for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + "freetoken": { + # What: arrange custom remain modelID canonical type alias for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + "custom": "remain", "modelID": "canonical", "type": "alias", + # What: arrange the grouped source fragment for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + }, + # What: arrange the grouped source fragment for the scenario; why: test router test model list renders capability metadata for canonical and alias requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that private gguf is absent from str data; why: this assertion protects the model list renders capability metadata for canonical and alias regression after the test's arranged inputs and exercised call. + assert "private.gguf" not in str(data) + + +# What: define the test_model_list_omits_empty_capability_metadata test around local fixtures; why: this test groups the arrange, act, and assertions that protect the model list omits empty capability metadata outcome. +def test_model_list_omits_empty_capability_metadata(): + # What: act by calling Manager and capture manager; why: the model list omits empty capability metadata test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the model list omits empty capability metadata test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({"plain": ModelProfile("plain", "private.gguf", ())}) + # What: act by calling RoutingCoordinator and capture router; why: the model list omits empty capability metadata test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_model_list_omits_empty_capability_metadata releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the model list omits empty capability metadata test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_model_list_omits_empty_capability_metadata; why: test_model_list_omits_empty_capability_metadata consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the model list omits empty capability metadata scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_model_list_omits_empty_capability_metadata groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.json and capture record; why: the model list omits empty capability metadata test asserts the response, state, or failure produced by this call. + record = TestClient(app).get("/v1/models").json()["data"][0] + + # What: assert the expected not outcome; why: test router test model list omits empty capability metadata protects its regression by requiring this observable result after the exercised behavior. + assert not { + # What: arrange architecture capabilities supported parameters context length for the scenario; why: test router test model list omits empty capability metadata requires this concrete input or helper state before exercising the behavior under test. + "architecture", "capabilities", "supported_parameters", "context_length", + # What: arrange context window for the scenario; why: test router test model list omits empty capability metadata requires this concrete input or helper state before exercising the behavior under test. + "context_window", + # What: arrange intersection record for the scenario; why: test router test model list omits empty capability metadata requires this concrete input or helper state before exercising the behavior under test. + }.intersection(record) + # What: assert that record meta equals freetoken type model; why: this assertion protects the model list omits empty capability metadata regression after the test's arranged inputs and exercised call. + assert record["meta"] == {"freetoken": {"type": "model"}} + + +# What: define the test_openai_model_list_reports_canonical_and_alias_loaded_while_activating test around local fixtures; why: this test groups the arrange, act, and assertions that protect the openai model list reports canonical and alias loaded while activating outcome. +def test_openai_model_list_reports_canonical_and_alias_loaded_while_activating(): + # What: act by calling Manager and capture manager; why: the openai model list reports canonical and alias loaded while activating test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling threading.Event and capture activation started; why: the openai model list reports canonical and alias loaded while activating test asserts the response, state, or failure produced by this call. + activation_started = threading.Event() + # What: act by calling threading.Event and capture finish activation; why: the openai model list reports canonical and alias loaded while activating test asserts the response, state, or failure produced by this call. + finish_activation = threading.Event() + # What: act by calling ModelCatalog and capture catalog doc; why: the openai model list reports canonical and alias loaded while activating test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the canonical field as model profile and canonical and shared and gguf and compat id; why: test_openai_model_list_reports_canonical_and_alias_loaded_while_activating carries canonical through catalog doc into router routing coordinator manager catalog doc object ready fn blocking ready. + {"canonical": ModelProfile( + # What: arrange aliases to ModelProfile; why: the openai model list reports canonical and alias loaded while activating scenario binds this compat id value to ModelProfile's aliases input. + "canonical", "shared.gguf", (), aliases=("compat-id",) + # What: arrange the catalog_doc mapping with canonical; why: test_openai_model_list_reports_canonical_and_alias_loaded_while_activating groups the supplied clauses as one catalog_doc mapping before its value is consumed. + )}, + # What: arrange settings to RouterSettings; why: the openai model list reports canonical and alias loaded while activating scenario binds this router settings and true value to RouterSettings's settings input. + settings=RouterSettings(include_aliases_in_list=True), + # What: arrange the ModelCatalog call with settings; why: test_openai_model_list_reports_canonical_and_alias_loaded_while_activating groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + + # What: define the blocking_ready test helper around manager and probe and pid and port and timeout s; why: the openai model list reports canonical and alias loaded while activating scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def blocking_ready(manager, probe, *, pid, port, timeout_s): + # What: act by calling activation_started.set with the declared inputs; why: the openai model list reports canonical and alias loaded while activating scenario observes the activation_started.set return value during assert finish activation wait. + activation_started.set() + # What: assert that finish activation wait 2; why: this assertion protects the openai model list reports canonical and alias loaded while activating regression after the test's arranged inputs and exercised call. + assert finish_activation.wait(2) + # What: arrange the ready field as true; why: blocking_ready carries ready into return {"ready": True, "health": {"status": "ok"}}. + return {"ready": True, "health": {"status": "ok"}} + + # What: act by calling RoutingCoordinator and capture router; why: the openai model list reports canonical and alias loaded while activating test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=blocking_ready) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_openai_model_list_reports_canonical_and_alias_loaded_while_activating releases this resource or lock after app build app on both success and failure paths. + with ( + # What: act by calling ThreadPoolExecutor with 1; why: the openai model list reports canonical and alias loaded while activating scenario observes the ThreadPoolExecutor return value during thread pool executor as lifecycle. + ThreadPoolExecutor(1) as activation, + # What: act by calling ThreadPoolExecutor with 1; why: the openai model list reports canonical and alias loaded while activating scenario observes the ThreadPoolExecutor return value during thread pool executor as proxy. + ThreadPoolExecutor(1) as lifecycle, + # What: act by calling ThreadPoolExecutor with 1; why: the openai model list reports canonical and alias loaded while activating scenario observes the ThreadPoolExecutor return value while evaluating ThreadPoolExecutor(1) as proxy. + ThreadPoolExecutor(1) as proxy, + # What: arrange the grouped source fragment for the scenario; why: test openai model list reports canonical and alias loaded while activating requires this concrete input or helper state before exercising the behavior under test. + ): + # What: act by calling build_app and capture app; why: the openai model list reports canonical and alias loaded while activating test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_openai_model_list_reports_canonical_and_alias_loaded_while_activating; why: test_openai_model_list_reports_canonical_and_alias_loaded_while_activating consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the openai model list reports canonical and alias loaded while activating scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_openai_model_list_reports_canonical_and_alias_loaded_while_activating groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the openai model list reports canonical and alias loaded while activating test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling activation.submit and capture future; why: the openai model list reports canonical and alias loaded while activating test asserts the response, state, or failure produced by this call. + future = activation.submit(router.acquire, "compat-id") + # What: assert that activation started wait 2; why: this assertion protects the openai model list reports canonical and alias loaded while activating regression after the test's arranged inputs and exercised call. + assert activation_started.wait(2) + # What: establish the handler boundary for the protected operation; why: test_openai_model_list_reports_canonical_and_alias_loaded_while_activating routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: act by calling operation.json and capture status; why: the openai model list reports canonical and alias loaded while activating test asserts the response, state, or failure produced by this call. + status = client.get("/router/status").json() + # What: act by calling operation.json and capture listed; why: the openai model list reports canonical and alias loaded while activating test asserts the response, state, or failure produced by this call. + listed = client.get("/v1/models").json()["data"] + # What: run finish activation set on every exit path; why: test_openai_model_list_reports_canonical_and_alias_loaded_while_activating performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: act by calling finish_activation.set with the declared inputs; why: the openai model list reports canonical and alias loaded while activating scenario observes the finish_activation.set return value during lease future result timeout. + finish_activation.set() + # What: act by calling future.result and capture lease; why: the openai model list reports canonical and alias loaded while activating test asserts the response, state, or failure produced by this call. + lease = future.result(timeout=2) + # What: act by calling lease.release with the declared inputs; why: the openai model list reports canonical and alias loaded while activating scenario observes the lease.release return value during assert status active profile is. + lease.release() + + # What: assert that status active profile is group delimiter; why: this assertion protects the openai model list reports canonical and alias loaded while activating regression after the test's arranged inputs and exercised call. + assert status["activeProfile"] is None + # What: assert that status activating profile equals canonical; why: this assertion protects the openai model list reports canonical and alias loaded while activating regression after the test's arranged inputs and exercised call. + assert status["activatingProfile"] == "canonical" + # What: assert the expected item id item status value for item in listed == outcome; why: test router test openai model list reports canonical and alias loaded while activating protects its regression by requiring this observable result after the exercised behavior. + assert {item["id"]: item["status"]["value"] for item in listed} == { + # What: arrange canonical loaded compat id loaded for the scenario; why: test router test openai model list reports canonical and alias loaded while activating requires this concrete input or helper state before exercising the behavior under test. + "canonical": "loaded", "compat-id": "loaded", + # What: arrange the grouped source fragment for the scenario; why: test router test openai model list reports canonical and alias loaded while activating requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that router status activating profile is group delimiter; why: this assertion protects the openai model list reports canonical and alias loaded while activating regression after the test's arranged inputs and exercised call. + assert router.status()["activatingProfile"] is None + + +# What: define the test_model_listing_does_not_claim_loaded_before_manager_owns_starting_child test around local fixtures; why: this test groups the arrange, act, and assertions that protect the model listing does not claim loaded before manager owns starting child outcome. +def test_model_listing_does_not_claim_loaded_before_manager_owns_starting_child(): + # What: act by calling threading.Event and capture start entered; why: the model listing does not claim loaded before manager owns starting child test asserts the response, state, or failure produced by this call. + start_entered = threading.Event() + # What: act by calling threading.Event and capture finish start; why: the model listing does not claim loaded before manager owns starting child test asserts the response, state, or failure produced by this call. + finish_start = threading.Event() + + # What: define BlockingStartManager as the owner of start; why: daemon callers use this class boundary so those methods share one blocking start manager state invariant. + class BlockingStartManager(Manager): + # What: define the start test helper around model and port and args; why: the model listing does not claim loaded before manager owns starting child scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def start(self, model, port, args): + # What: act by calling start_entered.set with the declared inputs; why: the model listing does not claim loaded before manager owns starting child scenario observes the start_entered.set return value during assert finish start wait. + start_entered.set() + # What: assert that finish start wait 2; why: this assertion protects the model listing does not claim loaded before manager owns starting child regression after the test's arranged inputs and exercised call. + assert finish_start.wait(2) + # What: return start and model and port and args from the start test helper; why: the model listing does not claim loaded before manager owns starting child scenario uses this helper result in its subsequent act or assertion. + return super().start(model, port, args) + + # What: act by calling BlockingStartManager and capture manager; why: the model listing does not claim loaded before manager owns starting child test asserts the response, state, or failure produced by this call. + manager = BlockingStartManager() + # What: act by calling RoutingCoordinator and capture router; why: the model listing does not claim loaded before manager owns starting child test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: enter the ThreadPoolExecutor managed context before future activation submit router acquire low; why: test_model_listing_does_not_claim_loaded_before_manager_owns_starting_child releases this resource or lock after future activation submit router acquire low on both success and failure paths. + with ThreadPoolExecutor(1) as activation: + # What: act by calling activation.submit and capture future; why: the model listing does not claim loaded before manager owns starting child test asserts the response, state, or failure produced by this call. + future = activation.submit(router.acquire, "low") + # What: assert that start entered wait 2; why: this assertion protects the model listing does not claim loaded before manager owns starting child regression after the test's arranged inputs and exercised call. + assert start_entered.wait(2) + # What: establish the handler boundary for the protected operation; why: test_model_listing_does_not_claim_loaded_before_manager_owns_starting_child routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: act by calling router.model_listing_snapshot and capture and loaded profiles; why: the model listing does not claim loaded before manager owns starting child test asserts the response, state, or failure produced by this call. + _, loaded_profiles = router.model_listing_snapshot() + # What: act by calling router.status and capture status; why: the model listing does not claim loaded before manager owns starting child test asserts the response, state, or failure produced by this call. + status = router.status() + # What: run finish start set on every exit path; why: test_model_listing_does_not_claim_loaded_before_manager_owns_starting_child performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: act by calling finish_start.set with the declared inputs; why: the model listing does not claim loaded before manager owns starting child scenario observes the finish_start.set return value during lease future result timeout. + finish_start.set() + # What: act by calling future.result and capture lease; why: the model listing does not claim loaded before manager owns starting child test asserts the response, state, or failure produced by this call. + lease = future.result(timeout=2) + # What: act by calling lease.release with the declared inputs; why: the model listing does not claim loaded before manager owns starting child scenario observes the lease.release return value during assert loaded profiles frozenset. + lease.release() + + # What: assert that loaded profiles equals frozenset; why: this assertion protects the model listing does not claim loaded before manager owns starting child regression after the test's arranged inputs and exercised call. + assert loaded_profiles == frozenset() + # What: assert that status activating profile equals low; why: this assertion protects the model listing does not claim loaded before manager owns starting child regression after the test's arranged inputs and exercised call. + assert status["activatingProfile"] == "low" + # What: assert that router model listing snapshot 1 equals frozenset low; why: this assertion protects the model listing does not claim loaded before manager owns starting child regression after the test's arranged inputs and exercised call. + assert router.model_listing_snapshot()[1] == frozenset({"low"}) + + +# What: define the test_browser_cors_preflight_is_side_effect_free_and_sanitizes_headers test around local fixtures; why: this test groups the arrange, act, and assertions that protect the browser cors preflight is side effect free and sanitizes headers outcome. +def test_browser_cors_preflight_is_side_effect_free_and_sanitizes_headers(): + # What: act by calling Manager and capture manager; why: the browser cors preflight is side effect free and sanitizes headers test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_browser_cors_preflight_is_side_effect_free_and_sanitizes_headers releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the browser cors preflight is side effect free and sanitizes headers test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_browser_cors_preflight_is_side_effect_free_and_sanitizes_headers; why: test_browser_cors_preflight_is_side_effect_free_and_sanitizes_headers consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to catalog; why: the browser cors preflight is side effect free and sanitizes headers scenario binds this lifecycle value to catalog's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog(), token="control-secret", + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_browser_cors_preflight_is_side_effect_free_and_sanitizes_headers groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the browser cors preflight is side effect free and sanitizes headers test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling client.options and capture preflight; why: the browser cors preflight is side effect free and sanitizes headers test asserts the response, state, or failure produced by this call. + preflight = client.options( + # What: arrange the does not exist portion of preflight; why: the browser cors preflight is side effect free and sanitizes headers scenario uses this clause to evaluate preflight as one grouped value. + "/does-not-exist", + # What: arrange the access control request headers field as content type and bad and header and x ft token; why: test_browser_cors_preflight_is_side_effect_free_and_sanitizes_headers carries access control request headers through preflight into assert preflight status code equals 204. + headers={"Access-Control-Request-Headers": "Content-Type, bad header, X-FT-Token"}, + # What: arrange the client.options call with headers; why: test_browser_cors_preflight_is_side_effect_free_and_sanitizes_headers groups the supplied clauses as one client.options call before its value is consumed. + ) + # What: act by calling client.options and capture default preflight; why: the browser cors preflight is side effect free and sanitizes headers test asserts the response, state, or failure produced by this call. + default_preflight = client.options("/v1/chat/completions") + + # What: assert that preflight status code equals 204; why: this assertion protects the browser cors preflight is side effect free and sanitizes headers regression after the test's arranged inputs and exercised call. + assert preflight.status_code == 204 + # What: assert that preflight headers access control allow origin equals group delimiter; why: this assertion protects the browser cors preflight is side effect free and sanitizes headers regression after the test's arranged inputs and exercised call. + assert preflight.headers["access-control-allow-origin"] == "*" + # What: assert the expected preflight headers access control allow methods == outcome; why: test router test browser cors preflight is side effect free and sanitizes headers protects its regression by requiring this observable result after the exercised behavior. + assert preflight.headers["access-control-allow-methods"] == ( + # What: arrange GET POST PUT PATCH DELETE OPTIONS for the scenario; why: test router test browser cors preflight is side effect free and sanitizes headers requires this concrete input or helper state before exercising the behavior under test. + "GET, POST, PUT, PATCH, DELETE, OPTIONS" + # What: arrange the grouped source fragment for the scenario; why: test router test browser cors preflight is side effect free and sanitizes headers requires this concrete input or helper state before exercising the behavior under test. + ) + # What: assert that preflight headers access control allow headers equals content type x ft token; why: this assertion protects the browser cors preflight is side effect free and sanitizes headers regression after the test's arranged inputs and exercised call. + assert preflight.headers["access-control-allow-headers"] == "Content-Type, X-FT-Token" + # What: assert that preflight headers access control max age equals 86400; why: this assertion protects the browser cors preflight is side effect free and sanitizes headers regression after the test's arranged inputs and exercised call. + assert preflight.headers["access-control-max-age"] == "86400" + # What: assert the expected default preflight headers access control allow headers == outcome; why: test router test browser cors preflight is side effect free and sanitizes headers protects its regression by requiring this observable result after the exercised behavior. + assert default_preflight.headers["access-control-allow-headers"] == ( + # What: arrange Content Type Authorization Accept X Requested With for the scenario; why: test router test browser cors preflight is side effect free and sanitizes headers requires this concrete input or helper state before exercising the behavior under test. + "Content-Type, Authorization, Accept, X-Requested-With" + # What: arrange the grouped source fragment for the scenario; why: test router test browser cors preflight is side effect free and sanitizes headers requires this concrete input or helper state before exercising the behavior under test. + ) + # What: assert that manager calls equals group delimiter; why: this assertion protects the browser cors preflight is side effect free and sanitizes headers regression after the test's arranged inputs and exercised call. + assert manager.calls == [] + + +# What: define the test_models_alias_matches_public_listing_and_keeps_control_auth_separate test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the models alias matches public listing and keeps control auth separate outcome. +def test_models_alias_matches_public_listing_and_keeps_control_auth_separate(monkeypatch): + # What: arrange the exact monkeypatch setattr freetoken daemon app time time lambda fixture fragment; why: the models alias matches public listing and keeps control auth separate scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.time.time", lambda: 1234567890 before asse. + monkeypatch.setattr("freetoken.daemon.app.time.time", lambda: 1234567890) + # What: act by calling Manager and capture manager; why: the models alias matches public listing and keeps control auth separate test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the models alias matches public listing and keeps control auth separate test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the visible field as model profile and visible and private and gguf; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate carries visible through catalog doc into lifecycle pool lifecycle proxy pool proxy catalog catalog doc. + {"visible": ModelProfile("visible", "private.gguf", ())}, + # What: arrange settings to RouterSettings; why: the models alias matches public listing and keeps control auth separate scenario binds this router settings and router key value to RouterSettings's settings input. + settings=RouterSettings(api_keys=("router-key",)), + # What: arrange the ModelCatalog call with settings; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the models alias matches public listing and keeps control auth separate test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_models_alias_matches_public_listing_and_keeps_control_auth_separate; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the models alias matches public listing and keeps control auth separate scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, + # What: arrange token to build_app; why: the models alias matches public listing and keeps control auth separate scenario binds this control secret value to build_app's token input. + token="control-secret", + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the models alias matches public listing and keeps control auth separate test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling client.get and capture denied; why: the models alias matches public listing and keeps control auth separate test asserts the response, state, or failure produced by this call. + denied = client.get("/v1/models", headers={"Origin": "https://client.example"}) + # What: act by calling client.get and capture listed; why: the models alias matches public listing and keeps control auth separate test asserts the response, state, or failure produced by this call. + listed = client.get( + # What: arrange the v1 models portion of listed; why: the models alias matches public listing and keeps control auth separate scenario uses this clause to evaluate listed as one grouped value. + "/v1/models", + # What: arrange the origin field as https and client and example; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate carries origin through listed into assert listed status code equals 200. + headers={"Origin": "https://client.example", "Authorization": "Bearer router-key"}, + # What: arrange the client.get call with headers; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate groups the supplied clauses as one client.get call before its value is consumed. + ) + # What: act by calling client.get and capture alias denied; why: the models alias matches public listing and keeps control auth separate test asserts the response, state, or failure produced by this call. + alias_denied = client.get("/models", headers={"X-FT-Token": "control-secret"}) + # What: act by calling client.get and capture alias; why: the models alias matches public listing and keeps control auth separate test asserts the response, state, or failure produced by this call. + alias = client.get( + # What: arrange the models portion of alias; why: the models alias matches public listing and keeps control auth separate scenario uses this clause to evaluate alias as one grouped value. + "/models", + # What: arrange the origin field as https and client and example; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate carries origin through alias into assert alias status code equals 200. + headers={"Origin": "https://client.example", "Authorization": "Bearer router-key"}, + # What: arrange the client.get call with headers; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate groups the supplied clauses as one client.get call before its value is consumed. + ) + # What: act by calling client.get and capture profiles; why: the models alias matches public listing and keeps control auth separate test asserts the response, state, or failure produced by this call. + profiles = client.get( + # What: arrange the x ft token field as control secret; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate carries x ft token through profiles into router profiles headers authorization bearer router key. + "/router/profiles", headers={"X-FT-Token": "control-secret"} + # What: arrange the client.get call with headers; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate groups the supplied clauses as one client.get call before its value is consumed. + ) + # What: act by calling client.get and capture profiles denied; why: the models alias matches public listing and keeps control auth separate test asserts the response, state, or failure produced by this call. + profiles_denied = client.get( + # What: arrange the authorization field as bearer and router key; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate carries authorization through profiles denied into assert profiles denied status code equals 401. + "/router/profiles", headers={"Authorization": "Bearer router-key"} + # What: arrange the client.get call with headers; why: test_models_alias_matches_public_listing_and_keeps_control_auth_separate groups the supplied clauses as one client.get call before its value is consumed. + ) + + # What: assert that denied status code equals 401; why: this assertion protects the models alias matches public listing and keeps control auth separate regression after the test's arranged inputs and exercised call. + assert denied.status_code == 401 + # What: assert that listed status code equals 200; why: this assertion protects the models alias matches public listing and keeps control auth separate regression after the test's arranged inputs and exercised call. + assert listed.status_code == 200 + # What: assert that listed headers access control allow origin equals https client example; why: this assertion protects the models alias matches public listing and keeps control auth separate regression after the test's arranged inputs and exercised call. + assert listed.headers["access-control-allow-origin"] == "https://client.example" + # What: assert that item id for item in listed json equals visible; why: this assertion protects the models alias matches public listing and keeps control auth separate regression after the test's arranged inputs and exercised call. + assert [item["id"] for item in listed.json()["data"]] == ["visible"] + # What: assert that alias denied status code equals 401; why: this assertion protects the models alias matches public listing and keeps control auth separate regression after the test's arranged inputs and exercised call. + assert alias_denied.status_code == 401 + # What: assert that alias status code equals 200; why: this assertion protects the models alias matches public listing and keeps control auth separate regression after the test's arranged inputs and exercised call. + assert alias.status_code == 200 + # What: assert that alias json equals listed json; why: this assertion protects the models alias matches public listing and keeps control auth separate regression after the test's arranged inputs and exercised call. + assert alias.json() == listed.json() + # What: assert that alias headers access control allow origin equals https client example; why: this assertion protects the models alias matches public listing and keeps control auth separate regression after the test's arranged inputs and exercised call. + assert alias.headers["access-control-allow-origin"] == "https://client.example" + # What: assert that profiles status code equals 200; why: this assertion protects the models alias matches public listing and keeps control auth separate regression after the test's arranged inputs and exercised call. + assert profiles.status_code == 200 + # What: assert that profiles json data 0 model equals private gguf; why: this assertion protects the models alias matches public listing and keeps control auth separate regression after the test's arranged inputs and exercised call. + assert profiles.json()["data"][0]["model"] == "private.gguf" + # What: assert that profiles denied status code equals 401; why: this assertion protects the models alias matches public listing and keeps control auth separate regression after the test's arranged inputs and exercised call. + assert profiles_denied.status_code == 401 + + +# What: define the test_explicit_cancel_while_upstream_connects_closes_result_and_releases_lease test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the explicit cancel while upstream connects closes result and releases lease outcome. +def test_explicit_cancel_while_upstream_connects_closes_result_and_releases_lease(monkeypatch): + # What: act by calling Manager and capture manager; why: the explicit cancel while upstream connects closes result and releases lease test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the explicit cancel while upstream connects closes result and releases lease test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({"low": ModelProfile("low", "low.gguf", ())}) + # What: act by calling RoutingCoordinator and capture router; why: the explicit cancel while upstream connects closes result and releases lease test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: act by calling threading.Event and capture connecting; why: the explicit cancel while upstream connects closes result and releases lease test asserts the response, state, or failure produced by this call. + connecting = threading.Event() + # What: act by calling threading.Event and capture finish connect; why: the explicit cancel while upstream connects closes result and releases lease test asserts the response, state, or failure produced by this call. + finish_connect = threading.Event() + # What: act by calling BytesIO and capture raw; why: the explicit cancel while upstream connects closes result and releases lease test asserts the response, state, or failure produced by this call. + raw = BytesIO(b"must not stream") + + # What: define the upstream test helper around captured fixture state; why: the explicit cancel while upstream connects closes result and releases lease scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: act by calling connecting.set with the declared inputs; why: the explicit cancel while upstream connects closes result and releases lease scenario observes the connecting.set return value during assert finish connect wait. + connecting.set() + # What: assert that finish connect wait 2; why: this assertion protects the explicit cancel while upstream connects closes result and releases lease regression after the test's arranged inputs and exercised call. + assert finish_connect.wait(2) + # What: arrange the helper response as UpstreamResponse 200 Content Type text event stream raw; why: test explicit cancel while upstream connects closes feeds this result into the behavior whose outcome is asserted. + return UpstreamResponse(200, {"Content-Type": "text/event-stream"}, raw) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the explicit cancel while upstream connects closes result and releases lease scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) befor. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_explicit_cancel_while_upstream_connects_closes_result_and_releases_lease releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the explicit cancel while upstream connects closes result and releases lease test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_explicit_cancel_while_upstream_connects_closes_result_and_releases_lease; why: test_explicit_cancel_while_upstream_connects_closes_result_and_releases_lease consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the explicit cancel while upstream connects closes result and releases lease scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_explicit_cancel_while_upstream_connects_closes_result_and_releases_lease groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the explicit cancel while upstream connects closes result and releases lease test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: arrange response as the fixture input; why: the explicit cancel while upstream connects closes result and releases lease test consumes this named precondition before exercising the behavior. + response = [] + # What: act by calling threading.Thread and capture thread; why: the explicit cancel while upstream connects closes result and releases lease test asserts the response, state, or failure produced by this call. + thread = threading.Thread(target=lambda: response.append(client.post( + # What: arrange the model field as low; why: test_explicit_cancel_while_upstream_connects_closes_result_and_releases_lease sends this field through thread so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "low"}, + # What: arrange the x ft request id field as cancel during connect; why: test_explicit_cancel_while_upstream_connects_closes_result_and_releases_lease carries x ft request id through thread into thread start. + headers={"X-FT-Request-ID": "cancel-during-connect"}, + # What: arrange the threading.Thread call with target; why: test_explicit_cancel_while_upstream_connects_closes_result_and_releases_lease groups the supplied clauses as one threading.Thread call before its value is consumed. + ))) + # What: act by calling thread.start with the declared inputs; why: the explicit cancel while upstream connects closes result and releases lease scenario observes the thread.start return value during assert connecting wait. + thread.start() + # What: assert that connecting wait 1; why: this assertion protects the explicit cancel while upstream connects closes result and releases lease regression after the test's arranged inputs and exercised call. + assert connecting.wait(1) + # What: assert the expected client get router requests json data == outcome; why: test router test explicit cancel while upstream connects closes result and releases lease protects its regression by requiring this observable result after the exercised behavior. + assert client.get("/router/requests").json()["data"] == [ + # What: arrange id cancel during connect profile low for the scenario; why: test router test explicit cancel while upstream connects closes result and releases lease requires this concrete input or helper state before exercising the behavior under test. + {"id": "cancel-during-connect", "profile": "low"} + # What: arrange the grouped source fragment for the scenario; why: test router test explicit cancel while upstream connects closes result and releases lease requires this concrete input or helper state before exercising the behavior under test. + ] + # What: act by calling client.post and capture cancelled; why: the explicit cancel while upstream connects closes result and releases lease test asserts the response, state, or failure produced by this call. + cancelled = client.post("/router/requests/cancel-during-connect/cancel") + # What: assert that cancelled json equals cancelled true id cancel during connect; why: this assertion protects the explicit cancel while upstream connects closes result and releases lease regression after the test's arranged inputs and exercised call. + assert cancelled.json() == {"cancelled": True, "id": "cancel-during-connect"} + # What: act by calling finish_connect.set with the declared inputs; why: the explicit cancel while upstream connects closes result and releases lease scenario observes the finish_connect.set return value during thread join. + finish_connect.set() + # What: act by calling thread.join with 2; why: the explicit cancel while upstream connects closes result and releases lease scenario observes the thread.join return value during assert not thread is alive. + thread.join(2) + # What: assert that thread is alive is false; why: this assertion protects the explicit cancel while upstream connects closes result and releases lease regression after the test's arranged inputs and exercised call. + assert not thread.is_alive() + + # What: assert that response 0 status code equals 409; why: this assertion protects the explicit cancel while upstream connects closes result and releases lease regression after the test's arranged inputs and exercised call. + assert response[0].status_code == 409 + # What: assert that response 0 json error type equals request cancelled; why: this assertion protects the explicit cancel while upstream connects closes result and releases lease regression after the test's arranged inputs and exercised call. + assert response[0].json()["error"]["type"] == "request_cancelled" + # What: assert that raw closed; why: this assertion protects the explicit cancel while upstream connects closes result and releases lease regression after the test's arranged inputs and exercised call. + assert raw.closed + # What: assert that router status active requests equals 0; why: this assertion protects the explicit cancel while upstream connects closes result and releases lease regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + # What: assert that router status admissions equals 1; why: this assertion protects the explicit cancel while upstream connects closes result and releases lease regression after the test's arranged inputs and exercised call. + assert router.status()["admissions"] == 1 + # What: assert that router status cancellations equals 1; why: this assertion protects the explicit cancel while upstream connects closes result and releases lease regression after the test's arranged inputs and exercised call. + assert router.status()["cancellations"] == 1 + # What: assert that router status terminal streams equals 0; why: this assertion protects the explicit cancel while upstream connects closes result and releases lease regression after the test's arranged inputs and exercised call. + assert router.status()["terminalStreams"] == 0 + + +# What: define the test_disconnect_while_upstream_connects_closes_orphaned_result test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the disconnect while upstream connects closes orphaned result outcome. +def test_disconnect_while_upstream_connects_closes_orphaned_result(monkeypatch): + # What: act by calling Manager and capture manager; why: the disconnect while upstream connects closes orphaned result test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the disconnect while upstream connects closes orphaned result test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({"low": ModelProfile("low", "low.gguf", ())}) + # What: act by calling RoutingCoordinator and capture router; why: the disconnect while upstream connects closes orphaned result test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: act by calling threading.Event and capture connecting; why: the disconnect while upstream connects closes orphaned result test asserts the response, state, or failure produced by this call. + connecting = threading.Event() + # What: act by calling threading.Event and capture finish connect; why: the disconnect while upstream connects closes orphaned result test asserts the response, state, or failure produced by this call. + finish_connect = threading.Event() + # What: act by calling BytesIO and capture raw; why: the disconnect while upstream connects closes orphaned result test asserts the response, state, or failure produced by this call. + raw = BytesIO(b"must not stream") + + # What: define the upstream test helper around captured fixture state; why: the disconnect while upstream connects closes orphaned result scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: act by calling connecting.set with the declared inputs; why: the disconnect while upstream connects closes orphaned result scenario observes the connecting.set return value during assert finish connect wait. + connecting.set() + # What: assert that finish connect wait 2; why: this assertion protects the disconnect while upstream connects closes orphaned result regression after the test's arranged inputs and exercised call. + assert finish_connect.wait(2) + # What: arrange the helper response as UpstreamResponse 200 Content Type text event stream raw; why: test router test disconnect while upstream connects closes orphaned result feeds this result into the behavior whose outcome is asserted. + return UpstreamResponse(200, {"Content-Type": "text/event-stream"}, raw) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the disconnect while upstream connects closes orphaned result scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) before asserting its. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + + # What: define the scenario test helper around app; why: the disconnect while upstream connects closes orphaned result scenario calls this helper to produce or observe the exact behavior checked by its assertions. + async def scenario(app): + # What: act by calling httpx.ASGITransport and capture transport; why: the disconnect while upstream connects closes orphaned result test asserts the response, state, or failure produced by this call. + transport = httpx.ASGITransport(app=app) + # What: arrange async with httpx AsyncClient transport transport base url http test as client for the scenario; why: test router test requires this concrete input or helper state before exercising the behavior under test. + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + # What: act by calling asyncio.create_task and capture request; why: the disconnect while upstream connects closes orphaned result test asserts the response, state, or failure produced by this call. + request = asyncio.create_task(client.post( + # What: arrange the model field as low; why: scenario sends this field through request so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "low"}, + # What: arrange the x ft request id field as disconnect during connect; why: scenario carries x ft request id through request into request cancel. + headers={"X-FT-Request-ID": "disconnect-during-connect"}, + # What: arrange the asyncio.create_task call with post; why: scenario groups the supplied clauses as one asyncio.create_task call before its value is consumed. + )) + # What: act across range to perform is set and connecting; why: the disconnect while upstream connects closes orphaned result scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on is set and connecting before the computed value; why: the disconnect while upstream connects closes orphaned result scenario admits the computed value only for this predicate and excludes the opposite state. + if connecting.is_set(): + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the disconnect while upstream connects closes orphaned result scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the disconnect while upstream connects closes orphaned result scenario observes the asyncio.sleep return value during assert connecting is set. + await asyncio.sleep(0.01) + # What: assert that connecting is set; why: this assertion protects the disconnect while upstream connects closes orphaned result regression after the test's arranged inputs and exercised call. + assert connecting.is_set() + # What: act by calling request.cancel with the declared inputs; why: the disconnect while upstream connects closes orphaned result scenario observes the request.cancel return value during with pytest raises asyncio cancelled error. + request.cancel() + # What: assert the pytest.raises failure context; why: the disconnect while upstream connects closes orphaned result scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(asyncio.CancelledError): + # What: arrange the await request portion of the enclosing predicate; why: this clause remains in the disconnect while upstream connects closes orphaned result scenario\'s enclosing expression so its grouping and evaluation order stay intact. + await request + # What: assert that router status active requests equals 0; why: this assertion protects the disconnect while upstream connects closes orphaned result regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + # What: assert that await client get router requests json data equals group delimiter; why: this assertion protects the disconnect while upstream connects closes orphaned result regression after the test's arranged inputs and exercised call. + assert (await client.get("/router/requests")).json()["data"] == [] + # What: act by calling finish_connect.set with the declared inputs; why: the disconnect while upstream connects closes orphaned result scenario observes the finish_connect.set return value during for value in range. + finish_connect.set() + # What: act across range to perform closed and raw; why: the disconnect while upstream connects closes orphaned result scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on closed and raw before the computed value; why: the disconnect while upstream connects closes orphaned result scenario admits the computed value only for this predicate and excludes the opposite state. + if raw.closed: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the disconnect while upstream connects closes orphaned result scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the disconnect while upstream connects closes orphaned result scenario observes the asyncio.sleep return value during assert raw closed. + await asyncio.sleep(0.01) + # What: assert that raw closed; why: this assertion protects the disconnect while upstream connects closes orphaned result regression after the test's arranged inputs and exercised call. + assert raw.closed + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_disconnect_while_upstream_connects_closes_orphaned_result releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the disconnect while upstream connects closes orphaned result test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_disconnect_while_upstream_connects_closes_orphaned_result; why: test_disconnect_while_upstream_connects_closes_orphaned_result consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the disconnect while upstream connects closes orphaned result scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_disconnect_while_upstream_connects_closes_orphaned_result groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling asyncio.run with scenario and app; why: the disconnect while upstream connects closes orphaned result scenario observes the asyncio.run return value during assert router status admissions. + asyncio.run(scenario(app)) + + # What: assert that router status admissions equals 1; why: this assertion protects the disconnect while upstream connects closes orphaned result regression after the test's arranged inputs and exercised call. + assert router.status()["admissions"] == 1 + # What: assert that router status cancellations equals 1; why: this assertion protects the disconnect while upstream connects closes orphaned result regression after the test's arranged inputs and exercised call. + assert router.status()["cancellations"] == 1 + # What: assert that router status terminal streams equals 0; why: this assertion protects the disconnect while upstream connects closes orphaned result regression after the test's arranged inputs and exercised call. + assert router.status()["terminalStreams"] == 0 + + +# What: parameterize test_router_inference_and_management_accept_pinned_api_key_forms with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test router inference and management accept pinned api key forms. +@pytest.mark.parametrize( + # What: arrange the headers portion of the enclosing predicate; why: this clause remains in the router inference and management accept pinned api key forms scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "headers", + # What: arrange the grouped source fragment for the scenario; why: test router inference and management accept pinned api key forms requires this concrete input or helper state before exercising the behavior under test. + [ + # What: arrange the authorization field as bearer and key; why: test_router_inference_and_management_accept_pinned_api_key_forms carries authorization into {"Authorization": "Bearer key"}. + {"Authorization": "Bearer key"}, + # What: arrange the authorization field as bearer and key; why: test_router_inference_and_management_accept_pinned_api_key_forms carries authorization into {"Authorization": "bearer key"}. + {"Authorization": "bearer key"}, + # What: arrange the authorization field as decode and b64encode and base64 and basic; why: test_router_inference_and_management_accept_pinned_api_key_forms carries authorization into {"Authorization": "Basic " + base64.b64encode(b"operator:key").decode()}. + {"Authorization": "Basic " + base64.b64encode(b"operator:key").decode()}, + # What: arrange the x api key field as key; why: test_router_inference_and_management_accept_pinned_api_key_forms carries x api key into {"X-Api-Key": "key"}. + {"X-Api-Key": "key"}, + # What: arrange the authorization field as basic and not base64; why: test_router_inference_and_management_accept_pinned_api_key_forms carries authorization into {"Authorization": "Basic !!!not-base64", "X-Api-Key": "key"}. + {"Authorization": "Basic !!!not-base64", "X-Api-Key": "key"}, + # What: arrange the grouped source fragment for the scenario; why: test router inference and management accept pinned api key forms requires this concrete input or helper state before exercising the behavior under test. + ], +# What: arrange the pytest.mark.parametrize call with decode; why: test_router_inference_and_management_accept_pinned_api_key_forms groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +) +# What: define the test_router_inference_and_management_accept_pinned_api_key_forms test around headers; why: this test groups the arrange, act, and assertions that protect the router inference and management accept pinned api key forms outcome. +def test_router_inference_and_management_accept_pinned_api_key_forms(headers): + # What: act by calling Manager and capture manager; why: the router inference and management accept pinned api key forms test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the router inference and management accept pinned api key forms test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf; why: test_router_inference_and_management_accept_pinned_api_key_forms carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": ModelProfile("low", "low.gguf", ())}, + # What: arrange settings to RouterSettings; why: the router inference and management accept pinned api key forms scenario binds this router settings and key value to RouterSettings's settings input. + settings=RouterSettings(api_keys=("key",)), + # What: arrange the ModelCatalog call with settings; why: test_router_inference_and_management_accept_pinned_api_key_forms groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the router inference and management accept pinned api key forms test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_router_inference_and_management_accept_pinned_api_key_forms releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the router inference and management accept pinned api key forms test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_router_inference_and_management_accept_pinned_api_key_forms; why: test_router_inference_and_management_accept_pinned_api_key_forms consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the router inference and management accept pinned api key forms scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_router_inference_and_management_accept_pinned_api_key_forms groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the router inference and management accept pinned api key forms test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling client.post and capture denied; why: the router inference and management accept pinned api key forms test asserts the response, state, or failure produced by this call. + denied = client.post("/v1/chat/completions", json={"model": "low"}) + # What: assert that denied status code equals 401; why: this assertion protects the router inference and management accept pinned api key forms regression after the test's arranged inputs and exercised call. + assert denied.status_code == 401 + # What: assert that denied headers www authenticate equals basic realm freetoken swap; why: this assertion protects the router inference and management accept pinned api key forms regression after the test's arranged inputs and exercised call. + assert denied.headers["www-authenticate"] == 'Basic realm="freetoken-swap"' + # What: assert that client get router status status code equals 401; why: this assertion protects the router inference and management accept pinned api key forms regression after the test's arranged inputs and exercised call. + assert client.get("/router/status").status_code == 401 + # What: act by calling client.get and capture allowed; why: the router inference and management accept pinned api key forms test asserts the response, state, or failure produced by this call. + allowed = client.get("/router/status", headers=headers) + # What: assert that allowed status code equals 200; why: this assertion protects the router inference and management accept pinned api key forms regression after the test's arranged inputs and exercised call. + assert allowed.status_code == 200 + # What: assert that manager calls equals group delimiter; why: this assertion protects the router inference and management accept pinned api key forms regression after the test's arranged inputs and exercised call. + assert manager.calls == [] + + +# What: parameterize test_explicit_authorization_key_takes_precedence_over_x_api_key with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test explicit authorization key takes precedence over x api key. +@pytest.mark.parametrize( + # What: arrange the authorization portion of the enclosing predicate; why: this clause remains in the explicit authorization key takes precedence over x api key scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "authorization", + # What: arrange the grouped source fragment for the scenario; why: test explicit authorization key takes precedence over x api key requires this concrete input or helper state before exercising the behavior under test. + [ + # What: arrange the bearer wrong portion of the enclosing predicate; why: this clause remains in the explicit authorization key takes precedence over x api key scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "Bearer wrong", + # What: act by calling operation.decode with the declared inputs; why: the explicit authorization key takes precedence over x api key scenario observes the operation.decode return value during basic base64 b64encode b operator xff decode. + "Basic " + base64.b64encode(b"operator:wrong").decode(), + # What: act by calling operation.decode with the declared inputs; why: the explicit authorization key takes precedence over x api key scenario observes the operation.decode return value while evaluating "Basic " + base64.b64encode(b"operator:\xff").decode(). + "Basic " + base64.b64encode(b"operator:\xff").decode(), + # What: arrange the grouped source fragment for the scenario; why: test explicit authorization key takes precedence over x api key requires this concrete input or helper state before exercising the behavior under test. + ], +# What: arrange the pytest.mark.parametrize call with decode; why: test_explicit_authorization_key_takes_precedence_over_x_api_key groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +) +# What: define the test_explicit_authorization_key_takes_precedence_over_x_api_key test around authorization; why: this test groups the arrange, act, and assertions that protect the explicit authorization key takes precedence over x api key outcome. +def test_explicit_authorization_key_takes_precedence_over_x_api_key(authorization): + # What: act by calling ModelCatalog and capture catalog doc; why: the explicit authorization key takes precedence over x api key test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf; why: test_explicit_authorization_key_takes_precedence_over_x_api_key carries low through catalog doc into lifecycle pool lifecycle proxy pool proxy catalog catalog doc. + {"low": ModelProfile("low", "low.gguf", ())}, + # What: arrange settings to RouterSettings; why: the explicit authorization key takes precedence over x api key scenario binds this router settings and key value to RouterSettings's settings input. + settings=RouterSettings(api_keys=("key",)), + # What: arrange the ModelCatalog call with settings; why: test_explicit_authorization_key_takes_precedence_over_x_api_key groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling Manager and capture manager; why: the explicit authorization key takes precedence over x api key test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_explicit_authorization_key_takes_precedence_over_x_api_key releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the explicit authorization key takes precedence over x api key test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_explicit_authorization_key_takes_precedence_over_x_api_key; why: test_explicit_authorization_key_takes_precedence_over_x_api_key consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the explicit authorization key takes precedence over x api key scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_explicit_authorization_key_takes_precedence_over_x_api_key groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.get and capture response; why: the explicit authorization key takes precedence over x api key test asserts the response, state, or failure produced by this call. + response = TestClient(app).get( + # What: arrange the router status portion of response; why: the explicit authorization key takes precedence over x api key scenario uses this clause to evaluate response as one grouped value. + "/router/status", + # What: arrange the authorization field as authorization; why: test_explicit_authorization_key_takes_precedence_over_x_api_key carries authorization through response into assert response status code equals 401. + headers={"Authorization": authorization, "X-Api-Key": "key"}, + # What: arrange the operation.get call with headers; why: test_explicit_authorization_key_takes_precedence_over_x_api_key groups the supplied clauses as one operation.get call before its value is consumed. + ) + + # What: assert that response status code equals 401; why: this assertion protects the explicit authorization key takes precedence over x api key regression after the test's arranged inputs and exercised call. + assert response.status_code == 401 + # What: assert that manager calls equals group delimiter; why: this assertion protects the explicit authorization key takes precedence over x api key regression after the test's arranged inputs and exercised call. + assert manager.calls == [] + + +# What: define the test_router_terminates_local_authentication_before_proxying test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the router terminates local authentication before proxying outcome. +def test_router_terminates_local_authentication_before_proxying(monkeypatch): + # What: act by calling Manager and capture manager; why: the router terminates local authentication before proxying test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the router terminates local authentication before proxying test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf; why: test_router_terminates_local_authentication_before_proxying carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": ModelProfile("low", "low.gguf", ())}, + # What: arrange settings to RouterSettings; why: the router terminates local authentication before proxying scenario binds this router settings and router test key value to RouterSettings's settings input. + settings=RouterSettings(api_keys=("router-test-key",)), + # What: arrange the ModelCatalog call with settings; why: test_router_terminates_local_authentication_before_proxying groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the router terminates local authentication before proxying test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange observed as the fixture input; why: the router terminates local authentication before proxying test consumes this named precondition before exercising the behavior. + observed = {} + + # What: define the upstream test helper around captured fixture state; why: the router terminates local authentication before proxying scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: arrange the exact observed update key lower value for key value fixture fragment; why: the router terminates local authentication before proxying scenario feeds this byte-preserved fragment through observed.update({key.lower(): value for key, value in forward_headers(kw before asserting its protocol or. + observed.update({key.lower(): value for key, value in forward_headers(kwargs["headers"]).items()}) + # What: arrange the helper response as UpstreamResponse 200 Content Type application json BytesIO b ok true; why: test router test feeds this result into the behavior whose outcome is asserted. + return UpstreamResponse(200, {"Content-Type": "application/json"}, BytesIO(b'{"ok":true}')) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the router terminates local authentication before proxying scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) before asserting its pr. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_router_terminates_local_authentication_before_proxying releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the router terminates local authentication before proxying test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_router_terminates_local_authentication_before_proxying; why: test_router_terminates_local_authentication_before_proxying consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the router terminates local authentication before proxying scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange token to build_app; why: the router terminates local authentication before proxying scenario binds this daemon control secret value to build_app's token input. + token="daemon-control-secret", + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_router_terminates_local_authentication_before_proxying groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture response; why: the router terminates local authentication before proxying test asserts the response, state, or failure produced by this call. + response = TestClient(app).post( + # What: arrange the v1 messages portion of response; why: the router terminates local authentication before proxying scenario uses this clause to evaluate response as one grouped value. + "/v1/messages", + # What: arrange content to operation.post; why: the router terminates local authentication before proxying scenario binds this the named fixture input value to operation.post's content input. + content=b'{"model":"low","messages":[]}', + # What: arrange headers to operation.post; why: the router terminates local authentication before proxying scenario binds this content type and authorization and x api key and x ft token and x correlation id value to operation.post's headers input. + headers={ + # What: arrange the content type field as application and json; why: test_router_terminates_local_authentication_before_proxying carries content type through response into assert response status code equals 200. + "Content-Type": "application/json", + # What: arrange the authorization field as basic and not base64; why: test_router_terminates_local_authentication_before_proxying carries authorization through response into assert response status code equals 200. + "Authorization": "Basic !!!not-base64", + # What: arrange the x api key field as router test key; why: test_router_terminates_local_authentication_before_proxying carries x api key through response into assert response status code equals 200. + "X-Api-Key": "router-test-key", + # What: arrange the x ft token field as daemon control secret; why: test_router_terminates_local_authentication_before_proxying carries x ft token through response into assert response status code equals 200. + "X-FT-Token": "daemon-control-secret", + # What: arrange the x correlation id field as client safe id; why: test_router_terminates_local_authentication_before_proxying carries x correlation id through response into assert response status code equals 200. + "X-Correlation-ID": "client-safe-id", + # What: arrange the response mapping with content type and authorization and x api key and x ft token and x correlation id; why: test_router_terminates_local_authentication_before_proxying groups the supplied clauses as one response mapping before its value is consumed. + }, + # What: arrange the operation.post call with content and headers; why: test_router_terminates_local_authentication_before_proxying groups the supplied clauses as one operation.post call before its value is consumed. + ) + # What: assert that response status code equals 200; why: this assertion protects the router terminates local authentication before proxying regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + # What: assert that observed x correlation id equals client safe id; why: this assertion protects the router terminates local authentication before proxying regression after the test's arranged inputs and exercised call. + assert observed["x-correlation-id"] == "client-safe-id" + # What: assert that authorization is absent from observed; why: this assertion protects the router terminates local authentication before proxying regression after the test's arranged inputs and exercised call. + assert "authorization" not in observed + # What: assert that x api key is absent from observed; why: this assertion protects the router terminates local authentication before proxying regression after the test's arranged inputs and exercised call. + assert "x-api-key" not in observed + # What: assert that x ft token is absent from observed; why: this assertion protects the router terminates local authentication before proxying regression after the test's arranged inputs and exercised call. + assert "x-ft-token" not in observed + + +# What: define the test_proxy_response_headers_do_not_apply_inbound_credential_filtering test around local fixtures; why: this test groups the arrange, act, and assertions that protect the proxy response headers do not apply inbound credential filtering outcome. +def test_proxy_response_headers_do_not_apply_inbound_credential_filtering(): + # What: Assert assert response headers in test_proxy_response_headers_do_not_apply_inbound_credential_filtering; why: test_proxy_response_headers_do_not_apply_inbound_credential_filtering uses this assert to implement the named assert response headers operation. + assert response_headers( + # What: Assert group delimiter in test_proxy_response_headers_do_not_apply_inbound_credential_filtering; why: test_proxy_response_headers_do_not_apply_inbound_credential_filtering uses this assert to implement the named group delimiter operation. + { + # What: Assert authorization engine challenge metadata in test_proxy_response_headers_do_not_apply_inbound_credential_filtering; why: test_proxy_response_headers_do_not_apply_inbound_credential_filtering uses this assert to implement the named authorization engine challenge metadata operation. + "Authorization": "Engine challenge metadata", + # What: Assert x ft token engine defined response value in test_proxy_response_headers_do_not_apply_inbound_credential_filtering; why: test_proxy_response_headers_do_not_apply_inbound_credential_filtering uses this assert to implement the named x ft token engine defined response value operation. + "X-FT-Token": "engine-defined-response-value", + # What: Assert connection close in test_proxy_response_headers_do_not_apply_inbound_credential_filtering; why: test_proxy_response_headers_do_not_apply_inbound_credential_filtering uses this assert to implement the named connection close operation. + "Connection": "close", + # What: Assert group delimiter in test_proxy_response_headers_do_not_apply_inbound_credential_filtering; why: test_proxy_response_headers_do_not_apply_inbound_credential_filtering uses this assert to implement the named group delimiter operation. + } + # What: Assert equals in test_proxy_response_headers_do_not_apply_inbound_credential_filtering; why: test_proxy_response_headers_do_not_apply_inbound_credential_filtering uses this assert to implement the named equals operation. + ) == { + # What: Assert authorization engine challenge metadata in test_proxy_response_headers_do_not_apply_inbound_credential_filtering; why: test_proxy_response_headers_do_not_apply_inbound_credential_filtering uses this assert to implement the named authorization engine challenge metadata operation. + "Authorization": "Engine challenge metadata", + # What: Assert x ft token engine defined response value in test_proxy_response_headers_do_not_apply_inbound_credential_filtering; why: test_proxy_response_headers_do_not_apply_inbound_credential_filtering uses this assert to implement the named x ft token engine defined response value operation. + "X-FT-Token": "engine-defined-response-value", + # What: Assert group delimiter in test_proxy_response_headers_do_not_apply_inbound_credential_filtering; why: test_proxy_response_headers_do_not_apply_inbound_credential_filtering uses this assert to implement the named group delimiter operation. + } + + +# What: define the test_router_reload_atomically_replaces_a_valid_catalog test around tmp path; why: this test groups the arrange, act, and assertions that protect the router reload atomically replaces a valid catalog outcome. +def test_router_reload_atomically_replaces_a_valid_catalog(tmp_path): + # What: arrange path as tmp path and models and toml; why: the router reload atomically replaces a valid catalog test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text models one nmodel one gguf n encoding fixture fragment; why: the router reload atomically replaces a valid catalog scenario feeds this byte-preserved fragment through path.write_text("[models.one]\nmodel = 'one.gguf'\n", encoding="utf-8") before asserting its protocol or p. + path.write_text("[models.one]\nmodel = 'one.gguf'\n", encoding="utf-8") + # What: act by calling Manager and capture manager; why: the router reload atomically replaces a valid catalog test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog.load and capture catalog doc; why: the router reload atomically replaces a valid catalog test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog.load(str(path)) + # What: act by calling RoutingCoordinator and capture router; why: the router reload atomically replaces a valid catalog test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_router_reload_atomically_replaces_a_valid_catalog releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the router reload atomically replaces a valid catalog test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_router_reload_atomically_replaces_a_valid_catalog; why: test_router_reload_atomically_replaces_a_valid_catalog consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the router reload atomically replaces a valid catalog scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange catalog path to str; why: the router reload atomically replaces a valid catalog scenario binds this str and path value to str's catalog path input. + catalog_path=str(path), + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_router_reload_atomically_replaces_a_valid_catalog groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the router reload atomically replaces a valid catalog test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: arrange the exact path write text models two nmodel two gguf n encoding fixture fragment; why: the router reload atomically replaces a valid catalog scenario feeds this byte-preserved fragment through path.write_text("[models.two]\nmodel = 'two.gguf'\n", encoding="utf-8") before asserting its protocol. + path.write_text("[models.two]\nmodel = 'two.gguf'\n", encoding="utf-8") + # What: act by calling client.post and capture reloaded; why: the router reload atomically replaces a valid catalog test asserts the response, state, or failure produced by this call. + reloaded = client.post("/router/reload") + # What: assert that reloaded status code equals 200; why: this assertion protects the router reload atomically replaces a valid catalog regression after the test's arranged inputs and exercised call. + assert reloaded.status_code == 200 + # What: assert that item name for item in reloaded json equals two; why: this assertion protects the router reload atomically replaces a valid catalog regression after the test's arranged inputs and exercised call. + assert [item["name"] for item in reloaded.json()["models"]] == ["two"] + # What: arrange the exact path write text models bad nmodel n encoding utf 8 fixture fragment; why: the router reload atomically replaces a valid catalog scenario feeds this byte-preserved fragment through path.write_text("[models.bad]\nmodel = ''\n", encoding="utf-8") before asserting its protocol or parser r. + path.write_text("[models.bad]\nmodel = ''\n", encoding="utf-8") + # What: act by calling client.post and capture rejected; why: the router reload atomically replaces a valid catalog test asserts the response, state, or failure produced by this call. + rejected = client.post("/router/reload") + # What: assert that rejected status code equals 400; why: this assertion protects the router reload atomically replaces a valid catalog regression after the test's arranged inputs and exercised call. + assert rejected.status_code == 400 + # What: assert that item name for item in client get equals two; why: this assertion protects the router reload atomically replaces a valid catalog regression after the test's arranged inputs and exercised call. + assert [item["name"] for item in client.get("/router/profiles").json()["data"]] == ["two"] + + +# What: define the test_router_catalog_reload_rotates_bearer_keys_atomically test around tmp path; why: this test groups the arrange, act, and assertions that protect the router catalog reload rotates bearer keys atomically outcome. +def test_router_catalog_reload_rotates_bearer_keys_atomically(tmp_path): + # What: arrange path as tmp path and models and toml; why: the router catalog reload rotates bearer keys atomically test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with router and api keys and first key and models; why: the router catalog reload rotates bearer keys atomically scenario observes the path.write_text return value during router napi keys first key n models low nmodel. + path.write_text( + # What: arrange the exact router napi keys first key n models low nmodel fixture fragment; why: the router catalog reload rotates bearer keys atomically scenario feeds this byte-preserved fragment through "[router]\napi_keys = ['first-key']\n[models.low]\nmodel = 'low.gguf'\n" before asserting its protocol or. + "[router]\napi_keys = ['first-key']\n[models.low]\nmodel = 'low.gguf'\n", + # What: arrange the exact encoding utf 8 fixture fragment; why: the router catalog reload rotates bearer keys atomically scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_router_catalog_reload_rotates_bearer_keys_atomically groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: act by calling Manager and capture manager; why: the router catalog reload rotates bearer keys atomically test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog.load and capture catalog doc; why: the router catalog reload rotates bearer keys atomically test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog.load(str(path)) + # What: act by calling RoutingCoordinator and capture router; why: the router catalog reload rotates bearer keys atomically test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_router_catalog_reload_rotates_bearer_keys_atomically releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the router catalog reload rotates bearer keys atomically test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_router_catalog_reload_rotates_bearer_keys_atomically; why: test_router_catalog_reload_rotates_bearer_keys_atomically consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the router catalog reload rotates bearer keys atomically scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange catalog path to str; why: the router catalog reload rotates bearer keys atomically scenario binds this str and path value to str's catalog path input. + catalog_path=str(path), + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_router_catalog_reload_rotates_bearer_keys_atomically groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the router catalog reload rotates bearer keys atomically test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling path.write_text with router and api keys and second key and models; why: the router catalog reload rotates bearer keys atomically scenario observes the path.write_text return value during router napi keys second key n models low nmodel. + path.write_text( + # What: arrange the exact router napi keys second key n models low nmodel fixture fragment; why: the router catalog reload rotates bearer keys atomically scenario feeds this byte-preserved fragment through "[router]\napi_keys = ['second-key']\n[models.low]\nmodel = 'low.gguf'\n before asserting its protoco. + "[router]\napi_keys = ['second-key']\n[models.low]\nmodel = 'low.gguf'\n", + # What: arrange the exact encoding utf 8 fixture fragment; why: the router catalog reload rotates bearer keys atomically scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_router_catalog_reload_rotates_bearer_keys_atomically groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: act by calling client.post and capture reloaded; why: the router catalog reload rotates bearer keys atomically test asserts the response, state, or failure produced by this call. + reloaded = client.post("/router/reload", headers={"Authorization": "Bearer first-key"}) + # What: act by calling client.get and capture old key; why: the router catalog reload rotates bearer keys atomically test asserts the response, state, or failure produced by this call. + old_key = client.get("/router/status", headers={"Authorization": "Bearer first-key"}) + # What: act by calling client.get and capture new key; why: the router catalog reload rotates bearer keys atomically test asserts the response, state, or failure produced by this call. + new_key = client.get("/router/status", headers={"Authorization": "Bearer second-key"}) + # What: assert that reloaded status code equals 200; why: this assertion protects the router catalog reload rotates bearer keys atomically regression after the test's arranged inputs and exercised call. + assert reloaded.status_code == 200 + # What: assert that old key status code equals 401; why: this assertion protects the router catalog reload rotates bearer keys atomically regression after the test's arranged inputs and exercised call. + assert old_key.status_code == 401 + # What: assert that new key status code equals 200; why: this assertion protects the router catalog reload rotates bearer keys atomically regression after the test's arranged inputs and exercised call. + assert new_key.status_code == 200 + + +# What: define the test_router_reload_rejects_redefining_active_profile test around local fixtures; why: this test groups the arrange, act, and assertions that protect the router reload rejects redefining active profile outcome. +def test_router_reload_rejects_redefining_active_profile(): + # What: act by calling Manager and capture manager; why: the router reload rejects redefining active profile test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the router reload rejects redefining active profile test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: act by calling router.acquire and capture lease; why: the router reload rejects redefining active profile test asserts the response, state, or failure produced by this call. + lease = router.acquire("low") + # What: act by calling ModelCatalog and capture replacement; why: the router reload rejects redefining active profile test asserts the response, state, or failure produced by this call. + replacement = ModelCatalog({"low": ModelProfile("low", "changed.gguf", ())}) + # What: assert the pytest.raises failure context; why: the router reload rejects redefining active profile scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RoutingError, match="cannot redefine") as exc: + # What: act by calling router.replace_catalog with replacement; why: the router reload rejects redefining active profile scenario observes the router.replace_catalog return value during assert exc value status code. + router.replace_catalog(replacement) + # What: assert that exc value status code equals 409; why: this assertion protects the router reload rejects redefining active profile regression after the test's arranged inputs and exercised call. + assert exc.value.status_code == 409 + # What: act by calling lease.release with the declared inputs; why: the router reload rejects redefining active profile scenario observes the lease.release return value during the enclosing return. + lease.release() + + +# What: define the test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes test around local fixtures; why: this test groups the arrange, act, and assertions that protect the router reload refuses active scheduling or effective lifecycle changes outcome. +def test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes(): + # What: act by calling Manager and capture manager; why: the router reload refuses active scheduling or effective lifecycle changes test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture current; why: the router reload refuses active scheduling or effective lifecycle changes test asserts the response, state, or failure produced by this call. + current = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf and g; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes carries low through current into router routing coordinator manager current object ready fn ready. + {"low": ModelProfile("low", "low.gguf", (), group="g")}, + # What: arrange settings to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this router settings and routing group and 4 and 12 and g value to RouterSettings's settings input. + settings=RouterSettings( + # What: arrange default ttl s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 4 value to RouterSettings's default ttl s input. + default_ttl_s=4, + # What: arrange unload timeout s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 12 value to RouterSettings's unload timeout s input. + unload_timeout_s=12, + # What: arrange groups to RoutingGroup; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this routing group and g and low and true and false value to RoutingGroup's groups input. + groups=(RoutingGroup("g", ("low",), swap=True, persistent=False),), + # What: arrange the RouterSettings call with default ttl s and unload timeout s and groups; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one RouterSettings call before its value is consumed. + ), + # What: arrange the ModelCatalog call with settings; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the router reload refuses active scheduling or effective lifecycle changes test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, current, object(), ready_fn=ready) + # What: act by calling router.acquire and capture lease; why: the router reload refuses active scheduling or effective lifecycle changes test asserts the response, state, or failure produced by this call. + lease = router.acquire("low") + + # What: act by calling ModelCatalog and capture changed priority; why: the router reload refuses active scheduling or effective lifecycle changes test asserts the response, state, or failure produced by this call. + changed_priority = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf and 1; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes carries low through changed priority into changed priority. + {"low": ModelProfile("low", "low.gguf", (), priority=1, group="g")}, + # What: arrange settings to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this router settings and routing group and 4 and 12 and g value to RouterSettings's settings input. + settings=RouterSettings( + # What: arrange default ttl s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 4 value to RouterSettings's default ttl s input. + default_ttl_s=4, + # What: arrange unload timeout s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 12 value to RouterSettings's unload timeout s input. + unload_timeout_s=12, + # What: arrange groups to RoutingGroup; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this routing group and g and low and true and false value to RoutingGroup's groups input. + groups=(RoutingGroup("g", ("low",), swap=True, persistent=False),), + # What: arrange the RouterSettings call with default ttl s and unload timeout s and groups; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one RouterSettings call before its value is consumed. + ), + # What: arrange the ModelCatalog call with settings; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling ModelCatalog and capture changed default ttl; why: the router reload refuses active scheduling or effective lifecycle changes test asserts the response, state, or failure produced by this call. + changed_default_ttl = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf and g; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes carries low through changed default ttl into changed default ttl. + {"low": ModelProfile("low", "low.gguf", (), group="g")}, + # What: arrange settings to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this router settings and routing group and 5 and 12 and g value to RouterSettings's settings input. + settings=RouterSettings( + # What: arrange default ttl s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 5 value to RouterSettings's default ttl s input. + default_ttl_s=5, + # What: arrange unload timeout s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 12 value to RouterSettings's unload timeout s input. + unload_timeout_s=12, + # What: arrange groups to RoutingGroup; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this routing group and g and low and true and false value to RoutingGroup's groups input. + groups=(RoutingGroup("g", ("low",), swap=True, persistent=False),), + # What: arrange the RouterSettings call with default ttl s and unload timeout s and groups; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one RouterSettings call before its value is consumed. + ), + # What: arrange the ModelCatalog call with settings; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling ModelCatalog and capture changed default unload; why: the router reload refuses active scheduling or effective lifecycle changes test asserts the response, state, or failure produced by this call. + changed_default_unload = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf and g; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes carries low through changed default unload into changed default unload. + {"low": ModelProfile("low", "low.gguf", (), group="g")}, + # What: arrange settings to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this router settings and routing group and 4 and 13 and g value to RouterSettings's settings input. + settings=RouterSettings( + # What: arrange default ttl s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 4 value to RouterSettings's default ttl s input. + default_ttl_s=4, + # What: arrange unload timeout s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 13 value to RouterSettings's unload timeout s input. + unload_timeout_s=13, + # What: arrange groups to RoutingGroup; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this routing group and g and low and true and false value to RoutingGroup's groups input. + groups=(RoutingGroup("g", ("low",), swap=True, persistent=False),), + # What: arrange the RouterSettings call with default ttl s and unload timeout s and groups; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one RouterSettings call before its value is consumed. + ), + # What: arrange the ModelCatalog call with settings; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling ModelCatalog and capture changed group policy; why: the router reload refuses active scheduling or effective lifecycle changes test asserts the response, state, or failure produced by this call. + changed_group_policy = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf and g; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes carries low through changed group policy into changed group policy. + {"low": ModelProfile("low", "low.gguf", (), group="g")}, + # What: arrange settings to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this router settings and routing group and 4 and 12 and g value to RouterSettings's settings input. + settings=RouterSettings( + # What: arrange default ttl s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 4 value to RouterSettings's default ttl s input. + default_ttl_s=4, + # What: arrange unload timeout s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 12 value to RouterSettings's unload timeout s input. + unload_timeout_s=12, + # What: arrange groups to RoutingGroup; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this routing group and g and low and false and true value to RoutingGroup's groups input. + groups=(RoutingGroup("g", ("low",), swap=False, persistent=True),), + # What: arrange the RouterSettings call with default ttl s and unload timeout s and groups; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one RouterSettings call before its value is consumed. + ), + # What: arrange the ModelCatalog call with settings; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling ModelCatalog and capture changed request filter; why: the router reload refuses active scheduling or effective lifecycle changes test asserts the response, state, or failure produced by this call. + changed_request_filter = ModelCatalog( + # What: arrange the low field as model profile and request field and low and low and gguf; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes carries low through changed request filter into changed request filter. + {"low": ModelProfile( + # What: arrange group to ModelProfile; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this g value to ModelProfile's group input. + "low", "low.gguf", (), group="g", + # What: arrange set fields to RequestField; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this request field and 0 2 and temperature value to RequestField's set fields input. + set_fields=(RequestField(("temperature",), "0.2"),), + # What: arrange the changed_request_filter mapping with low; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one changed_request_filter mapping before its value is consumed. + )}, + # What: arrange settings to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this router settings and routing group and 4 and 12 and g value to RouterSettings's settings input. + settings=RouterSettings( + # What: arrange default ttl s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 4 value to RouterSettings's default ttl s input. + default_ttl_s=4, + # What: arrange unload timeout s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 12 value to RouterSettings's unload timeout s input. + unload_timeout_s=12, + # What: arrange groups to RoutingGroup; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this routing group and g and low and true and false value to RoutingGroup's groups input. + groups=(RoutingGroup("g", ("low",), swap=True, persistent=False),), + # What: arrange the RouterSettings call with default ttl s and unload timeout s and groups; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one RouterSettings call before its value is consumed. + ), + # What: arrange the ModelCatalog call with settings; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling ModelCatalog and capture changed transport targets; why: the router reload refuses active scheduling or effective lifecycle changes test asserts the response, state, or failure produced by this call. + changed_transport_targets = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf and g; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes carries low through changed transport targets into changed transport targets. + {"low": ModelProfile( + # What: arrange low low gguf group g check endpoint ready for the scenario; why: test router test router reload refuses active scheduling or effective lifecycle changes requires this concrete input or helper state before exercising the behavior under test. + "low", "low.gguf", (), group="g", check_endpoint="/ready", + # What: arrange proxy to ModelProfile; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this http and port and gateway value to ModelProfile's proxy input. + proxy="http://127.0.0.1:${PORT}/gateway", + # What: arrange use model name to ModelProfile; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this engine low value to ModelProfile's use model name input. + use_model_name="engine-low", + # What: arrange the changed_transport_targets mapping with low; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one changed_transport_targets mapping before its value is consumed. + )}, + # What: arrange settings to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this router settings and routing group and 4 and 12 and g value to RouterSettings's settings input. + settings=RouterSettings( + # What: arrange default ttl s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 4 value to RouterSettings's default ttl s input. + default_ttl_s=4, + # What: arrange unload timeout s to RouterSettings; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this 12 value to RouterSettings's unload timeout s input. + unload_timeout_s=12, + # What: arrange groups to RoutingGroup; why: the router reload refuses active scheduling or effective lifecycle changes scenario binds this routing group and g and low and true and false value to RoutingGroup's groups input. + groups=(RoutingGroup("g", ("low",), swap=True, persistent=False),), + # What: arrange the RouterSettings call with default ttl s and unload timeout s and groups; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one RouterSettings call before its value is consumed. + ), + # What: arrange the ModelCatalog call with settings; why: test_router_reload_refuses_active_scheduling_or_effective_lifecycle_changes groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: arrange for replacement in for the scenario; why: test router reload refuses active scheduling or effective lifecycle changes requires this concrete input or helper state before exercising the behavior under test. + for replacement in ( + # What: arrange the changed priority portion of the enclosing predicate; why: this clause remains in the router reload refuses active scheduling or effective lifecycle changes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + changed_priority, + # What: arrange the changed default ttl portion of the enclosing predicate; why: this clause remains in the router reload refuses active scheduling or effective lifecycle changes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + changed_default_ttl, + # What: arrange the changed default unload portion of the enclosing predicate; why: this clause remains in the router reload refuses active scheduling or effective lifecycle changes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + changed_default_unload, + # What: arrange the changed group policy portion of the enclosing predicate; why: this clause remains in the router reload refuses active scheduling or effective lifecycle changes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + changed_group_policy, + # What: arrange the changed request filter portion of the enclosing predicate; why: this clause remains in the router reload refuses active scheduling or effective lifecycle changes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + changed_request_filter, + # What: arrange the changed transport targets portion of the enclosing predicate; why: this clause remains in the router reload refuses active scheduling or effective lifecycle changes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + changed_transport_targets, + # What: arrange the grouped source fragment for the scenario; why: test router reload refuses active scheduling or effective lifecycle changes requires this concrete input or helper state before exercising the behavior under test. + ): + # What: assert the pytest.raises failure context; why: the router reload refuses active scheduling or effective lifecycle changes scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RoutingError, match="cannot redefine") as exc: + # What: act by calling router.replace_catalog with replacement; why: the router reload refuses active scheduling or effective lifecycle changes scenario observes the router.replace_catalog return value during assert exc value status code. + router.replace_catalog(replacement) + # What: assert that exc value status code equals 409; why: this assertion protects the router reload refuses active scheduling or effective lifecycle changes regression after the test's arranged inputs and exercised call. + assert exc.value.status_code == 409 + # What: assert that router catalog is current; why: this assertion protects the router reload refuses active scheduling or effective lifecycle changes regression after the test's arranged inputs and exercised call. + assert router.catalog is current + # What: act by calling lease.release with the declared inputs; why: the router reload refuses active scheduling or effective lifecycle changes scenario observes the lease.release return value during the enclosing return. + lease.release() + + +# What: define the test_router_reload_cannot_race_atomic_profile_lookup_and_dynamic_port_binding test around local fixtures; why: this test groups the arrange, act, and assertions that protect the router reload cannot race atomic profile lookup and dynamic port binding outcome. +def test_router_reload_cannot_race_atomic_profile_lookup_and_dynamic_port_binding(): + # What: act by calling threading.Event and capture entered; why: the router reload cannot race atomic profile lookup and dynamic port binding test asserts the response, state, or failure produced by this call. + entered = threading.Event() + # What: act by calling threading.Event and capture release status; why: the router reload cannot race atomic profile lookup and dynamic port binding test asserts the response, state, or failure produced by this call. + release_status = threading.Event() + + # What: define BlockingStatusManager as the owner of status; why: daemon callers use this class boundary so those methods share one blocking status manager state invariant. + class BlockingStatusManager(Manager): + # What: arrange block next status False for the scenario; why: test router test router reload cannot race atomic profile lookup and dynamic port binding requires this concrete input or helper state before exercising the behavior under test. + block_next_status = False + + # What: define the status test helper around captured fixture state; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def status(self): + # What: act on block next status before block next status; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario admits block next status only for this predicate and excludes the opposite state. + if self.block_next_status: + # What: arrange self block next status False for the scenario; why: test router test router reload cannot race atomic profile lookup and dynamic port binding requires this concrete input or helper state before exercising the behavior under test. + self.block_next_status = False + # What: act by calling entered.set with the declared inputs; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario observes the entered.set return value during assert release status wait. + entered.set() + # What: assert that release status wait 2; why: this assertion protects the router reload cannot race atomic profile lookup and dynamic port binding regression after the test's arranged inputs and exercised call. + assert release_status.wait(2) + # What: return status and super from the status test helper; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario uses this helper result in its subsequent act or assertion. + return super().status() + + # What: act by calling BlockingStatusManager and capture manager; why: the router reload cannot race atomic profile lookup and dynamic port binding test asserts the response, state, or failure produced by this call. + manager = BlockingStatusManager() + # What: act by calling ModelCatalog and capture current; why: the router reload cannot race atomic profile lookup and dynamic port binding test asserts the response, state, or failure produced by this call. + current = ModelCatalog({"low": ModelProfile("low", "low.gguf", (), port=0)}) + # What: act by calling RoutingCoordinator and capture router; why: the router reload cannot race atomic profile lookup and dynamic port binding test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator( + # What: arrange ready fn to object; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario binds this ready value to object's ready fn input. + manager, current, object(), ready_fn=ready, port_allocator=lambda: 20101 + # What: arrange the RoutingCoordinator call with ready fn and port allocator; why: test_router_reload_cannot_race_atomic_profile_lookup_and_dynamic_port_binding groups the supplied clauses as one RoutingCoordinator call before its value is consumed. + ) + # What: arrange block next status as true; why: the router reload cannot race atomic profile lookup and dynamic port binding test consumes this named precondition before exercising the behavior. + manager.block_next_status = True + # What: arrange acquired as the fixture input; why: the router reload cannot race atomic profile lookup and dynamic port binding test consumes this named precondition before exercising the behavior. + acquired = [] + # What: act by calling threading.Thread and capture acquire thread; why: the router reload cannot race atomic profile lookup and dynamic port binding test asserts the response, state, or failure produced by this call. + acquire_thread = threading.Thread(target=lambda: acquired.append(router.acquire("low"))) + # What: act by calling acquire_thread.start with the declared inputs; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario observes the acquire_thread.start return value during assert entered wait. + acquire_thread.start() + # What: assert that entered wait 1; why: this assertion protects the router reload cannot race atomic profile lookup and dynamic port binding regression after the test's arranged inputs and exercised call. + assert entered.wait(1) + + # What: act by calling ModelCatalog and capture replacement; why: the router reload cannot race atomic profile lookup and dynamic port binding test asserts the response, state, or failure produced by this call. + replacement = ModelCatalog({"low": ModelProfile("low", "changed.gguf", (), port=0)}) + # What: arrange reload result as the fixture input; why: the router reload cannot race atomic profile lookup and dynamic port binding test consumes this named precondition before exercising the behavior. + reload_result = {} + + # What: define the reload_catalog test helper around captured fixture state; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def reload_catalog(): + # What: establish the handler boundary for the protected operation; why: reload_catalog routes failures to routing error while preserving cleanup and success flow. + try: + # What: act by calling router.replace_catalog with replacement; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario observes the router.replace_catalog return value during except routing error as exc. + router.replace_catalog(replacement) + # What: handle routing error by reload result error exc; why: reload_catalog converts that failure into this concrete recovery, response, or cleanup behavior. + except RoutingError as exc: + # What: arrange reload result entry as exc; why: the router reload cannot race atomic profile lookup and dynamic port binding test consumes this named precondition before exercising the behavior. + reload_result["error"] = exc + + # What: act by calling threading.Thread and capture reload thread; why: the router reload cannot race atomic profile lookup and dynamic port binding test asserts the response, state, or failure produced by this call. + reload_thread = threading.Thread(target=reload_catalog) + # What: act by calling reload_thread.start with the declared inputs; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario observes the reload_thread.start return value during time sleep. + reload_thread.start() + # What: act by calling time.sleep with 0 05; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario observes the time.sleep return value during assert reload thread is alive. + time.sleep(0.05) + # What: assert that reload thread is alive; why: this assertion protects the router reload cannot race atomic profile lookup and dynamic port binding regression after the test's arranged inputs and exercised call. + assert reload_thread.is_alive() + + # What: act by calling release_status.set with the declared inputs; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario observes the release_status.set return value during acquire thread join. + release_status.set() + # What: act by calling acquire_thread.join with 2; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario observes the acquire_thread.join return value during reload thread join. + acquire_thread.join(2) + # What: act by calling reload_thread.join with 2; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario observes the reload_thread.join return value during assert not acquire thread is alive and not reload thread is alive. + reload_thread.join(2) + # What: assert that not acquire thread is alive and not reload thread is alive; why: this assertion protects the router reload cannot race atomic profile lookup and dynamic port binding regression after the test's arranged inputs and exercised call. + assert not acquire_thread.is_alive() and not reload_thread.is_alive() + # What: assert that reload result error code equals reload conflict; why: this assertion protects the router reload cannot race atomic profile lookup and dynamic port binding regression after the test's arranged inputs and exercised call. + assert reload_result["error"].code == "reload_conflict" + # What: assert that router catalog is current; why: this assertion protects the router reload cannot race atomic profile lookup and dynamic port binding regression after the test's arranged inputs and exercised call. + assert router.catalog is current + # What: assert that manager model equals low gguf; why: this assertion protects the router reload cannot race atomic profile lookup and dynamic port binding regression after the test's arranged inputs and exercised call. + assert manager.model == "low.gguf" + # What: act by calling operation.release with the declared inputs; why: the router reload cannot race atomic profile lookup and dynamic port binding scenario observes the operation.release return value during the enclosing return. + acquired.pop().release() + + +# What: define the test_router_reload_cannot_redefine_a_profile_already_queued_for_admission test around local fixtures; why: this test groups the arrange, act, and assertions that protect the router reload cannot redefine a profile already queued for admission outcome. +def test_router_reload_cannot_redefine_a_profile_already_queued_for_admission(): + # What: act by calling Manager and capture manager; why: the router reload cannot redefine a profile already queued for admission test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling catalog and capture current; why: the router reload cannot redefine a profile already queued for admission test asserts the response, state, or failure produced by this call. + current = catalog() + # What: act by calling RoutingCoordinator and capture router; why: the router reload cannot redefine a profile already queued for admission test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, current, object(), ready_fn=ready) + # What: act by calling router.acquire and capture active; why: the router reload cannot redefine a profile already queued for admission test asserts the response, state, or failure produced by this call. + active = router.acquire("low") + # What: arrange queued lease as the fixture input; why: the router reload cannot redefine a profile already queued for admission test consumes this named precondition before exercising the behavior. + queued_lease = [] + # What: act by calling threading.Thread and capture queued; why: the router reload cannot redefine a profile already queued for admission test asserts the response, state, or failure produced by this call. + queued = threading.Thread(target=lambda: queued_lease.append(router.acquire("high"))) + # What: act by calling queued.start with the declared inputs; why: the router reload cannot redefine a profile already queued for admission scenario observes the queued.start return value during for value in range. + queued.start() + # What: act across range to perform status and router; why: the router reload cannot redefine a profile already queued for admission scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on status and router before the computed value; why: the router reload cannot redefine a profile already queued for admission scenario admits the computed value only for this predicate and excludes the opposite state. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the router reload cannot redefine a profile already queued for admission scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling time.sleep with 0 01; why: the router reload cannot redefine a profile already queued for admission scenario observes the time.sleep return value during assert router status queued requests. + time.sleep(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the router reload cannot redefine a profile already queued for admission regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + + # What: act by calling ModelCatalog and capture replacement; why: the router reload cannot redefine a profile already queued for admission test asserts the response, state, or failure produced by this call. + replacement = ModelCatalog({ + # What: arrange the low field as model profile and low and low and gguf; why: test_router_reload_cannot_redefine_a_profile_already_queued_for_admission carries low through replacement into router replace catalog replacement. + "low": ModelProfile("low", "low.gguf", ()), + # What: arrange the high field as model profile and high and changed and gguf and 10; why: test_router_reload_cannot_redefine_a_profile_already_queued_for_admission carries high through replacement into router replace catalog replacement. + "high": ModelProfile("high", "changed.gguf", (), priority=10), + # What: arrange the ModelCatalog call with model profile; why: test_router_reload_cannot_redefine_a_profile_already_queued_for_admission groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + # What: assert the pytest.raises failure context; why: the router reload cannot redefine a profile already queued for admission scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RoutingError, match="admission or lifecycle") as exc: + # What: act by calling router.replace_catalog with replacement; why: the router reload cannot redefine a profile already queued for admission scenario observes the router.replace_catalog return value during assert exc value code reload conflict. + router.replace_catalog(replacement) + # What: assert that exc value code equals reload conflict; why: this assertion protects the router reload cannot redefine a profile already queued for admission regression after the test's arranged inputs and exercised call. + assert exc.value.code == "reload_conflict" + # What: assert that router catalog is current; why: this assertion protects the router reload cannot redefine a profile already queued for admission regression after the test's arranged inputs and exercised call. + assert router.catalog is current + + # What: act by calling active.release with the declared inputs; why: the router reload cannot redefine a profile already queued for admission scenario observes the active.release return value during queued join. + active.release() + # What: act by calling queued.join with 2; why: the router reload cannot redefine a profile already queued for admission scenario observes the queued.join return value during assert not queued is alive. + queued.join(2) + # What: assert that queued is alive is false; why: this assertion protects the router reload cannot redefine a profile already queued for admission regression after the test's arranged inputs and exercised call. + assert not queued.is_alive() + # What: assert that manager model equals high gguf; why: this assertion protects the router reload cannot redefine a profile already queued for admission regression after the test's arranged inputs and exercised call. + assert manager.model == "high.gguf" + # What: act by calling operation.release with the declared inputs; why: the router reload cannot redefine a profile already queued for admission scenario observes the operation.release return value during the enclosing return. + queued_lease.pop().release() + + +# What: define the test_persistent_group_protects_the_single_resident_slot_until_unloaded test around local fixtures; why: this test groups the arrange, act, and assertions that protect the persistent group protects the single resident slot until unloaded outcome. +def test_persistent_group_protects_the_single_resident_slot_until_unloaded(): + # What: act by calling Manager and capture manager; why: the persistent group protects the single resident slot until unloaded test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the persistent group protects the single resident slot until unloaded test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with keep and other; why: test_persistent_group_protects_the_single_resident_slot_until_unloaded groups the supplied clauses as one catalog_doc mapping before its value. + { + # What: arrange the keep field as model profile and keep and keep and gguf and resident; why: test_persistent_group_protects_the_single_resident_slot_until_unloaded carries keep through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "keep": ModelProfile("keep", "keep.gguf", (), group="resident"), + # What: arrange the other field as model profile and other and other and gguf; why: test_persistent_group_protects_the_single_resident_slot_until_unloaded carries other through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "other": ModelProfile("other", "other.gguf", ()), + # What: arrange the catalog_doc mapping with keep and other; why: test_persistent_group_protects_the_single_resident_slot_until_unloaded groups the supplied clauses as one catalog_doc mapping before its value. + }, + # What: arrange settings to RouterSettings; why: the persistent group protects the single resident slot until unloaded scenario binds this router settings and routing group and resident and keep and false value to RouterSettings's settings input. + settings=RouterSettings(groups=( + # What: arrange swap to RoutingGroup; why: the persistent group protects the single resident slot until unloaded scenario binds this false value to RoutingGroup's swap input. + RoutingGroup("resident", ("keep",), swap=False, persistent=True), + # What: arrange the RouterSettings call with groups; why: test_persistent_group_protects_the_single_resident_slot_until_unloaded groups the supplied clauses as one RouterSettings call before its value is consumed. + )), + # What: arrange the ModelCatalog call with settings; why: test_persistent_group_protects_the_single_resident_slot_until_unloaded groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the persistent group protects the single resident slot until unloaded test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange the exact router acquire keep release fixture fragment; why: the persistent group protects the single resident slot until unloaded scenario feeds this byte-preserved fragment through router.acquire("keep").release() before asserting its protocol or parser result. + router.acquire("keep").release() + # What: assert the pytest.raises failure context; why: the persistent group protects the single resident slot until unloaded scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RoutingError, match="single resident-model slot") as exc: + # What: arrange the exact router acquire other fixture fragment; why: the persistent group protects the single resident slot until unloaded scenario feeds this byte-preserved fragment through router.acquire("other") before asserting its protocol or parser result. + router.acquire("other") + # What: assert that exc value code equals capacity unavailable; why: this assertion protects the persistent group protects the single resident slot until unloaded regression after the test's arranged inputs and exercised call. + assert exc.value.code == "capacity_unavailable" + # What: assert that router status resident profiles equals keep; why: this assertion protects the persistent group protects the single resident slot until unloaded regression after the test's arranged inputs and exercised call. + assert router.status()["residentProfiles"] == ["keep"] + # What: assert that router evict idle keep is true; why: this assertion protects the persistent group protects the single resident slot until unloaded regression after the test's arranged inputs and exercised call. + assert router.evict_idle("keep") is True + # What: arrange the exact router acquire other release fixture fragment; why: the persistent group protects the single resident slot until unloaded scenario feeds this byte-preserved fragment through router.acquire("other").release() before asserting its protocol or parser result. + router.acquire("other").release() + # What: assert the expected manager calls == outcome; why: test router test persistent group protects the single resident slot until unloaded protects its regression by requiring this observable result after the exercised behavior. + assert manager.calls == [ + # What: arrange start keep gguf stop 30.0 start other gguf for the scenario; why: test router test persistent group protects the single resident slot until unloaded requires this concrete input or helper state before exercising the behavior under test. + ("start", "keep.gguf"), ("stop", 30.0), ("start", "other.gguf"), + # What: arrange the grouped source fragment for the scenario; why: test router test persistent group protects the single resident slot until unloaded requires this concrete input or helper state before exercising the behavior under test. + ] + + +# What: define the test_explicit_router_cancel_closes_an_inflight_upstream test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the explicit router cancel closes an inflight upstream outcome. +def test_explicit_router_cancel_closes_an_inflight_upstream(monkeypatch): + # What: define BlockingRaw as the owner of __init__ and read and close; why: daemon callers use this class boundary so those methods share one blocking raw state invariant. + class BlockingRaw: + # What: define the __init__ test helper around captured fixture state; why: the explicit router cancel closes an inflight upstream scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def __init__(self): + # What: act by calling threading.Event and capture read started; why: the explicit router cancel closes an inflight upstream test asserts the response, state, or failure produced by this call. + self.read_started = threading.Event() + # What: act by calling threading.Event and capture closed; why: the explicit router cancel closes an inflight upstream test asserts the response, state, or failure produced by this call. + self.closed = threading.Event() + + # What: define the read test helper around size; why: the explicit router cancel closes an inflight upstream scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def read(self, size): + # What: act by calling self.read_started.set with the declared inputs; why: the explicit router cancel closes an inflight upstream scenario observes the self.read_started.set return value during self closed wait. + self.read_started.set() + # What: act by calling self.closed.wait with 2; why: the explicit router cancel closes an inflight upstream scenario observes the self.closed.wait return value during return b. + self.closed.wait(2) + # What: return the named fixture input from the read test helper; why: the explicit router cancel closes an inflight upstream scenario uses this helper result in its subsequent act or assertion. + return b"" + + # What: define the close test helper around captured fixture state; why: the explicit router cancel closes an inflight upstream scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def close(self): + # What: act by calling self.closed.set with the declared inputs; why: the explicit router cancel closes an inflight upstream scenario observes the self.closed.set return value during the enclosing return. + self.closed.set() + + # What: act by calling BlockingRaw and capture raw; why: the explicit router cancel closes an inflight upstream test asserts the response, state, or failure produced by this call. + raw = BlockingRaw() + # What: act by calling Manager and capture manager; why: the explicit router cancel closes an inflight upstream test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the explicit router cancel closes an inflight upstream test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({"low": ModelProfile("low", "low.gguf", ())}) + # What: act by calling RoutingCoordinator and capture router; why: the explicit router cancel closes an inflight upstream test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange upstream calls as the fixture input; why: the explicit router cancel closes an inflight upstream test consumes this named precondition before exercising the behavior. + upstream_calls = [] + + # What: define the upstream test helper around captured fixture state; why: the explicit router cancel closes an inflight upstream scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: act by calling upstream_calls.append with kwargs; why: the explicit router cancel closes an inflight upstream scenario observes the upstream_calls.append return value during return upstream response content type text event stream raw. + upstream_calls.append(kwargs) + # What: arrange the helper response as UpstreamResponse 200 Content Type text event stream raw; why: test router test explicit router cancel closes an inflight upstream feeds this result into the behavior whose outcome is asserted. + return UpstreamResponse(200, {"Content-Type": "text/event-stream"}, raw) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the explicit router cancel closes an inflight upstream scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) before asserting its protoc. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_explicit_router_cancel_closes_an_inflight_upstream releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the explicit router cancel closes an inflight upstream test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_explicit_router_cancel_closes_an_inflight_upstream; why: test_explicit_router_cancel_closes_an_inflight_upstream consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the explicit router cancel closes an inflight upstream scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_explicit_router_cancel_closes_an_inflight_upstream groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the explicit router cancel closes an inflight upstream test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: arrange response as the fixture input; why: the explicit router cancel closes an inflight upstream test consumes this named precondition before exercising the behavior. + response = [] + # What: act by calling threading.Thread and capture thread; why: the explicit router cancel closes an inflight upstream test asserts the response, state, or failure produced by this call. + thread = threading.Thread(target=lambda: response.append(client.post( + # What: arrange the model field as low; why: test_explicit_router_cancel_closes_an_inflight_upstream sends this field through thread so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "low"}, headers={"X-FT-Request-ID": "cancel-me"}, + # What: arrange the threading.Thread call with target; why: test_explicit_router_cancel_closes_an_inflight_upstream groups the supplied clauses as one threading.Thread call before its value is consumed. + ))) + # What: act by calling thread.start with the declared inputs; why: the explicit router cancel closes an inflight upstream scenario observes the thread.start return value during assert raw read started wait. + thread.start() + # What: assert that raw read started wait 1; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert raw.read_started.wait(1) + # What: act by calling operation.json and capture active; why: the explicit router cancel closes an inflight upstream test asserts the response, state, or failure produced by this call. + active = client.get("/router/requests").json()["data"] + # What: assert that active equals id cancel me profile low; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert active == [{"id": "cancel-me", "profile": "low"}] + # What: act by calling client.post and capture duplicate; why: the explicit router cancel closes an inflight upstream test asserts the response, state, or failure produced by this call. + duplicate = client.post( + # What: arrange the model field as low; why: test_explicit_router_cancel_closes_an_inflight_upstream sends this field through duplicate so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "low"}, + # What: arrange the x ft request id field as cancel me; why: test_explicit_router_cancel_closes_an_inflight_upstream carries x ft request id through duplicate into assert duplicate status code equals 409. + headers={"X-FT-Request-ID": "cancel-me"}, + # What: arrange the client.post call with json and headers; why: test_explicit_router_cancel_closes_an_inflight_upstream groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: assert that duplicate status code equals 409; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert duplicate.status_code == 409 + # What: assert that duplicate json error type equals request conflict; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert duplicate.json()["error"]["type"] == "request_conflict" + # What: assert that len upstream calls equals 1; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert len(upstream_calls) == 1 + # What: assert that router status admissions equals 1; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert router.status()["admissions"] == 1 + # What: act by calling client.post and capture cancelled; why: the explicit router cancel closes an inflight upstream test asserts the response, state, or failure produced by this call. + cancelled = client.post("/router/requests/cancel-me/cancel") + # What: assert that cancelled json equals cancelled true id cancel me; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert cancelled.json() == {"cancelled": True, "id": "cancel-me"} + # What: act by calling thread.join with 2; why: the explicit router cancel closes an inflight upstream scenario observes the thread.join return value during assert not thread is alive. + thread.join(2) + # What: assert that thread is alive is false; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert not thread.is_alive() + # What: assert that response 0 status code equals 200; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert response[0].status_code == 200 + # What: assert that router status cancellations equals 1; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert router.status()["cancellations"] == 1 + # What: assert that router status terminal streams equals 0; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert router.status()["terminalStreams"] == 0 + # What: assert that freetoken swap terminal streams total 0 is present in router prometheus; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert "freetoken_swap_terminal_streams_total 0" in router.prometheus() + # What: assert that router status active requests equals 0; why: this assertion protects the explicit router cancel closes an inflight upstream regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + + +# What: define the test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes test around local fixtures; why: this test groups the arrange, act, and assertions that protect the native proxy uses a real loopback http upstream and preserves sse bytes outcome. +def test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes(): + # What: arrange seen as the fixture input; why: the native proxy uses a real loopback http upstream and preserves sse bytes test consumes this named precondition before exercising the behavior. + seen = {} + + # What: define Handler as the owner of do_POST and log_message; why: daemon callers use this class boundary so those methods share one handler state invariant. + class Handler(BaseHTTPRequestHandler): + # What: define the do_POST test helper around captured fixture state; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def do_POST(self): + # What: arrange seen entry as path; why: the native proxy uses a real loopback http upstream and preserves sse bytes test consumes this named precondition before exercising the behavior. + seen["path"] = self.path + # What: act by calling self.rfile.read and capture seen entry; why: the native proxy uses a real loopback http upstream and preserves sse bytes test asserts the response, state, or failure produced by this call. + seen["body"] = self.rfile.read(int(self.headers["Content-Length"])) + # What: arrange seen authorization self headers get Authorization for the scenario; why: test router test native proxy uses a real loopback http upstream and preserves sse bytes requires this concrete input or helper state before exercising the behavior under test. + seen["authorization"] = self.headers.get("Authorization") + # What: arrange seen daemon token self headers get X FT Token for the scenario; why: test router test native proxy uses a real loopback http upstream and preserves sse bytes requires this concrete input or helper state before exercising the behavior under test. + seen["daemon_token"] = self.headers.get("X-FT-Token") + # What: arrange seen correlation self headers get X Correlation ID for the scenario; why: test router test native proxy uses a real loopback http upstream and preserves sse bytes requires this concrete input or helper state before exercising the behavior under test. + seen["correlation"] = self.headers.get("X-Correlation-ID") + # What: act by calling self.send_response with 200; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario observes the self.send_response return value during self send header content type text event stream. + self.send_response(200) + # What: arrange the exact self send header content type text event stream fixture fragment; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario feeds this byte-preserved fragment through self.send_header("Content-Type", "text/event-stream") before asserting its protoco. + self.send_header("Content-Type", "text/event-stream") + # What: arrange the exact self send header x engine loopback fixture fragment; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario feeds this byte-preserved fragment through self.send_header("X-Engine", "loopback") before asserting its protocol or parser result. + self.send_header("X-Engine", "loopback") + # What: act by calling self.end_headers with the declared inputs; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario observes the self.end_headers return value during self wfile write b data ok true n. + self.end_headers() + # What: act by calling self.wfile.write with the named fixture input; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario observes the self.wfile.write return value during the enclosing return. + self.wfile.write(b"data: {\"ok\":true}\n\ndata: [DONE]\n\n") + + # What: define the log_message test helper around format; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def log_message(self, format, *args): + # What: ignore the anticipated exception handled by this branch; why: log_message continues its retry or cleanup path instead of re-raising that transient failure. + pass + + # What: act by calling ThreadingHTTPServer and capture server; why: the native proxy uses a real loopback http upstream and preserves sse bytes test asserts the response, state, or failure produced by this call. + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + # What: act by calling threading.Thread and capture worker; why: the native proxy uses a real loopback http upstream and preserves sse bytes test asserts the response, state, or failure produced by this call. + worker = threading.Thread(target=server.serve_forever, daemon=True) + # What: act by calling worker.start with the declared inputs; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario observes the worker.start return value during try. + worker.start() + # What: establish the handler boundary for the protected operation; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: act by calling Manager and capture manager; why: the native proxy uses a real loopback http upstream and preserves sse bytes test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: arrange port as server address and server and 1; why: the native proxy uses a real loopback http upstream and preserves sse bytes test consumes this named precondition before exercising the behavior. + port = server.server_address[1] + # What: act by calling ModelCatalog and capture catalog doc; why: the native proxy uses a real loopback http upstream and preserves sse bytes test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and port and low and low and gguf; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": ModelProfile( + # What: arrange the low portion of catalog doc; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario uses this clause to evaluate catalog doc as one grouped value. + "low", + # What: arrange the low gguf portion of catalog doc; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario uses this clause to evaluate catalog doc as one grouped value. + "low.gguf", + # What: arrange the catalog_doc collection with ordered entries; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes groups the supplied clauses as one catalog_doc collection before its value is consumed. + (), + # What: arrange port to ModelProfile; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario binds this port value to ModelProfile's port input. + port=port, + # What: arrange proxy to ModelProfile; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario binds this http and port and gateway value to ModelProfile's proxy input. + proxy="http://127.0.0.1:${PORT}/gateway", + # What: arrange the catalog_doc mapping with low; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes groups the supplied clauses as one catalog_doc mapping before its value is consumed. + )}, + # What: arrange settings to RouterSettings; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario binds this router settings and router test key value to RouterSettings's settings input. + settings=RouterSettings(api_keys=("router-test-key",)), + # What: arrange the ModelCatalog call with settings; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the native proxy uses a real loopback http upstream and preserves sse bytes test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the native proxy uses a real loopback http upstream and preserves sse bytes test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange token to build_app; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario binds this daemon control secret value to build_app's token input. + token="daemon-control-secret", + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: arrange payload as the fixture input; why: the native proxy uses a real loopback http upstream and preserves sse bytes test consumes this named precondition before exercising the behavior. + payload = b'{"model":"low","stream":true,"messages":[]}' + # What: act by calling operation.post and capture response; why: the native proxy uses a real loopback http upstream and preserves sse bytes test asserts the response, state, or failure produced by this call. + response = TestClient(app).post( + # What: arrange content to operation.post; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario binds this payload value to operation.post's content input. + "/v1/chat/completions", content=payload, + # What: arrange headers to operation.post; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario binds this content type and authorization and x ft token and x correlation id and application value to operation.post's headers input. + headers={ + # What: arrange the content type field as application and json; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes carries content type through response into assert response status code equals 200. + "Content-Type": "application/json", + # What: arrange the authorization field as bearer and router test key; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes carries authorization through response into assert response status code equals 200. + "Authorization": "Bearer router-test-key", + # What: arrange the x ft token field as daemon control secret; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes carries x ft token through response into assert response status code equals 200. + "X-FT-Token": "daemon-control-secret", + # What: arrange the x correlation id field as client safe id; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes carries x correlation id through response into assert response status code equals 200. + "X-Correlation-ID": "client-safe-id", + # What: arrange the response mapping with content type and authorization and x ft token and x correlation id; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes groups the supplied clauses as one response mapping before its value is consumed. + }, + # What: arrange the operation.post call with content and headers; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes groups the supplied clauses as one operation.post call before its value is consumed. + ) + # What: assert that response status code equals 200; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + # What: assert that response headers x engine equals loopback; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert response.headers["x-engine"] == "loopback" + # What: assert that response content equals b data ok true n ndata; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert response.content == b"data: {\"ok\":true}\n\ndata: [DONE]\n\n" + # What: assert the expected seen == outcome; why: test router test native proxy uses a real loopback http upstream and preserves sse bytes protects its regression by requiring this observable result after the exercised behavior. + assert seen == { + # What: arrange path gateway v1 chat completions for the scenario; why: test router test native proxy uses a real loopback http upstream and preserves sse bytes requires this concrete input or helper state before exercising the behavior under test. + "path": "/gateway/v1/chat/completions", + # What: arrange body payload for the scenario; why: test router test native proxy uses a real loopback http upstream and preserves sse bytes requires this concrete input or helper state before exercising the behavior under test. + "body": payload, + # What: arrange authorization None for the scenario; why: test router test native proxy uses a real loopback http upstream and preserves sse bytes requires this concrete input or helper state before exercising the behavior under test. + "authorization": None, + # What: arrange daemon token None for the scenario; why: test router test native proxy uses a real loopback http upstream and preserves sse bytes requires this concrete input or helper state before exercising the behavior under test. + "daemon_token": None, + # What: arrange correlation client safe id for the scenario; why: test router test native proxy uses a real loopback http upstream and preserves sse bytes requires this concrete input or helper state before exercising the behavior under test. + "correlation": "client-safe-id", + # What: arrange the grouped source fragment for the scenario; why: test router test native proxy uses a real loopback http upstream and preserves sse bytes requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that router status active requests equals 0; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + # What: assert that router status terminal streams equals 1; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert router.status()["terminalStreams"] == 1 + # What: assert that router status last ttft ms is not group delimiter; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert router.status()["lastTtftMs"] is not None + # What: assert that router status last duration ms is not group delimiter; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert router.status()["lastDurationMs"] is not None + # What: assert that router status last activation ms is not group delimiter; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert router.status()["lastActivationMs"] is not None + # What: assert that router status last queue wait ms is not group delimiter; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert router.status()["lastQueueWaitMs"] is not None + # What: assert that router status last response bytes equals len response content; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert router.status()["lastResponseBytes"] == len(response.content) + # What: assert that router status last proxy bytes per second is not group delimiter; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert router.status()["lastProxyBytesPerSecond"] is not None + # What: act by calling router.prometheus and capture metrics; why: the native proxy uses a real loopback http upstream and preserves sse bytes test asserts the response, state, or failure produced by this call. + metrics = router.prometheus() + # What: assert that freetoken swap last ttft ms is present in metrics; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert "freetoken_swap_last_ttft_ms" in metrics + # What: assert that freetoken swap last activation ms is present in metrics; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert "freetoken_swap_last_activation_ms" in metrics + # What: assert that freetoken swap last queue wait ms is present in metrics; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert "freetoken_swap_last_queue_wait_ms" in metrics + # What: assert that f freetoken swap last response bytes len response content is present in metrics; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert f"freetoken_swap_last_response_bytes {len(response.content)}" in metrics + # What: assert that freetoken swap last proxy bytes per second is present in metrics; why: this assertion protects the native proxy uses a real loopback http upstream and preserves sse bytes regression after the test's arranged inputs and exercised call. + assert "freetoken_swap_last_proxy_bytes_per_second" in metrics + # What: run server shutdown on every exit path; why: test_native_proxy_uses_a_real_loopback_http_upstream_and_preserves_sse_bytes performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: act by calling server.shutdown with the declared inputs; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario observes the server.shutdown return value during server server close. + server.shutdown() + # What: act by calling server.server_close with the declared inputs; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario observes the server.server_close return value during worker join. + server.server_close() + # What: act by calling worker.join with 2; why: the native proxy uses a real loopback http upstream and preserves sse bytes scenario observes the worker.join return value during the enclosing return. + worker.join(2) + + +# What: define the test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the streaming chat emits cold queue feedback then preserves upstream sse outcome. +def test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse(monkeypatch): + # What: act by calling Manager and capture manager; why: the streaming chat emits cold queue feedback then preserves upstream sse test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the streaming chat emits cold queue feedback then preserves upstream sse test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with low and high; why: test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse groups the supplied clauses as one catalog_doc mapping before its value. + { + # What: arrange the low field as model profile and low and low and gguf; why: test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "low": ModelProfile("low", "low.gguf", ()), + # What: arrange the high field as model profile and request field and high and high and gguf; why: test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse carries high through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "high": ModelProfile( + # What: arrange the high high gguf portion of catalog doc; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario uses this clause to evaluate catalog doc as one grouped value. + "high", "high.gguf", (), + # What: arrange set fields to RequestField; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario binds this request field and 0 2 and temperature value to RequestField's set fields input. + set_fields=(RequestField(("temperature",), "0.2"),), + # What: arrange the ModelProfile call with set fields; why: test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse groups the supplied clauses as one ModelProfile call before its value is consumed. + ), + # What: arrange the catalog_doc mapping with low and high; why: test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse groups the supplied clauses as one catalog_doc mapping before its value. + }, + # What: arrange settings to RouterSettings; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario binds this router settings and true and 1 value to RouterSettings's settings input. + settings=RouterSettings(send_loading_state=True, capture_buffer_mb=1), + # What: arrange the ModelCatalog call with settings; why: test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the streaming chat emits cold queue feedback then preserves upstream sse test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: act by calling router.acquire and capture active; why: the streaming chat emits cold queue feedback then preserves upstream sse test asserts the response, state, or failure produced by this call. + active = router.acquire("low") + # What: arrange upstream body as the fixture input; why: the streaming chat emits cold queue feedback then preserves upstream sse test consumes this named precondition before exercising the behavior. + upstream_body = b'data: {"token":"real"}\n\ndata: [DONE]\n\n' + # What: arrange seen as the fixture input; why: the streaming chat emits cold queue feedback then preserves upstream sse test consumes this named precondition before exercising the behavior. + seen = {} + + # What: define the upstream test helper around captured fixture state; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: act by calling seen.update with kwargs; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario observes the seen.update return value during return upstream response. + seen.update(kwargs) + # What: return upstream response and bytes io and upstream body and 200 and content type from the upstream test helper; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario uses this helper result in its subsequent act or assertion. + return UpstreamResponse( + # What: arrange 200 Content Type text event stream BytesIO upstream body for the scenario; why: test router test streaming chat emits cold queue feedback then preserves upstream sse requires this concrete input or helper state before exercising the behavior under test. + 200, {"Content-Type": "text/event-stream"}, BytesIO(upstream_body) + # What: arrange the grouped source fragment for the scenario; why: test router test streaming chat emits cold queue feedback then preserves upstream sse requires this concrete input or helper state before exercising the behavior under test. + ) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) before as. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: arrange responses as the fixture input; why: the streaming chat emits cold queue feedback then preserves upstream sse test consumes this named precondition before exercising the behavior. + responses = [] + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(2) as lifecycle, ThreadPoolExecutor(2) as proxy: + # What: act by calling build_app and capture app; why: the streaming chat emits cold queue feedback then preserves upstream sse test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse; why: test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the streaming chat emits cold queue feedback then preserves upstream sse test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling threading.Thread and capture thread; why: the streaming chat emits cold queue feedback then preserves upstream sse test asserts the response, state, or failure produced by this call. + thread = threading.Thread(target=lambda: responses.append(client.post( + # What: arrange the v1 chat completions portion of thread; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario uses this clause to evaluate thread as one grouped value. + "/v1/chat/completions", + # What: arrange the model field as high; why: test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse sends this field through thread so the router selects the canonical model or alias for upstream dispatch. + json={"model": "high", "stream": True, "messages": []}, + # What: arrange the x ft request id field as cold feedback; why: test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse carries x ft request id through thread into thread start. + headers={"X-FT-Request-ID": "cold-feedback"}, + # What: arrange the threading.Thread call with target; why: test_streaming_chat_emits_cold_queue_feedback_then_preserves_upstream_sse groups the supplied clauses as one threading.Thread call before its value is consumed. + ))) + # What: act by calling thread.start with the declared inputs; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario observes the thread.start return value during for value in range. + thread.start() + # What: act across range to perform status and router; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on status and router before the computed value; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario admits the computed value only for this predicate and excludes the opposite state. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the streaming chat emits cold queue feedback then preserves upstream sse scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling time.sleep with 0 01; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario observes the time.sleep return value during assert router status queued requests. + time.sleep(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + # What: act by calling active.release with the declared inputs; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario observes the active.release return value during thread join. + active.release() + # What: act by calling thread.join with 3; why: the streaming chat emits cold queue feedback then preserves upstream sse scenario observes the thread.join return value during assert not thread is alive. + thread.join(3) + # What: assert that thread is alive is false; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert not thread.is_alive() + # What: act by calling operation.json and capture activity; why: the streaming chat emits cold queue feedback then preserves upstream sse test asserts the response, state, or failure produced by this call. + activity = client.get("/router/activity").json()["data"] + # What: assert that len activity equals 1 and activity 0 has capture is true; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert len(activity) == 1 and activity[0]["hasCapture"] is True + # What: act by calling operation.json and capture capture; why: the streaming chat emits cold queue feedback then preserves upstream sse test asserts the response, state, or failure produced by this call. + capture = client.get(f'/router/captures/{activity[0]["id"]}').json() + # What: assert that base64 b64decode capture response body base64 equals responses 0 content; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert base64.b64decode(capture["responseBodyBase64"]) == responses[0].content + + # What: arrange response as responses and 0; why: the streaming chat emits cold queue feedback then preserves upstream sse test consumes this named precondition before exercising the behavior. + response = responses[0] + # What: assert that response status code equals 200; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + # What: assert that response headers content type startswith text event stream; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert response.headers["content-type"].startswith("text/event-stream") + # What: assert that response headers x ft request id equals cold feedback; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert response.headers["x-ft-request-id"] == "cold-feedback" + # What: act by calling response.content.decode and capture content; why: the streaming chat emits cold queue feedback then preserves upstream sse test asserts the response, state, or failure produced by this call. + content = response.content.decode("utf-8") + # What: assert that reasoning content freetoken swap loading model high n is present in content; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert '"reasoning_content":"freetoken-swap loading model: high\\n"' in content + # What: assert that reasoning content n queue position 1 is present in content; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert '"reasoning_content":"\\nQueue position: #1 "' in content + # What: assert that response content endswith upstream body; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert response.content.endswith(upstream_body) + # What: assert that json loads seen body temperature equals 0 2; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert json.loads(seen["body"])["temperature"] == 0.2 + # What: assert that router status reserved requests equals 0; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 0 + # What: assert that router status active requests equals 0; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + # What: assert that router status terminal streams equals 1; why: this assertion protects the streaming chat emits cold queue feedback then preserves upstream sse regression after the test's arranged inputs and exercised call. + assert router.status()["terminalStreams"] == 1 + + +# What: define the test_loading_feedback_warm_path_and_per_model_disable_preserve_exact_response test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the loading feedback warm path and per model disable preserve exact response outcome. +def test_loading_feedback_warm_path_and_per_model_disable_preserve_exact_response(monkeypatch): + # What: arrange upstream body as the fixture input; why: the loading feedback warm path and per model disable preserve exact response test consumes this named precondition before exercising the behavior. + upstream_body = b'data: {"token":"unchanged"}\n\ndata: [DONE]\n\n' + + # What: define the upstream test helper around captured fixture state; why: the loading feedback warm path and per model disable preserve exact response scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: return upstream response and bytes io and upstream body and 201 and content type from the upstream test helper; why: the loading feedback warm path and per model disable preserve exact response scenario uses this helper result in its subsequent act or assertion. + return UpstreamResponse( + # What: arrange the grouped expression portion of the enclosing predicate; why: this clause remains in the loading feedback warm path and per model disable preserve exact response scenario\'s enclosing expression so its grouping and evaluation order stay intact. + 201, + # What: arrange Content Type text event stream X Engine exact for the scenario; why: test router test loading feedback warm path and per model disable preserve exact response requires this concrete input or helper state before exercising the behavior under test. + {"Content-Type": "text/event-stream", "X-Engine": "exact"}, + # What: act by calling BytesIO with upstream body; why: the loading feedback warm path and per model disable preserve exact response scenario observes the BytesIO return value while evaluating BytesIO(upstream_body). + BytesIO(upstream_body), + # What: arrange the grouped source fragment for the scenario; why: test router test loading feedback warm path and per model disable preserve exact response requires this concrete input or helper state before exercising the behavior under test. + ) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the loading feedback warm path and per model disable preserve exact response scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) befor. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: act across the computed value to perform manager and manager; why: the loading feedback warm path and per model disable preserve exact response scenario repeats the body only while or for the loop header admits an iteration. + for warm, override in ((True, None), (False, False)): + # What: act by calling Manager and capture manager; why: the loading feedback warm path and per model disable preserve exact response test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelProfile and capture profile; why: the loading feedback warm path and per model disable preserve exact response test asserts the response, state, or failure produced by this call. + profile = ModelProfile("low", "low.gguf", (), send_loading_state=override) + # What: act by calling ModelCatalog and capture catalog doc; why: the loading feedback warm path and per model disable preserve exact response test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as profile; why: test_loading_feedback_warm_path_and_per_model_disable_preserve_exact_response carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": profile}, settings=RouterSettings(send_loading_state=True) + # What: arrange the ModelCatalog call with settings; why: test_loading_feedback_warm_path_and_per_model_disable_preserve_exact_response groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the loading feedback warm path and per model disable preserve exact response test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: act on warm before release and acquire and router; why: the loading feedback warm path and per model disable preserve exact response scenario admits release and acquire and router only for this predicate and excludes the opposite state. + if warm: + # What: arrange the exact router acquire low release fixture fragment; why: the loading feedback warm path and per model disable preserve exact response scenario feeds this byte-preserved fragment through router.acquire("low").release() before asserting its protocol or parser result. + router.acquire("low").release() + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_loading_feedback_warm_path_and_per_model_disable_preserve_exact_response releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the loading feedback warm path and per model disable preserve exact response test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_loading_feedback_warm_path_and_per_model_disable_preserve_exact_response; why: test_loading_feedback_warm_path_and_per_model_disable_preserve_exact_response consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the loading feedback warm path and per model disable preserve exact response scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_loading_feedback_warm_path_and_per_model_disable_preserve_exact_response groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture response; why: the loading feedback warm path and per model disable preserve exact response test asserts the response, state, or failure produced by this call. + response = TestClient(app).post( + # What: arrange the v1 chat completions portion of response; why: the loading feedback warm path and per model disable preserve exact response scenario uses this clause to evaluate response as one grouped value. + "/v1/chat/completions", + # What: arrange the model field as low; why: test_loading_feedback_warm_path_and_per_model_disable_preserve_exact_response sends this field through response so the router selects the canonical model or alias for upstream dispatch. + json={"model": "low", "stream": True, "messages": []}, + # What: arrange the operation.post call with json; why: test_loading_feedback_warm_path_and_per_model_disable_preserve_exact_response groups the supplied clauses as one operation.post call before its value is consumed. + ) + # What: assert that response status code equals 201; why: this assertion protects the loading feedback warm path and per model disable preserve exact response regression after the test's arranged inputs and exercised call. + assert response.status_code == 201 + # What: assert that response headers x engine equals exact; why: this assertion protects the loading feedback warm path and per model disable preserve exact response regression after the test's arranged inputs and exercised call. + assert response.headers["x-engine"] == "exact" + # What: assert that response content equals upstream body; why: this assertion protects the loading feedback warm path and per model disable preserve exact response regression after the test's arranged inputs and exercised call. + assert response.content == upstream_body + + +# What: define the test_loading_feedback_never_turns_concurrency_rejection_into_sse test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the loading feedback never turns concurrency rejection into sse outcome. +def test_loading_feedback_never_turns_concurrency_rejection_into_sse(monkeypatch): + # What: act by calling Manager and capture manager; why: the loading feedback never turns concurrency rejection into sse test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelProfile and capture profile; why: the loading feedback never turns concurrency rejection into sse test asserts the response, state, or failure produced by this call. + profile = ModelProfile("low", "low.gguf", (), concurrency_limit=1) + # What: act by calling ModelCatalog and capture catalog doc; why: the loading feedback never turns concurrency rejection into sse test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as profile; why: test_loading_feedback_never_turns_concurrency_rejection_into_sse carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": profile}, settings=RouterSettings(send_loading_state=True) + # What: arrange the ModelCatalog call with settings; why: test_loading_feedback_never_turns_concurrency_rejection_into_sse groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the loading feedback never turns concurrency rejection into sse test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: act by calling router.acquire and capture active; why: the loading feedback never turns concurrency rejection into sse test asserts the response, state, or failure produced by this call. + active = router.acquire("low") + # What: arrange monkeypatch setattr for the scenario; why: test loading feedback never turns concurrency rejection into sse requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the loading feedback never turns concurrency rejection into sse scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the exact lambda kwargs pytest fail over limit request reached fixture fragment; why: the loading feedback never turns concurrency rejection into sse scenario feeds this byte-preserved fragment through lambda **kwargs: pytest.fail("over-limit request reached upstream") before asserting its prot. + lambda **kwargs: pytest.fail("over-limit request reached upstream"), + # What: arrange the monkeypatch.setattr call with fail; why: test_loading_feedback_never_turns_concurrency_rejection_into_sse groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_loading_feedback_never_turns_concurrency_rejection_into_sse releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the loading feedback never turns concurrency rejection into sse test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_loading_feedback_never_turns_concurrency_rejection_into_sse; why: test_loading_feedback_never_turns_concurrency_rejection_into_sse consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the loading feedback never turns concurrency rejection into sse scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_loading_feedback_never_turns_concurrency_rejection_into_sse groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture response; why: the loading feedback never turns concurrency rejection into sse test asserts the response, state, or failure produced by this call. + response = TestClient(app).post( + # What: arrange the v1 chat completions portion of response; why: the loading feedback never turns concurrency rejection into sse scenario uses this clause to evaluate response as one grouped value. + "/v1/chat/completions", + # What: arrange the model field as low; why: test_loading_feedback_never_turns_concurrency_rejection_into_sse sends this field through response so the router selects the canonical model or alias for upstream dispatch. + json={"model": "low", "stream": True, "messages": []}, + # What: arrange the operation.post call with json; why: test_loading_feedback_never_turns_concurrency_rejection_into_sse groups the supplied clauses as one operation.post call before its value is consumed. + ) + + # What: assert that response status code equals 429; why: this assertion protects the loading feedback never turns concurrency rejection into sse regression after the test's arranged inputs and exercised call. + assert response.status_code == 429 + # What: assert that response headers content type startswith application json; why: this assertion protects the loading feedback never turns concurrency rejection into sse regression after the test's arranged inputs and exercised call. + assert response.headers["content-type"].startswith("application/json") + # What: assert that response headers retry after equals 1; why: this assertion protects the loading feedback never turns concurrency rejection into sse regression after the test's arranged inputs and exercised call. + assert response.headers["retry-after"] == "1" + # What: assert that response json error type equals concurrency limit; why: this assertion protects the loading feedback never turns concurrency rejection into sse regression after the test's arranged inputs and exercised call. + assert response.json()["error"]["type"] == "concurrency_limit" + # What: assert that b loading model is absent from response content; why: this assertion protects the loading feedback never turns concurrency rejection into sse regression after the test's arranged inputs and exercised call. + assert b"loading model" not in response.content + # What: act by calling active.release with the declared inputs; why: the loading feedback never turns concurrency rejection into sse scenario observes the active.release return value during the enclosing return. + active.release() + + +# What: define the test_loading_feedback_frames_activation_failure_and_done test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the loading feedback frames activation failure and done outcome. +def test_loading_feedback_frames_activation_failure_and_done(monkeypatch): + # What: act by calling Manager and capture manager; why: the loading feedback frames activation failure and done test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the loading feedback frames activation failure and done test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf; why: test_loading_feedback_frames_activation_failure_and_done carries low through catalog doc into router routing coordinator manager catalog doc object ready fn fail ready. + {"low": ModelProfile("low", "low.gguf", ())}, + # What: arrange settings to RouterSettings; why: the loading feedback frames activation failure and done scenario binds this router settings and true value to RouterSettings's settings input. + settings=RouterSettings(send_loading_state=True), + # What: arrange the ModelCatalog call with settings; why: test_loading_feedback_frames_activation_failure_and_done groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + + # What: define the fail_ready test helper around manager and probe and pid and port and timeout s; why: the loading feedback frames activation failure and done scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def fail_ready(manager, probe, *, pid, port, timeout_s): + # What: act by calling time.sleep with 0 05; why: the loading feedback frames activation failure and done scenario observes the time.sleep return value during return ready reason qualification failed. + time.sleep(0.05) + # What: arrange the ready field as false; why: fail_ready carries ready into return {"ready": False, "reason": "qualification failed"}. + return {"ready": False, "reason": "qualification failed"} + + # What: act by calling RoutingCoordinator and capture router; why: the loading feedback frames activation failure and done test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=fail_ready) + # What: arrange monkeypatch setattr for the scenario; why: test loading feedback frames activation failure and done requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the loading feedback frames activation failure and done scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the exact lambda kwargs pytest fail failed activation reached fixture fragment; why: the loading feedback frames activation failure and done scenario feeds this byte-preserved fragment through lambda **kwargs: pytest.fail("failed activation reached upstream") before asserting its protocol or pa. + lambda **kwargs: pytest.fail("failed activation reached upstream"), + # What: arrange the monkeypatch.setattr call with fail; why: test_loading_feedback_frames_activation_failure_and_done groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_loading_feedback_frames_activation_failure_and_done releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the loading feedback frames activation failure and done test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_loading_feedback_frames_activation_failure_and_done; why: test_loading_feedback_frames_activation_failure_and_done consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the loading feedback frames activation failure and done scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_loading_feedback_frames_activation_failure_and_done groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture response; why: the loading feedback frames activation failure and done test asserts the response, state, or failure produced by this call. + response = TestClient(app).post( + # What: arrange the v1 chat completions portion of response; why: the loading feedback frames activation failure and done scenario uses this clause to evaluate response as one grouped value. + "/v1/chat/completions", + # What: arrange the model field as low; why: test_loading_feedback_frames_activation_failure_and_done sends this field through response so the router selects the canonical model or alias for upstream dispatch. + json={"model": "low", "stream": True, "messages": []}, + # What: arrange the operation.post call with json; why: test_loading_feedback_frames_activation_failure_and_done groups the supplied clauses as one operation.post call before its value is consumed. + ) + + # What: assert that response status code equals 200; why: this assertion protects the loading feedback frames activation failure and done regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + # What: assert that response headers content type startswith text event stream; why: this assertion protects the loading feedback frames activation failure and done regression after the test's arranged inputs and exercised call. + assert response.headers["content-type"].startswith("text/event-stream") + # What: assert that b qualification failed is present in response content; why: this assertion protects the loading feedback frames activation failure and done regression after the test's arranged inputs and exercised call. + assert b"qualification failed" in response.content + # What: assert that b type engine not ready is present in response content; why: this assertion protects the loading feedback frames activation failure and done regression after the test's arranged inputs and exercised call. + assert b'"type":"engine_not_ready"' in response.content + # What: assert that response content endswith b data done n n; why: this assertion protects the loading feedback frames activation failure and done regression after the test's arranged inputs and exercised call. + assert response.content.endswith(b"data: [DONE]\n\n") + # What: assert the expected all outcome; why: test router test loading feedback frames activation failure and done protects its regression by requiring this observable result after the exercised behavior. + assert all( + # What: arrange not line or line startswith b data for the scenario; why: test router test loading feedback frames activation failure and done requires this concrete input or helper state before exercising the behavior under test. + not line or line.startswith(b"data: ") + # What: arrange for line in response content rstrip splitlines for the scenario; why: test router test loading feedback frames activation failure and done requires this concrete input or helper state before exercising the behavior under test. + for line in response.content.rstrip().splitlines() + # What: arrange the grouped source fragment for the scenario; why: test router test loading feedback frames activation failure and done requires this concrete input or helper state before exercising the behavior under test. + ) + # What: assert that router status reserved requests equals 0; why: this assertion protects the loading feedback frames activation failure and done regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 0 + # What: assert that router status active requests equals 0; why: this assertion protects the loading feedback frames activation failure and done regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + + +# What: define the test_loading_feedback_frames_upstream_connect_failure_and_releases_lease test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the loading feedback frames upstream connect failure and releases lease outcome. +def test_loading_feedback_frames_upstream_connect_failure_and_releases_lease(monkeypatch): + # What: act by calling Manager and capture manager; why: the loading feedback frames upstream connect failure and releases lease test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the loading feedback frames upstream connect failure and releases lease test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf; why: test_loading_feedback_frames_upstream_connect_failure_and_releases_lease carries low through catalog doc into router routing coordinator manager catalog doc object ready fn slow ready. + {"low": ModelProfile("low", "low.gguf", ())}, + # What: arrange settings to RouterSettings; why: the loading feedback frames upstream connect failure and releases lease scenario binds this router settings and true value to RouterSettings's settings input. + settings=RouterSettings(send_loading_state=True), + # What: arrange the ModelCatalog call with settings; why: test_loading_feedback_frames_upstream_connect_failure_and_releases_lease groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + + # What: define the slow_ready test helper around manager and probe and pid and port and timeout s; why: the loading feedback frames upstream connect failure and releases lease scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def slow_ready(manager, probe, *, pid, port, timeout_s): + # What: act by calling time.sleep with 0 05; why: the loading feedback frames upstream connect failure and releases lease scenario observes the time.sleep return value during return ready health status ok. + time.sleep(0.05) + # What: arrange the ready field as true; why: slow_ready carries ready into return {"ready": True, "health": {"status": "ok"}}. + return {"ready": True, "health": {"status": "ok"}} + + # What: act by calling RoutingCoordinator and capture router; why: the loading feedback frames upstream connect failure and releases lease test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=slow_ready) + # What: arrange monkeypatch setattr for the scenario; why: test loading feedback frames upstream connect failure and releases lease requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the loading feedback frames upstream connect failure and releases lease scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the exact lambda kwargs value for value in fixture fragment; why: the loading feedback frames upstream connect failure and releases lease scenario feeds this byte-preserved fragment through lambda **kwargs: (_ for _ in ()).throw(OSError("connection refused")) before asserting its protocol or pa. + lambda **kwargs: (_ for _ in ()).throw(OSError("connection refused")), + # What: arrange the monkeypatch.setattr call with throw; why: test_loading_feedback_frames_upstream_connect_failure_and_releases_lease groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_loading_feedback_frames_upstream_connect_failure_and_releases_lease releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the loading feedback frames upstream connect failure and releases lease test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_loading_feedback_frames_upstream_connect_failure_and_releases_lease; why: test_loading_feedback_frames_upstream_connect_failure_and_releases_lease consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the loading feedback frames upstream connect failure and releases lease scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_loading_feedback_frames_upstream_connect_failure_and_releases_lease groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture response; why: the loading feedback frames upstream connect failure and releases lease test asserts the response, state, or failure produced by this call. + response = TestClient(app).post( + # What: arrange the v1 chat completions portion of response; why: the loading feedback frames upstream connect failure and releases lease scenario uses this clause to evaluate response as one grouped value. + "/v1/chat/completions", + # What: arrange the model field as low; why: test_loading_feedback_frames_upstream_connect_failure_and_releases_lease sends this field through response so the router selects the canonical model or alias for upstream dispatch. + json={"model": "low", "stream": True, "messages": []}, + # What: arrange the operation.post call with json; why: test_loading_feedback_frames_upstream_connect_failure_and_releases_lease groups the supplied clauses as one operation.post call before its value is consumed. + ) + + # What: assert that response status code equals 200; why: this assertion protects the loading feedback frames upstream connect failure and releases lease regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + # What: assert that b connection refused is present in response content; why: this assertion protects the loading feedback frames upstream connect failure and releases lease regression after the test's arranged inputs and exercised call. + assert b"connection refused" in response.content + # What: assert that b type upstream unavailable is present in response content; why: this assertion protects the loading feedback frames upstream connect failure and releases lease regression after the test's arranged inputs and exercised call. + assert b'"type":"upstream_unavailable"' in response.content + # What: assert that response content endswith b data done n n; why: this assertion protects the loading feedback frames upstream connect failure and releases lease regression after the test's arranged inputs and exercised call. + assert response.content.endswith(b"data: [DONE]\n\n") + # What: assert that router status active requests equals 0; why: this assertion protects the loading feedback frames upstream connect failure and releases lease regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + # What: assert that router status reserved requests equals 0; why: this assertion protects the loading feedback frames upstream connect failure and releases lease regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 0 + + +# What: define the test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the loading feedback explicit queue cancellation is in band and releases outcome. +def test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases(monkeypatch): + # What: act by calling Manager and capture manager; why: the loading feedback explicit queue cancellation is in band and releases test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the loading feedback explicit queue cancellation is in band and releases test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with low and high; why: test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases groups the supplied clauses as one catalog_doc mapping before its value. + { + # What: arrange the low field as model profile and low and low and gguf; why: test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "low": ModelProfile("low", "low.gguf", ()), + # What: arrange the high field as model profile and high and high and gguf; why: test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases carries high through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "high": ModelProfile("high", "high.gguf", ()), + # What: arrange the catalog_doc mapping with low and high; why: test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases groups the supplied clauses as one catalog_doc mapping before its value. + }, + # What: arrange settings to RouterSettings; why: the loading feedback explicit queue cancellation is in band and releases scenario binds this router settings and true value to RouterSettings's settings input. + settings=RouterSettings(send_loading_state=True), + # What: arrange the ModelCatalog call with settings; why: test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the loading feedback explicit queue cancellation is in band and releases test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: act by calling router.acquire and capture active; why: the loading feedback explicit queue cancellation is in band and releases test asserts the response, state, or failure produced by this call. + active = router.acquire("low") + # What: arrange monkeypatch setattr for the scenario; why: test loading feedback explicit queue cancellation is in band and releases requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the loading feedback explicit queue cancellation is in band and releases scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the exact lambda kwargs pytest fail cancelled request reached fixture fragment; why: the loading feedback explicit queue cancellation is in band and releases scenario feeds this byte-preserved fragment through lambda **kwargs: pytest.fail("cancelled request reached upstream") before asserting i. + lambda **kwargs: pytest.fail("cancelled request reached upstream"), + # What: arrange the monkeypatch.setattr call with fail; why: test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + # What: arrange responses as the fixture input; why: the loading feedback explicit queue cancellation is in band and releases test consumes this named precondition before exercising the behavior. + responses = [] + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(2) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the loading feedback explicit queue cancellation is in band and releases test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases; why: test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the loading feedback explicit queue cancellation is in band and releases scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the loading feedback explicit queue cancellation is in band and releases test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling threading.Thread and capture thread; why: the loading feedback explicit queue cancellation is in band and releases test asserts the response, state, or failure produced by this call. + thread = threading.Thread(target=lambda: responses.append(client.post( + # What: arrange the v1 chat completions portion of thread; why: the loading feedback explicit queue cancellation is in band and releases scenario uses this clause to evaluate thread as one grouped value. + "/v1/chat/completions", + # What: arrange the model field as high; why: test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases sends this field through thread so the router selects the canonical model or alias for upstream dispatch. + json={"model": "high", "stream": True, "messages": []}, + # What: arrange the x ft request id field as cancel loading; why: test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases carries x ft request id through thread into thread start. + headers={"X-FT-Request-ID": "cancel-loading"}, + # What: arrange the threading.Thread call with target; why: test_loading_feedback_explicit_queue_cancellation_is_in_band_and_releases groups the supplied clauses as one threading.Thread call before its value is consumed. + ))) + # What: act by calling thread.start with the declared inputs; why: the loading feedback explicit queue cancellation is in band and releases scenario observes the thread.start return value during for value in range. + thread.start() + # What: act across range to perform status and router; why: the loading feedback explicit queue cancellation is in band and releases scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on status and router before the computed value; why: the loading feedback explicit queue cancellation is in band and releases scenario admits the computed value only for this predicate and excludes the opposite state. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the loading feedback explicit queue cancellation is in band and releases scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling time.sleep with 0 01; why: the loading feedback explicit queue cancellation is in band and releases scenario observes the time.sleep return value during assert router status queued requests. + time.sleep(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the loading feedback explicit queue cancellation is in band and releases regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + # What: act by calling client.post and capture cancelled; why: the loading feedback explicit queue cancellation is in band and releases test asserts the response, state, or failure produced by this call. + cancelled = client.post("/router/requests/cancel-loading/cancel") + # What: assert that cancelled json equals cancelled true id cancel loading; why: this assertion protects the loading feedback explicit queue cancellation is in band and releases regression after the test's arranged inputs and exercised call. + assert cancelled.json() == {"cancelled": True, "id": "cancel-loading"} + # What: act by calling thread.join with 3; why: the loading feedback explicit queue cancellation is in band and releases scenario observes the thread.join return value during assert not thread is alive. + thread.join(3) + # What: assert that thread is alive is false; why: this assertion protects the loading feedback explicit queue cancellation is in band and releases regression after the test's arranged inputs and exercised call. + assert not thread.is_alive() + + # What: assert that responses 0 status code equals 200; why: this assertion protects the loading feedback explicit queue cancellation is in band and releases regression after the test's arranged inputs and exercised call. + assert responses[0].status_code == 200 + # What: assert that b type request cancelled is present in responses 0 content; why: this assertion protects the loading feedback explicit queue cancellation is in band and releases regression after the test's arranged inputs and exercised call. + assert b'"type":"request_cancelled"' in responses[0].content + # What: assert that responses 0 content endswith b data done n n; why: this assertion protects the loading feedback explicit queue cancellation is in band and releases regression after the test's arranged inputs and exercised call. + assert responses[0].content.endswith(b"data: [DONE]\n\n") + # What: assert that router status queued requests equals 0; why: this assertion protects the loading feedback explicit queue cancellation is in band and releases regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 0 + # What: assert that router status reserved requests equals 1; why: this assertion protects the loading feedback explicit queue cancellation is in band and releases regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 1 + # What: act by calling active.release with the declared inputs; why: the loading feedback explicit queue cancellation is in band and releases scenario observes the active.release return value during the enclosing return. + active.release() + + +# What: define the test_loading_feedback_cancellation_during_activation_is_not_completion_credit test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the loading feedback cancellation during activation is not completion credit outcome. +def test_loading_feedback_cancellation_during_activation_is_not_completion_credit(monkeypatch): + # What: act by calling Manager and capture manager; why: the loading feedback cancellation during activation is not completion credit test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the loading feedback cancellation during activation is not completion credit test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf; why: test_loading_feedback_cancellation_during_activation_is_not_completion_credit carries low through catalog doc into router routing coordinator manager catalog doc object ready fn blocking ready. + {"low": ModelProfile("low", "low.gguf", ())}, + # What: arrange settings to RouterSettings; why: the loading feedback cancellation during activation is not completion credit scenario binds this router settings and true value to RouterSettings's settings input. + settings=RouterSettings(send_loading_state=True), + # What: arrange the ModelCatalog call with settings; why: test_loading_feedback_cancellation_during_activation_is_not_completion_credit groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling threading.Event and capture activation started; why: the loading feedback cancellation during activation is not completion credit test asserts the response, state, or failure produced by this call. + activation_started = threading.Event() + # What: act by calling threading.Event and capture finish activation; why: the loading feedback cancellation during activation is not completion credit test asserts the response, state, or failure produced by this call. + finish_activation = threading.Event() + + # What: define the blocking_ready test helper around manager and probe and pid and port and timeout s; why: the loading feedback cancellation during activation is not completion credit scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def blocking_ready(manager, probe, *, pid, port, timeout_s): + # What: act by calling activation_started.set with the declared inputs; why: the loading feedback cancellation during activation is not completion credit scenario observes the activation_started.set return value during assert finish activation wait. + activation_started.set() + # What: assert that finish activation wait 2; why: this assertion protects the loading feedback cancellation during activation is not completion credit regression after the test's arranged inputs and exercised call. + assert finish_activation.wait(2) + # What: arrange the ready field as true; why: blocking_ready carries ready into return {"ready": True, "health": {"status": "ok"}}. + return {"ready": True, "health": {"status": "ok"}} + + # What: act by calling RoutingCoordinator and capture router; why: the loading feedback cancellation during activation is not completion credit test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=blocking_ready) + # What: arrange monkeypatch setattr for the scenario; why: test loading feedback cancellation during activation is not completion credit requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the loading feedback cancellation during activation is not completion credit scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the exact lambda kwargs pytest fail cancelled activation reached fixture fragment; why: the loading feedback cancellation during activation is not completion credit scenario feeds this byte-preserved fragment through lambda **kwargs: pytest.fail("cancelled activation reached upstream") before a. + lambda **kwargs: pytest.fail("cancelled activation reached upstream"), + # What: arrange the monkeypatch.setattr call with fail; why: test_loading_feedback_cancellation_during_activation_is_not_completion_credit groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + # What: arrange responses as the fixture input; why: the loading feedback cancellation during activation is not completion credit test consumes this named precondition before exercising the behavior. + responses = [] + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_loading_feedback_cancellation_during_activation_is_not_completion_credit releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(2) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the loading feedback cancellation during activation is not completion credit test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_loading_feedback_cancellation_during_activation_is_not_completion_credit; why: test_loading_feedback_cancellation_during_activation_is_not_completion_credit consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the loading feedback cancellation during activation is not completion credit scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_loading_feedback_cancellation_during_activation_is_not_completion_credit groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the loading feedback cancellation during activation is not completion credit test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling threading.Thread and capture thread; why: the loading feedback cancellation during activation is not completion credit test asserts the response, state, or failure produced by this call. + thread = threading.Thread(target=lambda: responses.append(client.post( + # What: arrange the v1 chat completions portion of thread; why: the loading feedback cancellation during activation is not completion credit scenario uses this clause to evaluate thread as one grouped value. + "/v1/chat/completions", + # What: arrange the model field as low; why: test_loading_feedback_cancellation_during_activation_is_not_completion_credit sends this field through thread so the router selects the canonical model or alias for upstream dispatch. + json={"model": "low", "stream": True, "messages": []}, + # What: arrange the x ft request id field as cancel activation; why: test_loading_feedback_cancellation_during_activation_is_not_completion_credit carries x ft request id through thread into thread start. + headers={"X-FT-Request-ID": "cancel-activation"}, + # What: arrange the threading.Thread call with target; why: test_loading_feedback_cancellation_during_activation_is_not_completion_credit groups the supplied clauses as one threading.Thread call before its value is consumed. + ))) + # What: act by calling thread.start with the declared inputs; why: the loading feedback cancellation during activation is not completion credit scenario observes the thread.start return value during assert activation started wait. + thread.start() + # What: assert that activation started wait 1; why: this assertion protects the loading feedback cancellation during activation is not completion credit regression after the test's arranged inputs and exercised call. + assert activation_started.wait(1) + # What: act by calling client.post and capture cancelled; why: the loading feedback cancellation during activation is not completion credit test asserts the response, state, or failure produced by this call. + cancelled = client.post("/router/requests/cancel-activation/cancel") + # What: assert that cancelled json equals cancelled true id cancel activation; why: this assertion protects the loading feedback cancellation during activation is not completion credit regression after the test's arranged inputs and exercised call. + assert cancelled.json() == {"cancelled": True, "id": "cancel-activation"} + # What: act by calling finish_activation.set with the declared inputs; why: the loading feedback cancellation during activation is not completion credit scenario observes the finish_activation.set return value during thread join. + finish_activation.set() + # What: act by calling thread.join with 3; why: the loading feedback cancellation during activation is not completion credit scenario observes the thread.join return value during assert not thread is alive. + thread.join(3) + # What: assert that thread is alive is false; why: this assertion protects the loading feedback cancellation during activation is not completion credit regression after the test's arranged inputs and exercised call. + assert not thread.is_alive() + + # What: assert that responses 0 status code equals 200; why: this assertion protects the loading feedback cancellation during activation is not completion credit regression after the test's arranged inputs and exercised call. + assert responses[0].status_code == 200 + # What: assert that b type request cancelled is present in responses 0 content; why: this assertion protects the loading feedback cancellation during activation is not completion credit regression after the test's arranged inputs and exercised call. + assert b'"type":"request_cancelled"' in responses[0].content + # What: assert that responses 0 content endswith b data done n n; why: this assertion protects the loading feedback cancellation during activation is not completion credit regression after the test's arranged inputs and exercised call. + assert responses[0].content.endswith(b"data: [DONE]\n\n") + # What: assert that router status active requests equals 0; why: this assertion protects the loading feedback cancellation during activation is not completion credit regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + # What: assert that router status reserved requests equals 0; why: this assertion protects the loading feedback cancellation during activation is not completion credit regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 0 + # What: assert that router status terminal streams equals 0; why: this assertion protects the loading feedback cancellation during activation is not completion credit regression after the test's arranged inputs and exercised call. + assert router.status()["terminalStreams"] == 0 + # What: assert that router status cancellations equals 1; why: this assertion protects the loading feedback cancellation during activation is not completion credit regression after the test's arranged inputs and exercised call. + assert router.status()["cancellations"] == 1 + + +# What: define the test_loading_feedback_disconnect_cancels_queued_ownership test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the loading feedback disconnect cancels queued ownership outcome. +def test_loading_feedback_disconnect_cancels_queued_ownership(monkeypatch): + # What: act by calling Manager and capture manager; why: the loading feedback disconnect cancels queued ownership test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the loading feedback disconnect cancels queued ownership test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the catalog_doc mapping with low and high; why: test_loading_feedback_disconnect_cancels_queued_ownership groups the supplied clauses as one catalog_doc mapping before its value is consumed. + { + # What: arrange the low field as model profile and low and low and gguf; why: test_loading_feedback_disconnect_cancels_queued_ownership carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "low": ModelProfile("low", "low.gguf", ()), + # What: arrange the high field as model profile and high and high and gguf; why: test_loading_feedback_disconnect_cancels_queued_ownership carries high through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "high": ModelProfile("high", "high.gguf", ()), + # What: arrange the catalog_doc mapping with low and high; why: test_loading_feedback_disconnect_cancels_queued_ownership groups the supplied clauses as one catalog_doc mapping before its value is consumed. + }, + # What: arrange settings to RouterSettings; why: the loading feedback disconnect cancels queued ownership scenario binds this router settings and true value to RouterSettings's settings input. + settings=RouterSettings(send_loading_state=True), + # What: arrange the ModelCatalog call with settings; why: test_loading_feedback_disconnect_cancels_queued_ownership groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the loading feedback disconnect cancels queued ownership test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: act by calling router.acquire and capture active; why: the loading feedback disconnect cancels queued ownership test asserts the response, state, or failure produced by this call. + active = router.acquire("low") + # What: arrange monkeypatch setattr for the scenario; why: test loading feedback disconnect cancels queued ownership requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the loading feedback disconnect cancels queued ownership scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the exact lambda kwargs pytest fail disconnected request reached fixture fragment; why: the loading feedback disconnect cancels queued ownership scenario feeds this byte-preserved fragment through lambda **kwargs: pytest.fail("disconnected request reached upstream") before asserting its protoco. + lambda **kwargs: pytest.fail("disconnected request reached upstream"), + # What: arrange the monkeypatch.setattr call with fail; why: test_loading_feedback_disconnect_cancels_queued_ownership groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + + # What: define the scenario test helper around app; why: the loading feedback disconnect cancels queued ownership scenario calls this helper to produce or observe the exact behavior checked by its assertions. + async def scenario(app): + # What: act by calling operation.encode and capture body; why: the loading feedback disconnect cancels queued ownership test asserts the response, state, or failure produced by this call. + body = json.dumps({"model": "high", "stream": True, "messages": []}).encode() + # What: act by calling asyncio.Event and capture disconnect; why: the loading feedback disconnect cancels queued ownership test asserts the response, state, or failure produced by this call. + disconnect = asyncio.Event() + # What: arrange request sent as false; why: the loading feedback disconnect cancels queued ownership test consumes this named precondition before exercising the behavior. + request_sent = False + # What: arrange sent as the fixture input; why: the loading feedback disconnect cancels queued ownership test consumes this named precondition before exercising the behavior. + sent = [] + + # What: define the receive test helper around captured fixture state; why: the loading feedback disconnect cancels queued ownership scenario calls this helper to produce or observe the exact behavior checked by its assertions. + async def receive(): + # What: arrange the nonlocal request sent portion of the enclosing predicate; why: this clause remains in the loading feedback disconnect cancels queued ownership scenario\'s enclosing expression so its grouping and evaluation order stay intact. + nonlocal request_sent + # What: act on request sent before request sent; why: the loading feedback disconnect cancels queued ownership scenario admits request sent only for this predicate and excludes the opposite state. + if not request_sent: + # What: arrange request sent as true; why: the loading feedback disconnect cancels queued ownership test consumes this named precondition before exercising the behavior. + request_sent = True + # What: arrange the type field as http and request; why: receive carries type into return {"type": "http.request", "body": body, "more_body": False}. + return {"type": "http.request", "body": body, "more_body": False} + # What: act by calling disconnect.wait with the declared inputs; why: the loading feedback disconnect cancels queued ownership scenario observes the disconnect.wait return value during return type http disconnect. + await disconnect.wait() + # What: arrange the type field as http and disconnect; why: receive carries type into return {"type": "http.disconnect"}. + return {"type": "http.disconnect"} + + # What: define the send test helper around message; why: the loading feedback disconnect cancels queued ownership scenario calls this helper to produce or observe the exact behavior checked by its assertions. + async def send(message): + # What: act by calling sent.append with message; why: the loading feedback disconnect cancels queued ownership scenario observes the sent.append return value during the enclosing return. + sent.append(message) + + # What: act by calling operation.encode and capture scope; why: the loading feedback disconnect cancels queued ownership test asserts the response, state, or failure produced by this call. + scope = { + # What: arrange the version field as 3 0; why: scenario carries version through scope into request asyncio create task app scope receive send. + "type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1", + # What: arrange the method field as post; why: scenario carries method through scope into request asyncio create task app scope receive send. + "method": "POST", "scheme": "http", "path": "/v1/chat/completions", + # What: arrange the raw path field as the fixture input; why: scenario carries raw path through scope into request asyncio create task app scope receive send. + "raw_path": b"/v1/chat/completions", "query_string": b"", "root_path": "", + # What: arrange the headers field as encode and str and len and body; why: scenario carries headers through scope into request asyncio create task app scope receive send. + "headers": [ + # What: arrange the b content type b application json portion of scope; why: the loading feedback disconnect cancels queued ownership scenario uses this clause to evaluate scope as one grouped value. + (b"content-type", b"application/json"), + # What: act by calling operation.encode with the declared inputs; why: the loading feedback disconnect cancels queued ownership scenario observes the operation.encode return value during b x ft request id b disconnect loading. + (b"content-length", str(len(body)).encode()), + # What: arrange the b x ft request id b disconnect loading portion of scope; why: the loading feedback disconnect cancels queued ownership scenario uses this clause to evaluate scope as one grouped value. + (b"x-ft-request-id", b"disconnect-loading"), + # What: arrange the scope collection with the named fixture input and encode and str and len and body and the named fixture input; why: scenario groups the supplied clauses as one scope collection before its value is consumed. + ], + # What: arrange the client field as 127 0 0 1 and 1; why: scenario carries client through scope into request asyncio create task app scope receive send. + "client": ("127.0.0.1", 1), "server": ("127.0.0.1", 80), + # What: arrange the scope mapping with type and asgi and http version and method and scheme; why: scenario groups the supplied clauses as one scope mapping before its value is consumed. + } + # What: act by calling asyncio.create_task and capture request; why: the loading feedback disconnect cancels queued ownership test asserts the response, state, or failure produced by this call. + request = asyncio.create_task(app(scope, receive, send)) + # What: act across range to perform status and router; why: the loading feedback disconnect cancels queued ownership scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: arrange if router status queuedRequests == 1 for the scenario; why: test router test loading feedback disconnect cancels queued ownership requires this concrete input or helper state before exercising the behavior under test. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the loading feedback disconnect cancels queued ownership scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the loading feedback disconnect cancels queued ownership scenario observes the asyncio.sleep return value during assert router status queued requests. + await asyncio.sleep(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the loading feedback disconnect cancels queued ownership regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + # What: act by calling disconnect.set with the declared inputs; why: the loading feedback disconnect cancels queued ownership scenario observes the disconnect.set return value during await asyncio wait for request. + disconnect.set() + # What: act by calling asyncio.wait_for with request and 2; why: the loading feedback disconnect cancels queued ownership scenario observes the asyncio.wait_for return value during for value in range. + await asyncio.wait_for(request, 2) + # What: act across range to perform status and router; why: the loading feedback disconnect cancels queued ownership scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: arrange if router status queuedRequests == 0 for the scenario; why: test router test loading feedback disconnect cancels queued ownership requires this concrete input or helper state before exercising the behavior under test. + if router.status()["queuedRequests"] == 0: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the loading feedback disconnect cancels queued ownership scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the loading feedback disconnect cancels queued ownership scenario observes the asyncio.sleep return value during assert router status queued requests. + await asyncio.sleep(0.01) + # What: assert that router status queued requests equals 0; why: this assertion protects the loading feedback disconnect cancels queued ownership regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 0 + # What: assert that any message type equals http response start for message in sent; why: this assertion protects the loading feedback disconnect cancels queued ownership regression after the test's arranged inputs and exercised call. + assert any(message["type"] == "http.response.start" for message in sent) + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_loading_feedback_disconnect_cancels_queued_ownership releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(2) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the loading feedback disconnect cancels queued ownership test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_loading_feedback_disconnect_cancels_queued_ownership; why: test_loading_feedback_disconnect_cancels_queued_ownership consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the loading feedback disconnect cancels queued ownership scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_loading_feedback_disconnect_cancels_queued_ownership groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling asyncio.run with scenario and app; why: the loading feedback disconnect cancels queued ownership scenario observes the asyncio.run return value during assert router status reserved requests. + asyncio.run(scenario(app)) + + # What: assert that router status reserved requests equals 1; why: this assertion protects the loading feedback disconnect cancels queued ownership regression after the test's arranged inputs and exercised call. + assert router.status()["reservedRequests"] == 1 + # What: assert that router status cancellations equals 1; why: this assertion protects the loading feedback disconnect cancels queued ownership regression after the test's arranged inputs and exercised call. + assert router.status()["cancellations"] == 1 + # What: act by calling active.release with the declared inputs; why: the loading feedback disconnect cancels queued ownership scenario observes the active.release return value during assert manager calls start low gguf. + active.release() + # What: assert that manager calls equals start low gguf; why: this assertion protects the loading feedback disconnect cancels queued ownership regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf")] + + +# What: parameterize test_loading_feedback_is_only_for_strictly_streaming_chat_requests with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test loading feedback is only for strictly streaming chat requests. +@pytest.mark.parametrize( + # What: arrange the path payload portion of the enclosing predicate; why: this clause remains in the loading feedback is only for strictly streaming chat requests scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "path,payload", + # What: arrange the grouped source fragment for the scenario; why: test loading feedback is only for strictly streaming chat requests requires this concrete input or helper state before exercising the behavior under test. + [ + # What: arrange v1 chat completions model low stream False messages for the scenario; why: test router test loading feedback is only for strictly streaming chat requests requires this concrete input or helper state before exercising the behavior under test. + ("/v1/chat/completions", {"model": "low", "stream": False, "messages": []}), + # What: arrange v1 chat completions model low stream 1 messages for the scenario; why: test router test loading feedback is only for strictly streaming chat requests requires this concrete input or helper state before exercising the behavior under test. + ("/v1/chat/completions", {"model": "low", "stream": 1, "messages": []}), + # What: arrange v1 completions model low stream True prompt for the scenario; why: test router test loading feedback is only for strictly streaming chat requests requires this concrete input or helper state before exercising the behavior under test. + ("/v1/completions", {"model": "low", "stream": True, "prompt": ""}), + # What: arrange v1 messages model low stream True messages for the scenario; why: test router test loading feedback is only for strictly streaming chat requests requires this concrete input or helper state before exercising the behavior under test. + ("/v1/messages", {"model": "low", "stream": True, "messages": []}), + # What: arrange the grouped source fragment for the scenario; why: test loading feedback is only for strictly streaming chat requests requires this concrete input or helper state before exercising the behavior under test. + ], +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_loading_feedback_is_only_for_strictly_streaming_chat_requests groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +) +# What: define the test_loading_feedback_is_only_for_strictly_streaming_chat_requests test around monkeypatch and path and payload; why: this test groups the arrange, act, and assertions that protect the loading feedback is only for strictly streaming chat requests outcome. +def test_loading_feedback_is_only_for_strictly_streaming_chat_requests( + # What: arrange the monkeypatch input for test_loading_feedback_is_only_for_strictly_streaming_chat_requests; why: test_loading_feedback_is_only_for_strictly_streaming_chat_requests consumes monkeypatch during monkeypatch setattr, so callers must bind it with the other signature inputs. + monkeypatch, path, payload +# What: arrange the grouped source fragment for the scenario; why: test loading feedback is only for strictly streaming chat requests requires this concrete input or helper state before exercising the. +): + # What: arrange body as the fixture input; why: the loading feedback is only for strictly streaming chat requests test consumes this named precondition before exercising the behavior. + body = b'{"ordinary":true}' + # What: act by calling ModelCatalog and capture catalog doc; why: the loading feedback is only for strictly streaming chat requests test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf; why: test_loading_feedback_is_only_for_strictly_streaming_chat_requests carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": ModelProfile("low", "low.gguf", ())}, + # What: arrange settings to RouterSettings; why: the loading feedback is only for strictly streaming chat requests scenario binds this router settings and true value to RouterSettings's settings input. + settings=RouterSettings(send_loading_state=True), + # What: arrange the ModelCatalog call with settings; why: test_loading_feedback_is_only_for_strictly_streaming_chat_requests groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling Manager and capture manager; why: the loading feedback is only for strictly streaming chat requests test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the loading feedback is only for strictly streaming chat requests test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange monkeypatch setattr for the scenario; why: test loading feedback is only for strictly streaming chat requests requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact freetoken daemon app open upstream fixture fragment; why: the loading feedback is only for strictly streaming chat requests scenario feeds this byte-preserved fragment through "freetoken.daemon.app.open_upstream" before asserting its protocol or parser result. + "freetoken.daemon.app.open_upstream", + # What: arrange the kwargs input for test_loading_feedback_is_only_for_strictly_streaming_chat_requests; why: test_loading_feedback_is_only_for_strictly_streaming_chat_requests consumes kwargs during signature binding, so callers must bind it with the other signature inputs. + lambda **kwargs: UpstreamResponse( + # What: arrange the content type field as application and json; why: test_loading_feedback_is_only_for_strictly_streaming_chat_requests carries content type into 202, {"Content-Type": "application/json", "X-Mode": "ordinary"}, BytesIO. + 202, {"Content-Type": "application/json", "X-Mode": "ordinary"}, BytesIO(body) + # What: arrange the UpstreamResponse call with bytes io; why: test_loading_feedback_is_only_for_strictly_streaming_chat_requests groups the supplied clauses as one UpstreamResponse call before its value is consumed. + ), + # What: arrange the monkeypatch.setattr call with upstream response; why: test_loading_feedback_is_only_for_strictly_streaming_chat_requests groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_loading_feedback_is_only_for_strictly_streaming_chat_requests releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the loading feedback is only for strictly streaming chat requests test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_loading_feedback_is_only_for_strictly_streaming_chat_requests; why: test_loading_feedback_is_only_for_strictly_streaming_chat_requests consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the loading feedback is only for strictly streaming chat requests scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_loading_feedback_is_only_for_strictly_streaming_chat_requests groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture response; why: the loading feedback is only for strictly streaming chat requests test asserts the response, state, or failure produced by this call. + response = TestClient(app).post(path, json=payload) + + # What: assert that response status code equals 202; why: this assertion protects the loading feedback is only for strictly streaming chat requests regression after the test's arranged inputs and exercised call. + assert response.status_code == 202 + # What: assert that response headers x mode equals ordinary; why: this assertion protects the loading feedback is only for strictly streaming chat requests regression after the test's arranged inputs and exercised call. + assert response.headers["x-mode"] == "ordinary" + # What: assert that response content equals body; why: this assertion protects the loading feedback is only for strictly streaming chat requests regression after the test's arranged inputs and exercised call. + assert response.content == body + + +# What: define the test_request_filter_is_explicit_top_level_removal_and_default_is_byte_preserving test around local fixtures; why: this test groups the arrange, act, and assertions that protect the request filter is explicit top level removal and default is byte preserving outcome. +def test_request_filter_is_explicit_top_level_removal_and_default_is_byte_preserving(): + # What: arrange raw as the fixture input; why: the request filter is explicit top level removal and default is byte preserving test consumes this named precondition before exercising the behavior. + raw = b'{"model":"low", "metadata":{"private":true}, "user":"operator"}' + # What: assert that filter request body raw equals raw; why: this assertion protects the request filter is explicit top level removal and default is byte preserving regression after the test's arranged inputs and exercised call. + assert filter_request_body(raw, ()) == raw + # What: assert that filter request body raw metadata user equals b model low; why: this assertion protects the request filter is explicit top level removal and default is byte preserving regression after the test's arranged inputs and exercised call. + assert filter_request_body(raw, ("metadata", "user")) == b'{"model":"low"}' + + +# What: define the test_request_filter_applies_nested_drop_global_and_requested_id_fields_in_order test around local fixtures; why: this test groups the arrange, act, and assertions that protect the request filter applies nested drop global and requested id fields in order outcome. +def test_request_filter_applies_nested_drop_global_and_requested_id_fields_in_order(): + # What: arrange raw as the fixture input; why: the request filter applies nested drop global and requested id fields in order test consumes this named precondition before exercising the behavior. + raw = ( + # What: arrange the b model low high metadata private portion of raw; why: the request filter applies nested drop global and requested id fields in order scenario uses this clause to evaluate raw as one grouped value. + b'{"model":"low:high","metadata":{"private":true,"keep":1},' + # What: arrange the b max tokens top p stream false stop portion of raw; why: the request filter applies nested drop global and requested id fields in order scenario uses this clause to evaluate raw as one grouped value. + b'"max_tokens":7,"top_p":0.9,"stream":false,"stop":null,' + # What: arrange the b chat template kwargs enable thinking false portion of raw; why: the request filter applies nested drop global and requested id fields in order scenario uses this clause to evaluate raw as one grouped value. + b'"chat_template_kwargs":{"enable_thinking":false}}' + # What: arrange the raw expression with raw b model low high metadata private true keep; why: test_request_filter_applies_nested_drop_global_and_requested_id_fields_in_order groups the supplied clauses as one raw expression before its value is consumed. + ) + # What: act by calling RequestField and capture global fields; why: the request filter applies nested drop global and requested id fields in order test asserts the response, state, or failure produced by this call. + global_fields = ( + # What: act by calling RequestField with max tokens and 1000; why: the request filter applies nested drop global and requested id fields in order scenario observes the RequestField return value during request field stream true soft. + RequestField(("max_tokens",), "1000"), + # What: arrange RequestField stream true soft True for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + RequestField(("stream",), "true", soft=True), + # What: arrange RequestField stop configured soft True for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + RequestField(("stop",), '"configured"', soft=True), + # What: arrange RequestField top p 0.2 soft True for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + RequestField(("top_p",), "0.2", soft=True), + # What: act by calling RequestField with temperature and 0 5; why: the request filter applies nested drop global and requested id fields in order scenario observes the RequestField return value during request field chat template kwargs reasoning effort medium. + RequestField(("temperature",), "0.5"), + # What: act by calling RequestField with chat template kwargs and reasoning effort and medium; why: the request filter applies nested drop global and requested id fields in order scenario observes the RequestField return value while evaluating RequestField(("chat_template_kwargs", "reasoning_effort"), '"medium. + RequestField(("chat_template_kwargs", "reasoning_effort"), '"medium"'), + # What: arrange the grouped source fragment for the scenario; why: test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + ) + # What: act by calling RequestField and capture by id; why: the request filter applies nested drop global and requested id fields in order test asserts the response, state, or failure produced by this call. + by_id = (("low:high", ( + # What: arrange RequestField max tokens 2000 soft True for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + RequestField(("max_tokens",), "2000", soft=True), + # What: act by calling RequestField with temperature and 0 1; why: the request filter applies nested drop global and requested id fields in order scenario observes the RequestField return value during request field chat template kwargs reasoning effort high. + RequestField(("temperature",), "0.1"), + # What: act by calling RequestField with chat template kwargs and reasoning effort and high; why: the request filter applies nested drop global and requested id fields in order scenario observes the RequestField return value while evaluating RequestField(("chat_template_kwargs", "reasoning_effort"), '"high"'). + RequestField(("chat_template_kwargs", "reasoning_effort"), '"high"'), + # What: arrange the grouped expression portion of by id; why: the request filter applies nested drop global and requested id fields in order scenario uses this clause to evaluate by id as one grouped value. + )),) + + # What: act by calling json.loads and capture filtered; why: the request filter applies nested drop global and requested id fields in order test asserts the response, state, or failure produced by this call. + filtered = json.loads(filter_request_body( + # What: arrange the raw portion of filtered; why: the request filter applies nested drop global and requested id fields in order scenario uses this clause to evaluate filtered as one grouped value. + raw, + # What: arrange the metadata private top p portion of filtered; why: the request filter applies nested drop global and requested id fields in order scenario uses this clause to evaluate filtered as one grouped value. + ("metadata.private", "top_p"), + # What: arrange the global fields portion of filtered; why: the request filter applies nested drop global and requested id fields in order scenario uses this clause to evaluate filtered as one grouped value. + global_fields, + # What: arrange the by id portion of filtered; why: the request filter applies nested drop global and requested id fields in order scenario uses this clause to evaluate filtered as one grouped value. + by_id, + # What: arrange requested model to json.loads; why: the request filter applies nested drop global and requested id fields in order scenario binds this low and high value to json.loads's requested model input. + requested_model="low:high", + # What: arrange rewrite model to json.loads; why: the request filter applies nested drop global and requested id fields in order scenario binds this engine model value to json.loads's rewrite model input. + rewrite_model="engine-model", + # What: arrange the json.loads call with filter request body; why: test_request_filter_applies_nested_drop_global_and_requested_id_fields_in_order groups the supplied clauses as one json.loads call before its value is consumed. + )) + + # What: assert the expected filtered == outcome; why: test router test request filter applies nested drop global and requested id fields in order protects its regression by requiring this observable result after the exercised behavior. + assert filtered == { + # What: arrange model engine model for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + "model": "engine-model", + # What: arrange metadata keep 1 for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + "metadata": {"keep": 1}, + # What: arrange max tokens 1000 for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + "max_tokens": 1000, + # What: arrange stream False for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + "stream": False, + # What: arrange stop None for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + "stop": None, + # What: arrange top p 0.2 for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + "top_p": 0.2, + # What: arrange temperature 0.1 for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + "temperature": 0.1, + # What: arrange chat template kwargs for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + "chat_template_kwargs": { + # What: arrange enable thinking False for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + "enable_thinking": False, + # What: arrange reasoning effort high for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + "reasoning_effort": "high", + # What: arrange the grouped source fragment for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + }, + # What: arrange the grouped source fragment for the scenario; why: test router test request filter applies nested drop global and requested id fields in order requires this concrete input or helper state before exercising the behavior under test. + } + + +# What: define the test_loading_feedback_policy_fails_closed_for_a_removed_model test around local fixtures; why: this test groups the arrange, act, and assertions that protect the loading feedback policy fails closed for a removed model outcome. +def test_loading_feedback_policy_fails_closed_for_a_removed_model(): + # What: act by calling RoutingCoordinator and capture router; why: the loading feedback policy fails closed for a removed model test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(Manager(), catalog(), object(), ready_fn=ready) + # What: assert that router loading feedback enabled missing is false; why: this assertion protects the loading feedback policy fails closed for a removed model regression after the test's arranged inputs and exercised call. + assert router.loading_feedback_enabled("missing") is False + + +# What: define the test_router_applies_variant_filters_to_inference_and_json_upstream_only test around monkeypatch and tmp path; why: this test groups the arrange, act, and assertions that protect the router applies variant filters to inference and json upstream only outcome. +def test_router_applies_variant_filters_to_inference_and_json_upstream_only( + # What: arrange monkeypatch tmp path for the scenario; why: test router applies variant filters to inference and json upstream only requires this concrete input or helper state before exercising the behavior under test. + monkeypatch, tmp_path +# What: arrange the grouped source fragment for the scenario; why: test router applies variant filters to inference and json upstream only requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange path as tmp path and models and toml; why: the router applies variant filters to inference and json upstream only test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: act by calling path.write_text with models and low and model and private; why: the router applies variant filters to inference and json upstream only scenario observes the path.write_text return value during models low. + path.write_text( + # What: arrange the exact models low fixture fragment; why: the router applies variant filters to inference and json upstream only scenario feeds this byte-preserved fragment through """[models.low] before asserting its protocol or parser result. + # What: arrange the exact model private gguf fixture fragment; why: the router applies variant filters to inference and json upstream only scenario feeds this byte-preserved fragment through """[models.low] before asserting its protocol or parser result. + # What: arrange the exact use model name engine model fixture fragment; why: the router applies variant filters to inference and json upstream only scenario feeds this byte-preserved fragment through """[models.low] before asserting its protocol or parser result. + # What: arrange the exact drop fields user fixture fragment; why: the router applies variant filters to inference and json upstream only scenario feeds this byte-preserved fragment through """[models.low] before asserting its protocol or parser result. + # What: arrange the exact models low set fields fixture fragment; why: the router applies variant filters to inference and json upstream only scenario feeds this byte-preserved fragment through """[models.low] before asserting its protocol or parser result. + # What: arrange models low for the scenario; why: test router test router applies variant filters to inference and json upstream only requires this concrete input or helper state before exercising the behavior under test. + # What: arrange the exact max tokens fixture fragment; why: the router applies variant filters to inference and json upstream only scenario feeds this byte-preserved fragment through """[models.low] before asserting its protocol or parser result. + # What: arrange the exact models low set fields by id low high fixture fragment; why: the router applies variant filters to inference and json upstream only scenario feeds this byte-preserved fragment through """[models.low] before asserting its protocol or parser result. + # What: arrange models low for the scenario; why: test router test router applies variant filters to inference and json upstream only requires this concrete input or helper state before exercising the behavior under test. + # What: arrange the exact metadata variant high fixture fragment; why: the router applies variant filters to inference and json upstream only scenario feeds this byte-preserved fragment through """[models.low] before asserting its protocol or parser result. + # What: arrange the exact the grouped expression fixture fragment; why: the router applies variant filters to inference and json upstream only scenario feeds this byte-preserved fragment through """[models.low] before asserting its protocol or parser result. + """[models.low] +model = "private.gguf" +use_model_name = "engine-model" +drop_fields = ["user"] +[models.low.set_fields] +temperature = 0.5 +"max_tokens?" = 100 +[models.low.set_fields_by_id."low:high"] +temperature = 0.1 +"metadata.variant" = "high" +""", + # What: arrange the exact encoding utf 8 fixture fragment; why: the router applies variant filters to inference and json upstream only scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_router_applies_variant_filters_to_inference_and_json_upstream_only groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: act by calling ModelCatalog.load and capture catalog doc; why: the router applies variant filters to inference and json upstream only test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog.load(str(path)) + # What: act by calling Manager and capture manager; why: the router applies variant filters to inference and json upstream only test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling RoutingCoordinator and capture router; why: the router applies variant filters to inference and json upstream only test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange seen as the fixture input; why: the router applies variant filters to inference and json upstream only test consumes this named precondition before exercising the behavior. + seen = [] + + # What: define the upstream test helper around captured fixture state; why: the router applies variant filters to inference and json upstream only scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: arrange the exact seen append kwargs body fixture fragment; why: the router applies variant filters to inference and json upstream only scenario feeds this byte-preserved fragment through seen.append(kwargs["body"]) before asserting its protocol or parser result. + seen.append(kwargs["body"]) + # What: arrange the helper response as UpstreamResponse 200 Content Type application json BytesIO b; why: test router applies variant filter feeds this result into the behavior whose outcome is asserted. + return UpstreamResponse(200, {"Content-Type": "application/json"}, BytesIO(b'{}')) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the router applies variant filters to inference and json upstream only scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) before asse. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_router_applies_variant_filters_to_inference_and_json_upstream_only releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the router applies variant filters to inference and json upstream only test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_router_applies_variant_filters_to_inference_and_json_upstream_only; why: test_router_applies_variant_filters_to_inference_and_json_upstream_only consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the router applies variant filters to inference and json upstream only scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_router_applies_variant_filters_to_inference_and_json_upstream_only groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the router applies variant filters to inference and json upstream only test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling client.post and capture routed; why: the router applies variant filters to inference and json upstream only test asserts the response, state, or failure produced by this call. + routed = client.post( + # What: arrange the v1 chat completions portion of routed; why: the router applies variant filters to inference and json upstream only scenario uses this clause to evaluate routed as one grouped value. + "/v1/chat/completions", + # What: arrange the model field as low and high; why: test_router_applies_variant_filters_to_inference_and_json_upstream_only sends this field through routed so the router selects the canonical model or alias for upstream dispatch. + json={"model": "low:high", "max_tokens": 7, "user": "private"}, + # What: arrange the client.post call with json; why: test_router_applies_variant_filters_to_inference_and_json_upstream_only groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling client.post and capture direct json; why: the router applies variant filters to inference and json upstream only test asserts the response, state, or failure produced by this call. + direct_json = client.post( + # What: arrange the upstream low high custom portion of direct json; why: the router applies variant filters to inference and json upstream only scenario uses this clause to evaluate direct json as one grouped value. + "/upstream/low:high/custom", + # What: arrange content b model low high user private for the scenario; why: test router test router applies variant filters to inference and json upstream only requires this concrete input or helper state before exercising the behavior under test. + content=b'{"model":"low:high","user":"private"}', + # What: arrange the content type field as application and json; why: test_router_applies_variant_filters_to_inference_and_json_upstream_only carries content type through direct json into assert routed status code equals direct json status code equals direct raw status code equals. + headers={"Content-Type": "application/json"}, + # What: arrange the client.post call with content and headers; why: test_router_applies_variant_filters_to_inference_and_json_upstream_only groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling client.post and capture direct raw; why: the router applies variant filters to inference and json upstream only test asserts the response, state, or failure produced by this call. + direct_raw = client.post( + # What: arrange the upstream low high custom portion of direct raw; why: the router applies variant filters to inference and json upstream only scenario uses this clause to evaluate direct raw as one grouped value. + "/upstream/low:high/custom", + # What: arrange content b not json private body for the scenario; why: test router test router applies variant filters to inference and json upstream only requires this concrete input or helper state before exercising the behavior under test. + content=b"not-json-private-body", + # What: arrange the content type field as application and octet stream; why: test_router_applies_variant_filters_to_inference_and_json_upstream_only carries content type through direct raw into assert routed status code equals direct json status code equals direct raw status code equals. + headers={"Content-Type": "application/octet-stream"}, + # What: arrange the client.post call with content and headers; why: test_router_applies_variant_filters_to_inference_and_json_upstream_only groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling client.post and capture malformed json; why: the router applies variant filters to inference and json upstream only test asserts the response, state, or failure produced by this call. + malformed_json = client.post( + # What: arrange the upstream low high custom portion of malformed json; why: the router applies variant filters to inference and json upstream only scenario uses this clause to evaluate malformed json as one grouped value. + "/upstream/low:high/custom", + # What: arrange content to client.post; why: the router applies variant filters to inference and json upstream only scenario binds this the named fixture input value to client.post's content input. + content=b"{", + # What: arrange the content type field as application and json; why: test_router_applies_variant_filters_to_inference_and_json_upstream_only carries content type through malformed json into assert malformed json status code equals 400. + headers={"Content-Type": "application/json"}, + # What: arrange the client.post call with content and headers; why: test_router_applies_variant_filters_to_inference_and_json_upstream_only groups the supplied clauses as one client.post call before its value is consumed. + ) + + # What: assert that routed status code equals direct json status code equals direct raw status code equals 200; why: this assertion protects the router applies variant filters to inference and json upstream only regression after the test's arranged inputs and exercised call. + assert routed.status_code == direct_json.status_code == direct_raw.status_code == 200 + # What: assert that malformed json status code equals 400; why: this assertion protects the router applies variant filters to inference and json upstream only regression after the test's arranged inputs and exercised call. + assert malformed_json.status_code == 400 + # What: assert that malformed json json error type equals invalid request; why: this assertion protects the router applies variant filters to inference and json upstream only regression after the test's arranged inputs and exercised call. + assert malformed_json.json()["error"]["type"] == "invalid_request" + # What: assert the expected json loads seen 0 == outcome; why: test router test router applies variant filters to inference and json upstream only protects its regression by requiring this observable result after the exercised behavior. + assert json.loads(seen[0]) == { + # What: arrange model engine model max tokens 7 temperature 0.1 for the scenario; why: test router test router applies variant filters to inference and json upstream only requires this concrete input or helper state before exercising the behavior under test. + "model": "engine-model", "max_tokens": 7, "temperature": 0.1, + # What: arrange metadata variant high for the scenario; why: test router test router applies variant filters to inference and json upstream only requires this concrete input or helper state before exercising the behavior under test. + "metadata": {"variant": "high"}, + # What: arrange the grouped source fragment for the scenario; why: test router test router applies variant filters to inference and json upstream only requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert the expected json loads seen 1 == outcome; why: test router test router applies variant filters to inference and json upstream only protects its regression by requiring this observable result after the exercised behavior. + assert json.loads(seen[1]) == { + # What: arrange model engine model temperature 0.1 max tokens 100 for the scenario; why: test router test router applies variant filters to inference and json upstream only requires this concrete input or helper state before exercising the behavior under test. + "model": "engine-model", "temperature": 0.1, "max_tokens": 100, + # What: arrange metadata variant high for the scenario; why: test router test router applies variant filters to inference and json upstream only requires this concrete input or helper state before exercising the behavior under test. + "metadata": {"variant": "high"}, + # What: arrange the grouped source fragment for the scenario; why: test router test router applies variant filters to inference and json upstream only requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that seen 2 equals b not json private body; why: this assertion protects the router applies variant filters to inference and json upstream only regression after the test's arranged inputs and exercised call. + assert seen[2] == b"not-json-private-body" + # What: assert that len seen equals 3; why: this assertion protects the router applies variant filters to inference and json upstream only regression after the test's arranged inputs and exercised call. + assert len(seen) == 3 + # What: assert that router status active requests equals 0; why: this assertion protects the router applies variant filters to inference and json upstream only regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + + +# What: define the test_router_event_log_is_bounded_private_and_protected test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the router event log is bounded private and protected outcome. +def test_router_event_log_is_bounded_private_and_protected(monkeypatch): + """Router events are useful operational evidence without retaining prompts or secrets.""" + # What: document router events are useful operational evidence in the test_router_event_log_is_bounded_private_and_protected docstring; why: introspection and maintainers read this exact docstring fragment to understand test router event log is bounded private and protected behavior without executing it. + # What: act by calling Manager and capture manager; why: the router event log is bounded private and protected test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the router event log is bounded private and protected test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf; why: test_router_event_log_is_bounded_private_and_protected carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": ModelProfile("low", "low.gguf", ())}, + # What: arrange settings to RouterSettings; why: the router event log is bounded private and protected scenario binds this router settings and router test key value to RouterSettings's settings input. + settings=RouterSettings(api_keys=("router-test-key",)), + # What: arrange the ModelCatalog call with settings; why: test_router_event_log_is_bounded_private_and_protected groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the router event log is bounded private and protected test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: act by calling LogRing and capture router ring; why: the router event log is bounded private and protected test asserts the response, state, or failure produced by this call. + router_ring = LogRing(capacity=1) + + # What: define the upstream test helper around captured fixture state; why: the router event log is bounded private and protected scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def upstream(**kwargs): + # What: arrange the helper response as UpstreamResponse 200 Content Type application json BytesIO b ok true; why: test router event log is bounded private feeds this result into the behavior whose outcome is asserted. + return UpstreamResponse(200, {"Content-Type": "application/json"}, BytesIO(b'{"ok":true}')) + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream upstream fixture fragment; why: the router event log is bounded private and protected scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) before asserting its protoco. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_router_event_log_is_bounded_private_and_protected releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the router event log is bounded private and protected test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_router_event_log_is_bounded_private_and_protected; why: test_router_event_log_is_bounded_private_and_protected consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the router event log is bounded private and protected scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange router ring to build_app; why: the router event log is bounded private and protected scenario binds this router ring value to build_app's router ring input. + router_ring=router_ring, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_router_event_log_is_bounded_private_and_protected groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the router event log is bounded private and protected test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: assert that client get router logs status code equals 401; why: this assertion protects the router event log is bounded private and protected regression after the test's arranged inputs and exercised call. + assert client.get("/router/logs").status_code == 401 + # What: act by calling client.post and capture response; why: the router event log is bounded private and protected test asserts the response, state, or failure produced by this call. + response = client.post( + # What: arrange the upstream low private token in path access token do not log portion of response; why: the router event log is bounded private and protected scenario uses this clause to evaluate response as one grouped value. + "/upstream/low/private-token-in-path?access_token=do-not-log", + # What: arrange content to client.post; why: the router event log is bounded private and protected scenario binds this the named fixture input value to client.post's content input. + content=b'{"model":"low","messages":["private prompt"]}', + # What: arrange the content type field as application and json; why: test_router_event_log_is_bounded_private_and_protected carries content type through response into assert response status code equals 200. + headers={"Content-Type": "application/json", "Authorization": "Bearer router-test-key"}, + # What: arrange the client.post call with content and headers; why: test_router_event_log_is_bounded_private_and_protected groups the supplied clauses as one client.post call before its value is consumed. + ) + # A legacy direct lifecycle request must not replace a resident routed + # child behind the coordinator's lease/residency bookkeeping. + # What: act by calling client.post and capture blocked; why: the router event log is bounded private and protected test asserts the response, state, or failure produced by this call. + blocked = client.post("/engine/stop", headers={"Authorization": "Bearer router-test-key"}) + # What: assert that response status code equals 200; why: this assertion protects the router event log is bounded private and protected regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + # What: assert that blocked status code equals 409; why: this assertion protects the router event log is bounded private and protected regression after the test's arranged inputs and exercised call. + assert blocked.status_code == 409 + # What: assert that manager model equals low gguf; why: this assertion protects the router event log is bounded private and protected regression after the test's arranged inputs and exercised call. + assert manager.model == "low.gguf" + # What: act by calling router_ring.since and capture records and cursor; why: the router event log is bounded private and protected test asserts the response, state, or failure produced by this call. + records, cursor = router_ring.since(0) + # What: assert that cursor equals 2; why: this assertion protects the router event log is bounded private and protected regression after the test's arranged inputs and exercised call. + assert cursor == 2 + # What: assert that len records equals 1; why: this assertion protects the router event log is bounded private and protected regression after the test's arranged inputs and exercised call. + assert len(records) == 1 # the configured bounded ring evicted admission + # What: act by calling json.loads and capture events; why: the router event log is bounded private and protected test asserts the response, state, or failure produced by this call. + events = [json.loads(record["text"]) for record in records] + # What: assert that event event for event in events equals request finished; why: this assertion protects the router event log is bounded private and protected regression after the test's arranged inputs and exercised call. + assert [event["event"] for event in events] == ["request_finished"] + # What: assert that events 1 response bytes equals len b ok true; why: this assertion protects the router event log is bounded private and protected regression after the test's arranged inputs and exercised call. + assert events[-1]["responseBytes"] == len(b'{"ok":true}') + # What: act by calling json.dumps and capture serialized; why: the router event log is bounded private and protected test asserts the response, state, or failure produced by this call. + serialized = json.dumps(events) + # What: assert that private prompt is absent from serialized; why: this assertion protects the router event log is bounded private and protected regression after the test's arranged inputs and exercised call. + assert "private prompt" not in serialized + # What: assert that do not log is absent from serialized; why: this assertion protects the router event log is bounded private and protected regression after the test's arranged inputs and exercised call. + assert "do-not-log" not in serialized + # What: assert that private token in path is absent from serialized; why: this assertion protects the router event log is bounded private and protected regression after the test's arranged inputs and exercised call. + assert "private-token-in-path" not in serialized + # What: assert that router test key is absent from serialized; why: this assertion protects the router event log is bounded private and protected regression after the test's arranged inputs and exercised call. + assert "router-test-key" not in serialized + + +# What: define the test_invalid_router_request_id_cannot_activate_an_engine test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the invalid router request id cannot activate an engine outcome. +def test_invalid_router_request_id_cannot_activate_an_engine(monkeypatch): + # What: act by calling Manager and capture manager; why: the invalid router request id cannot activate an engine test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the invalid router request id cannot activate an engine test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({"low": ModelProfile("low", "low.gguf", ())}) + # What: act by calling RoutingCoordinator and capture router; why: the invalid router request id cannot activate an engine test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + + # What: define the unexpected_upstream test helper around captured fixture state; why: the invalid router request id cannot activate an engine scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def unexpected_upstream(**kwargs): # pragma: no cover - establishes the no-activation contract + # What: raise AssertionError for the caller; why: unexpected_upstream stops this rejected path before it can mutate state, dispatch work, or report success. + raise AssertionError("invalid request ids must be rejected before proxy connection") + + # What: arrange the exact monkeypatch setattr freetoken daemon app open upstream unexpected upstream fixture frag; why: the invalid router request id cannot activate an engine scenario feeds this byte-preserved fragment through monkeypatch.setattr("freetoken.daemon.app.open_upstream", unexpected_ups before asserti. + monkeypatch.setattr("freetoken.daemon.app.open_upstream", unexpected_upstream) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_invalid_router_request_id_cannot_activate_an_engine releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the invalid router request id cannot activate an engine test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_invalid_router_request_id_cannot_activate_an_engine; why: test_invalid_router_request_id_cannot_activate_an_engine consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the invalid router request id cannot activate an engine scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_invalid_router_request_id_cannot_activate_an_engine groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture response; why: the invalid router request id cannot activate an engine test asserts the response, state, or failure produced by this call. + response = TestClient(app).post( + # What: arrange the model field as low; why: test_invalid_router_request_id_cannot_activate_an_engine sends this field through response so the router selects the canonical model or alias for upstream dispatch. + "/v1/chat/completions", json={"model": "low"}, headers={"X-FT-Request-ID": "x" * 129}, + # What: arrange the operation.post call with json and headers; why: test_invalid_router_request_id_cannot_activate_an_engine groups the supplied clauses as one operation.post call before its value is consumed. + ) + # What: assert that response status code equals 400; why: this assertion protects the invalid router request id cannot activate an engine regression after the test's arranged inputs and exercised call. + assert response.status_code == 400 + # What: assert that manager calls equals group delimiter; why: this assertion protects the invalid router request id cannot activate an engine regression after the test's arranged inputs and exercised call. + assert manager.calls == [] + + +# What: define the test_router_management_load_uses_native_admission_and_authentication test around local fixtures; why: this test groups the arrange, act, and assertions that protect the router management load uses native admission and authentication outcome. +def test_router_management_load_uses_native_admission_and_authentication(): + # What: act by calling Manager and capture manager; why: the router management load uses native admission and authentication test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the router management load uses native admission and authentication test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf and 0; why: test_router_management_load_uses_native_admission_and_authentication carries low through catalog doc into manager catalog doc object ready fn ready port allocator lambda. + {"low": ModelProfile("low", "low.gguf", (), port=0)}, + # What: arrange settings to RouterSettings; why: the router management load uses native admission and authentication scenario binds this router settings and router test key value to RouterSettings's settings input. + settings=RouterSettings(api_keys=("router-test-key",)), + # What: arrange the ModelCatalog call with settings; why: test_router_management_load_uses_native_admission_and_authentication groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the router management load uses native admission and authentication test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator( + # What: arrange ready fn to object; why: the router management load uses native admission and authentication scenario binds this ready value to object's ready fn input. + manager, catalog_doc, object(), ready_fn=ready, port_allocator=lambda: 20777, + # What: arrange the RoutingCoordinator call with ready fn and port allocator; why: test_router_management_load_uses_native_admission_and_authentication groups the supplied clauses as one RoutingCoordinator call before its value is consumed. + ) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_router_management_load_uses_native_admission_and_authentication releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the router management load uses native admission and authentication test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_router_management_load_uses_native_admission_and_authentication; why: test_router_management_load_uses_native_admission_and_authentication consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the router management load uses native admission and authentication scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_router_management_load_uses_native_admission_and_authentication groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the router management load uses native admission and authentication test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: assert that client post router load json name low equals 401; why: this assertion protects the router management load uses native admission and authentication regression after the test's arranged inputs and exercised call. + assert client.post("/router/load", json={"name": "low"}).status_code == 401 + # What: act by calling client.post and capture loaded; why: the router management load uses native admission and authentication test asserts the response, state, or failure produced by this call. + loaded = client.post( + # What: arrange the name field as low; why: test_router_management_load_uses_native_admission_and_authentication carries name through loaded into assert loaded status code equals 200. + "/router/load", json={"name": "low"}, headers={"Authorization": "Bearer router-test-key"}, + # What: arrange the client.post call with json and headers; why: test_router_management_load_uses_native_admission_and_authentication groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: act by calling client.post and capture missing; why: the router management load uses native admission and authentication test asserts the response, state, or failure produced by this call. + missing = client.post( + # What: arrange the name field as missing; why: test_router_management_load_uses_native_admission_and_authentication carries name through missing into assert missing status code equals 404. + "/router/load", json={"name": "missing"}, headers={"Authorization": "Bearer router-test-key"}, + # What: arrange the client.post call with json and headers; why: test_router_management_load_uses_native_admission_and_authentication groups the supplied clauses as one client.post call before its value is consumed. + ) + # What: assert that loaded status code equals 200; why: this assertion protects the router management load uses native admission and authentication regression after the test's arranged inputs and exercised call. + assert loaded.status_code == 200 + # What: assert that loaded json profile equals low; why: this assertion protects the router management load uses native admission and authentication regression after the test's arranged inputs and exercised call. + assert loaded.json()["profile"] == "low" + # What: assert that loaded json port equals 20777; why: this assertion protects the router management load uses native admission and authentication regression after the test's arranged inputs and exercised call. + assert loaded.json()["port"] == 20777 + # What: assert that loaded json router active profile equals low; why: this assertion protects the router management load uses native admission and authentication regression after the test's arranged inputs and exercised call. + assert loaded.json()["router"]["activeProfile"] == "low" + # What: assert that loaded json router active requests equals 0; why: this assertion protects the router management load uses native admission and authentication regression after the test's arranged inputs and exercised call. + assert loaded.json()["router"]["activeRequests"] == 0 + # What: assert that missing status code equals 404; why: this assertion protects the router management load uses native admission and authentication regression after the test's arranged inputs and exercised call. + assert missing.status_code == 404 + # What: assert that missing json error type equals unknown model; why: this assertion protects the router management load uses native admission and authentication regression after the test's arranged inputs and exercised call. + assert missing.json()["error"]["type"] == "unknown_model" + # What: assert that manager calls equals start low gguf; why: this assertion protects the router management load uses native admission and authentication regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf")] + + +# What: define the test_startup_profile_and_preload_use_native_routing_lifespan test around local fixtures; why: this test groups the arrange, act, and assertions that protect the startup profile and preload use native routing lifespan outcome. +def test_startup_profile_and_preload_use_native_routing_lifespan(): + # What: act by calling Manager and capture manager; why: the startup profile and preload use native routing lifespan test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the startup profile and preload use native routing lifespan test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf and compat low; why: test_startup_profile_and_preload_use_native_routing_lifespan carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": ModelProfile("low", "low.gguf", (), aliases=("compat-low",))}, + # What: arrange settings to RouterSettings; why: the startup profile and preload use native routing lifespan scenario binds this router settings and compat low and coding value to RouterSettings's settings input. + settings=RouterSettings( + # What: arrange preload model to RouterSettings; why: the startup profile and preload use native routing lifespan scenario binds this compat low value to RouterSettings's preload model input. + preload_model="compat-low", startup_routing_profile="coding", + # What: arrange the RouterSettings call with preload model and startup routing profile; why: test_startup_profile_and_preload_use_native_routing_lifespan groups the supplied clauses as one RouterSettings call before its value is consumed. + ), + # What: arrange routing profiles to ModelCatalog; why: the startup profile and preload use native routing lifespan scenario binds this routing profile and coding and coding and public and low value to ModelCatalog's routing profiles input. + routing_profiles={ + # What: arrange the coding field as routing profile and coding and public and low; why: test_startup_profile_and_preload_use_native_routing_lifespan carries coding through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + "coding": RoutingProfile("coding", (("public", "low"),)), + # What: arrange the catalog_doc mapping with coding; why: test_startup_profile_and_preload_use_native_routing_lifespan groups the supplied clauses as one catalog_doc mapping before its value is consumed. + }, + # What: arrange the ModelCatalog call with settings and routing profiles; why: test_startup_profile_and_preload_use_native_routing_lifespan groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the startup profile and preload use native routing lifespan test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_startup_profile_and_preload_use_native_routing_lifespan releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the startup profile and preload use native routing lifespan test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_startup_profile_and_preload_use_native_routing_lifespan; why: test_startup_profile_and_preload_use_native_routing_lifespan consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the startup profile and preload use native routing lifespan scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_startup_profile_and_preload_use_native_routing_lifespan groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: enter the TestClient managed context before status client get router status json; why: test_startup_profile_and_preload_use_native_routing_lifespan releases this resource or lock after status client get router status json on both success and failure paths. + with TestClient(app) as client: + # What: act by calling operation.json and capture status; why: the startup profile and preload use native routing lifespan test asserts the response, state, or failure produced by this call. + status = client.get("/router/status").json() + + # What: assert that manager calls equals start low gguf; why: this assertion protects the startup profile and preload use native routing lifespan regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf")] + # What: assert that status active profile equals low; why: this assertion protects the startup profile and preload use native routing lifespan regression after the test's arranged inputs and exercised call. + assert status["activeProfile"] == "low" + # What: assert that status active routing profile equals coding; why: this assertion protects the startup profile and preload use native routing lifespan regression after the test's arranged inputs and exercised call. + assert status["activeRoutingProfile"] == "coding" + # What: assert that status active requests equals 0; why: this assertion protects the startup profile and preload use native routing lifespan regression after the test's arranged inputs and exercised call. + assert status["activeRequests"] == 0 + + +# What: define the test_router_management_load_preserves_failed_switch_recovery_evidence test around local fixtures; why: this test groups the arrange, act, and assertions that protect the router management load preserves failed switch recovery evidence outcome. +def test_router_management_load_preserves_failed_switch_recovery_evidence(): + # What: act by calling Manager and capture manager; why: the router management load preserves failed switch recovery evidence test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling catalog and capture catalog doc; why: the router management load preserves failed switch recovery evidence test asserts the response, state, or failure produced by this call. + catalog_doc = catalog() + + # What: define the selective_ready test helper around manager and probe and pid and port and timeout s; why: the router management load preserves failed switch recovery evidence scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def selective_ready(manager, probe, *, pid, port, timeout_s): + # What: arrange the ready field as model and manager and low and gguf; why: selective_ready carries ready into return {"ready": manager.model == "low.gguf", "reason": "fixture-not-rea. + return {"ready": manager.model == "low.gguf", "reason": "fixture-not-ready"} + + # What: act by calling RoutingCoordinator and capture router; why: the router management load preserves failed switch recovery evidence test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=selective_ready) + # What: arrange the exact router acquire low release fixture fragment; why: the router management load preserves failed switch recovery evidence scenario feeds this byte-preserved fragment through router.acquire("low").release() before asserting its protocol or parser result. + router.acquire("low").release() + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_router_management_load_preserves_failed_switch_recovery_evidence releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the router management load preserves failed switch recovery evidence test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_router_management_load_preserves_failed_switch_recovery_evidence; why: test_router_management_load_preserves_failed_switch_recovery_evidence consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the router management load preserves failed switch recovery evidence scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_router_management_load_preserves_failed_switch_recovery_evidence groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.post and capture response; why: the router management load preserves failed switch recovery evidence test asserts the response, state, or failure produced by this call. + response = TestClient(app).post("/router/load", json={"name": "high"}) + + # What: assert that response status code equals 503; why: this assertion protects the router management load preserves failed switch recovery evidence regression after the test's arranged inputs and exercised call. + assert response.status_code == 503 + # What: assert that response json error type equals engine not ready; why: this assertion protects the router management load preserves failed switch recovery evidence regression after the test's arranged inputs and exercised call. + assert response.json()["error"]["type"] == "engine_not_ready" + # What: assert that response json recovery launched is true; why: this assertion protects the router management load preserves failed switch recovery evidence regression after the test's arranged inputs and exercised call. + assert response.json()["recovery"]["launched"] is True + # What: assert that manager model equals low gguf; why: this assertion protects the router management load preserves failed switch recovery evidence regression after the test's arranged inputs and exercised call. + assert manager.model == "low.gguf" + # What: assert that router status active profile equals low; why: this assertion protects the router management load preserves failed switch recovery evidence regression after the test's arranged inputs and exercised call. + assert router.status()["activeProfile"] == "low" + # What: assert that router status active identity matches engine is true; why: this assertion protects the router management load preserves failed switch recovery evidence regression after the test's arranged inputs and exercised call. + assert router.status()["activeIdentityMatchesEngine"] is True + + +# What: define the test_router_management_unloads_one_or_all_under_single_resident_policy test around local fixtures; why: this test groups the arrange, act, and assertions that protect the router management unloads one or all under single resident policy outcome. +def test_router_management_unloads_one_or_all_under_single_resident_policy(): + # What: act by calling Manager and capture manager; why: the router management unloads one or all under single resident policy test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling catalog and capture catalog doc; why: the router management unloads one or all under single resident policy test asserts the response, state, or failure produced by this call. + catalog_doc = catalog() + # What: act by calling RoutingCoordinator and capture router; why: the router management unloads one or all under single resident policy test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_router_management_unloads_one_or_all_under_single_resident_policy releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the router management unloads one or all under single resident policy test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_router_management_unloads_one_or_all_under_single_resident_policy; why: test_router_management_unloads_one_or_all_under_single_resident_policy consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the router management unloads one or all under single resident policy scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_router_management_unloads_one_or_all_under_single_resident_policy groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the router management unloads one or all under single resident policy test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: assert that client post router load json name low equals 200; why: this assertion protects the router management unloads one or all under single resident policy regression after the test's arranged inputs and exercised call. + assert client.post("/router/load", json={"name": "low"}).status_code == 200 + # What: act by calling client.post and capture wrong; why: the router management unloads one or all under single resident policy test asserts the response, state, or failure produced by this call. + wrong = client.post("/router/unload", json={"name": "high"}) + # What: assert that wrong json unloaded is false; why: this assertion protects the router management unloads one or all under single resident policy regression after the test's arranged inputs and exercised call. + assert wrong.json()["unloaded"] is False + # What: assert that wrong json router active profile equals low; why: this assertion protects the router management unloads one or all under single resident policy regression after the test's arranged inputs and exercised call. + assert wrong.json()["router"]["activeProfile"] == "low" + # What: act by calling client.post and capture one; why: the router management unloads one or all under single resident policy test asserts the response, state, or failure produced by this call. + one = client.post("/router/unload", json={"name": "low"}) + # What: assert that one json unloaded is true; why: this assertion protects the router management unloads one or all under single resident policy regression after the test's arranged inputs and exercised call. + assert one.json()["unloaded"] is True + # What: assert that one json router active profile is group delimiter; why: this assertion protects the router management unloads one or all under single resident policy regression after the test's arranged inputs and exercised call. + assert one.json()["router"]["activeProfile"] is None + + # What: assert that client post router load json name high equals 200; why: this assertion protects the router management unloads one or all under single resident policy regression after the test's arranged inputs and exercised call. + assert client.post("/router/load", json={"name": "high"}).status_code == 200 + # What: act by calling client.post and capture all residents; why: the router management unloads one or all under single resident policy test asserts the response, state, or failure produced by this call. + all_residents = client.post("/router/unload") + # What: assert that all residents json unloaded is true; why: this assertion protects the router management unloads one or all under single resident policy regression after the test's arranged inputs and exercised call. + assert all_residents.json()["unloaded"] is True + # What: assert that all residents json router resident profiles equals group delimiter; why: this assertion protects the router management unloads one or all under single resident policy regression after the test's arranged inputs and exercised call. + assert all_residents.json()["router"]["residentProfiles"] == [] + + # What: assert the expected manager calls == outcome; why: test router test router management unloads one or all under single resident policy protects its regression by requiring this observable result after the exercised behavior. + assert manager.calls == [ + # What: arrange start low gguf stop 30.0 for the scenario; why: test router test router management unloads one or all under single resident policy requires this concrete input or helper state before exercising the behavior under test. + ("start", "low.gguf"), ("stop", 30.0), + # What: arrange start high gguf stop 30.0 for the scenario; why: test router test router management unloads one or all under single resident policy requires this concrete input or helper state before exercising the behavior under test. + ("start", "high.gguf"), ("stop", 30.0), + # What: arrange the grouped source fragment for the scenario; why: test router test router management unloads one or all under single resident policy requires this concrete input or helper state before exercising the behavior under test. + ] + + +# What: define the test_router_model_list_hides_model_paths_and_ready_never_cold_loads test around local fixtures; why: this test groups the arrange, act, and assertions that protect the router model list hides model paths and ready never cold loads outcome. +def test_router_model_list_hides_model_paths_and_ready_never_cold_loads(): + # What: act by calling Manager and capture manager; why: the router model list hides model paths and ready never cold loads test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the router model list hides model paths and ready never cold loads test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and private and models and low; why: test_router_model_list_hides_model_paths_and_ready_never_cold_loads carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": ModelProfile( + # What: arrange description to ModelProfile; why: the router model list hides model paths and ready never cold loads scenario binds this public and description value to ModelProfile's description input. + "low", "/private/models/low.gguf", (), description="Public description" + # What: arrange the catalog_doc mapping with low; why: test_router_model_list_hides_model_paths_and_ready_never_cold_loads groups the supplied clauses as one catalog_doc mapping before its value is consumed. + )}, + # What: arrange settings to RouterSettings; why: the router model list hides model paths and ready never cold loads scenario binds this router settings and router test key value to RouterSettings's settings input. + settings=RouterSettings(api_keys=("router-test-key",)), + # What: arrange the ModelCatalog call with settings; why: test_router_model_list_hides_model_paths_and_ready_never_cold_loads groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the router model list hides model paths and ready never cold loads test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + + # What: define Probe as the owner of fresh_health; why: daemon callers use this class boundary so those methods share one probe state invariant. + class Probe: + # What: arrange the def fresh health self port test helper boundary; why: test router test router model list hides model paths and ready never cold loads uses this local double to isolate the behavior checked by its assertions. + def fresh_health(self, port): + # What: arrange the helper response as reachable True status ok maintenance serving; why: test router test router model list hides model paths and ready never cold loads feeds this result into the behavior whose outcome is asserted. + return {"reachable": True, "status": "ok", "maintenance": "serving"} + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_router_model_list_hides_model_paths_and_ready_never_cold_loads releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the router model list hides model paths and ready never cold loads test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_router_model_list_hides_model_paths_and_ready_never_cold_loads; why: test_router_model_list_hides_model_paths_and_ready_never_cold_loads consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=Probe(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the router model list hides model paths and ready never cold loads scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_router_model_list_hides_model_paths_and_ready_never_cold_loads groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the router model list hides model paths and ready never cold loads test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: assert that client get ready status code equals 503; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert client.get("/ready").status_code == 503 + # What: assert that manager calls equals group delimiter; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert manager.calls == [] + # What: assert that client get v1 models status code equals 401; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert client.get("/v1/models").status_code == 401 + # What: act by calling client.get and capture listed; why: the router model list hides model paths and ready never cold loads test asserts the response, state, or failure produced by this call. + listed = client.get("/v1/models", headers={"Authorization": "Bearer router-test-key"}) + # What: arrange the exact router acquire low release fixture fragment; why: the router model list hides model paths and ready never cold loads scenario feeds this byte-preserved fragment through router.acquire("low").release() before asserting its protocol or parser result. + router.acquire("low").release() + # What: assert that client get ready status code equals 200; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert client.get("/ready").status_code == 200 + # What: arrange model as unexpected and gguf; why: the router model list hides model paths and ready never cold loads test consumes this named precondition before exercising the behavior. + manager.model = "unexpected.gguf" + # What: assert that client get ready status code equals 503; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert client.get("/ready").status_code == 503 + # What: act by calling client.get and capture stale listing; why: the router model list hides model paths and ready never cold loads test asserts the response, state, or failure produced by this call. + stale_listing = client.get( + # What: arrange the authorization field as bearer and router test key; why: test_router_model_list_hides_model_paths_and_ready_never_cold_loads carries authorization through stale listing into assert stale listing json data 0 status equals value. + "/v1/models", headers={"Authorization": "Bearer router-test-key"} + # What: arrange the client.get call with headers; why: test_router_model_list_hides_model_paths_and_ready_never_cold_loads groups the supplied clauses as one client.get call before its value is consumed. + ) + # What: assert that stale listing json data 0 status equals value unloaded; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert stale_listing.json()["data"][0]["status"] == {"value": "unloaded"} + # What: act by calling client.get and capture stale status; why: the router model list hides model paths and ready never cold loads test asserts the response, state, or failure produced by this call. + stale_status = client.get("/router/status", headers={"Authorization": "Bearer router-test-key"}) + # What: assert that stale status json resident profiles equals group delimiter; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert stale_status.json()["residentProfiles"] == [] + # What: assert that stale status json active identity matches engine is false; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert stale_status.json()["activeIdentityMatchesEngine"] is False + # What: act by calling client.get and capture stale metrics; why: the router model list hides model paths and ready never cold loads test asserts the response, state, or failure produced by this call. + stale_metrics = client.get("/metrics", headers={"Authorization": "Bearer router-test-key"}) + # What: assert that freetoken swap active identity matches engine 0 is present in stale metrics text; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert "freetoken_swap_active_identity_matches_engine 0" in stale_metrics.text + # What: act by calling client.get and capture stale models; why: the router model list hides model paths and ready never cold loads test asserts the response, state, or failure produced by this call. + stale_models = client.get("/router/models", headers={"Authorization": "Bearer router-test-key"}) + # What: assert that stale models json data 0 resident is false; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert stale_models.json()["data"][0]["resident"] is False + # What: assert that stale models json capacity equals max resident models 1 available resident slots 0; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert stale_models.json()["capacity"] == {"maxResidentModels": 1, "availableResidentSlots": 0} + # What: arrange model as private and models and low and gguf; why: the router model list hides model paths and ready never cold loads test consumes this named precondition before exercising the behavior. + manager.model = "/private/models/low.gguf" + # What: arrange args as unexpected; why: the router model list hides model paths and ready never cold loads test consumes this named precondition before exercising the behavior. + manager.args = ["--unexpected"] + # What: assert that client get ready status code equals 503; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert client.get("/ready").status_code == 503 + # What: arrange args as the fixture input; why: the router model list hides model paths and ready never cold loads test consumes this named precondition before exercising the behavior. + manager.args = [] + # What: arrange port as 1999; why: the router model list hides model paths and ready never cold loads test consumes this named precondition before exercising the behavior. + manager.port = 1999 + # What: assert that client get ready status code equals 503; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert client.get("/ready").status_code == 503 + # What: assert that listed status code equals 200; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert listed.status_code == 200 + # What: act by calling listed.json and capture listed doc; why: the router model list hides model paths and ready never cold loads test asserts the response, state, or failure produced by this call. + listed_doc = listed.json() + # What: assert that listed doc object equals list; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert listed_doc["object"] == "list" + # What: assert that len listed doc data equals 1; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert len(listed_doc["data"]) == 1 + # What: arrange public model as listed doc and 0 and data; why: the router model list hides model paths and ready never cold loads test consumes this named precondition before exercising the behavior. + public_model = listed_doc["data"][0] + # What: assert that public model id equals low; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert public_model["id"] == "low" + # What: assert that public model object equals model; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert public_model["object"] == "model" + # What: assert that public model owned by equals freetoken; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert public_model["owned_by"] == "freetoken" + # What: assert that public model description equals public description; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert public_model["description"] == "Public description" + # What: assert that public model status equals value unloaded; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert public_model["status"] == {"value": "unloaded"} + # What: assert that isinstance public model created int and public model created 0; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert isinstance(public_model["created"], int) and public_model["created"] > 0 + # What: assert that private models is absent from json dumps listed doc; why: this assertion protects the router model list hides model paths and ready never cold loads regression after the test's arranged inputs and exercised call. + assert "/private/models" not in json.dumps(listed_doc) + + +# What: define the test_ready_probe_linearizes_before_a_conflicting_swap test around local fixtures; why: this test groups the arrange, act, and assertions that protect the ready probe linearizes before a conflicting swap outcome. +def test_ready_probe_linearizes_before_a_conflicting_swap(): + # What: act by calling Manager and capture manager; why: the ready probe linearizes before a conflicting swap test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling threading.Event and capture probe started; why: the ready probe linearizes before a conflicting swap test asserts the response, state, or failure produced by this call. + probe_started = threading.Event() + # What: act by calling threading.Event and capture finish probe; why: the ready probe linearizes before a conflicting swap test asserts the response, state, or failure produced by this call. + finish_probe = threading.Event() + + # What: define Probe as the owner of fresh_health; why: daemon callers use this class boundary so those methods share one probe state invariant. + class Probe: + # What: define an uncached health probe for the active engine port; why: readiness checks must bypass a replaced generation's cached response before accepting the new process. + def fresh_health(self, port): + # What: act by calling probe_started.set with the declared inputs; why: the ready probe linearizes before a conflicting swap scenario observes the probe_started.set return value during assert finish probe wait. + probe_started.set() + # What: assert that finish probe wait 2; why: this assertion protects the ready probe linearizes before a conflicting swap regression after the test's arranged inputs and exercised call. + assert finish_probe.wait(2) + # What: arrange the helper response as reachable True status ok maintenance serving; why: test router test ready probe linearizes before a conflicting swap feeds this result into the behavior whose outcome is asserted. + return {"reachable": True, "status": "ok", "maintenance": "serving"} + + # What: act by calling Probe and capture probe; why: the ready probe linearizes before a conflicting swap test asserts the response, state, or failure produced by this call. + probe = Probe() + # What: act by calling RoutingCoordinator and capture router; why: the ready probe linearizes before a conflicting swap test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), probe, ready_fn=ready) + # What: arrange the exact router acquire low release fixture fragment; why: the ready probe linearizes before a conflicting swap scenario feeds this byte-preserved fragment through router.acquire("low").release() before asserting its protocol or parser result. + router.acquire("low").release() + # What: arrange ready response as the fixture input; why: the ready probe linearizes before a conflicting swap test consumes this named precondition before exercising the behavior. + ready_response = [] + # What: arrange switched as the fixture input; why: the ready probe linearizes before a conflicting swap test consumes this named precondition before exercising the behavior. + switched = [] + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_ready_probe_linearizes_before_a_conflicting_swap releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the ready probe linearizes before a conflicting swap test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_ready_probe_linearizes_before_a_conflicting_swap; why: test_ready_probe_linearizes_before_a_conflicting_swap consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=probe, footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to catalog; why: the ready probe linearizes before a conflicting swap scenario binds this lifecycle value to catalog's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog(), router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_ready_probe_linearizes_before_a_conflicting_swap groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the ready probe linearizes before a conflicting swap test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling threading.Thread and capture ready thread; why: the ready probe linearizes before a conflicting swap test asserts the response, state, or failure produced by this call. + ready_thread = threading.Thread( + # What: arrange target to ready_response.append; why: the ready probe linearizes before a conflicting swap scenario binds this append and ready response and get and client and ready value to ready_response.append's target input. + target=lambda: ready_response.append(client.get("/ready")) + # What: arrange the threading.Thread call with target; why: test_ready_probe_linearizes_before_a_conflicting_swap groups the supplied clauses as one threading.Thread call before its value is consumed. + ) + # What: act by calling ready_thread.start with the declared inputs; why: the ready probe linearizes before a conflicting swap scenario observes the ready_thread.start return value during assert probe started wait. + ready_thread.start() + # What: assert that probe started wait 1; why: this assertion protects the ready probe linearizes before a conflicting swap regression after the test's arranged inputs and exercised call. + assert probe_started.wait(1) + + # What: define the switch test helper around captured fixture state; why: the ready probe linearizes before a conflicting swap scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def switch(): + # What: act by calling router.acquire and capture lease; why: the ready probe linearizes before a conflicting swap test asserts the response, state, or failure produced by this call. + lease = router.acquire("high") + # What: act by calling switched.append with name and profile and lease; why: the ready probe linearizes before a conflicting swap scenario observes the switched.append return value during lease release. + switched.append(lease.profile.name) + # What: act by calling lease.release with the declared inputs; why: the ready probe linearizes before a conflicting swap scenario observes the lease.release return value during the enclosing return. + lease.release() + + # What: act by calling threading.Thread and capture switch thread; why: the ready probe linearizes before a conflicting swap test asserts the response, state, or failure produced by this call. + switch_thread = threading.Thread(target=switch) + # What: act by calling switch_thread.start with the declared inputs; why: the ready probe linearizes before a conflicting swap scenario observes the switch_thread.start return value during switch thread join. + switch_thread.start() + # What: act by calling switch_thread.join with 0 05; why: the ready probe linearizes before a conflicting swap scenario observes the switch_thread.join return value during assert switch thread is alive. + switch_thread.join(0.05) + # What: assert that switch thread is alive; why: this assertion protects the ready probe linearizes before a conflicting swap regression after the test's arranged inputs and exercised call. + assert switch_thread.is_alive() + # What: assert that manager calls equals start low gguf; why: this assertion protects the ready probe linearizes before a conflicting swap regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf")] + # What: act by calling finish_probe.set with the declared inputs; why: the ready probe linearizes before a conflicting swap scenario observes the finish_probe.set return value during ready thread join. + finish_probe.set() + # What: act by calling ready_thread.join with 2; why: the ready probe linearizes before a conflicting swap scenario observes the ready_thread.join return value during switch thread join. + ready_thread.join(2) + # What: act by calling switch_thread.join with 2; why: the ready probe linearizes before a conflicting swap scenario observes the switch_thread.join return value during assert not ready thread is alive and not switch thread is alive. + switch_thread.join(2) + # What: assert that not ready thread is alive and not switch thread is alive; why: this assertion protects the ready probe linearizes before a conflicting swap regression after the test's arranged inputs and exercised call. + assert not ready_thread.is_alive() and not switch_thread.is_alive() + + # What: assert that ready response 0 status code equals 200; why: this assertion protects the ready probe linearizes before a conflicting swap regression after the test's arranged inputs and exercised call. + assert ready_response[0].status_code == 200 + # What: assert that switched equals high; why: this assertion protects the ready probe linearizes before a conflicting swap regression after the test's arranged inputs and exercised call. + assert switched == ["high"] + # What: assert that manager calls equals start low gguf switch high gguf; why: this assertion protects the ready probe linearizes before a conflicting swap regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf"), ("switch", "high.gguf")] + + +# What: define the test_manual_engine_start_holds_router_lifecycle_barrier test around local fixtures; why: this test groups the arrange, act, and assertions that protect the manual engine start holds router lifecycle barrier outcome. +def test_manual_engine_start_holds_router_lifecycle_barrier(): + # What: act by calling threading.Event and capture entered; why: the manual engine start holds router lifecycle barrier test asserts the response, state, or failure produced by this call. + entered = threading.Event() + # What: act by calling threading.Event and capture finish manual; why: the manual engine start holds router lifecycle barrier test asserts the response, state, or failure produced by this call. + finish_manual = threading.Event() + + # What: define BlockingManager as the owner of start; why: daemon callers use this class boundary so those methods share one blocking manager state invariant. + class BlockingManager(Manager): + # What: define the start test helper around model and port and args; why: the manual engine start holds router lifecycle barrier scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def start(self, model, port, args): + # What: arrange the exact self calls append manual start model fixture fragment; why: the manual engine start holds router lifecycle barrier scenario feeds this byte-preserved fragment through self.calls.append(("manual-start", model)) before asserting its protocol or parser result. + self.calls.append(("manual-start", model)) + # What: act by calling entered.set with the declared inputs; why: the manual engine start holds router lifecycle barrier scenario observes the entered.set return value during assert finish manual wait. + entered.set() + # What: assert that finish manual wait 2; why: this assertion protects the manual engine start holds router lifecycle barrier regression after the test's arranged inputs and exercised call. + assert finish_manual.wait(2) + # What: act by calling list and capture model and port and args; why: the manual engine start holds router lifecycle barrier test asserts the response, state, or failure produced by this call. + self.model, self.port, self.args = model, port, list(args) + # What: arrange pid from 1; why: the manual engine start holds router lifecycle barrier scenario uses pid during return pid self pid before checking the protected result. + self.pid += 1 + # What: arrange the pid field as pid; why: BlockingManager.start carries pid into return {"pid": self.pid}. + return {"pid": self.pid} + + # What: act by calling BlockingManager and capture manager; why: the manual engine start holds router lifecycle barrier test asserts the response, state, or failure produced by this call. + manager = BlockingManager() + # What: act by calling catalog and capture catalog doc; why: the manual engine start holds router lifecycle barrier test asserts the response, state, or failure produced by this call. + catalog_doc = catalog() + # What: act by calling RoutingCoordinator and capture router; why: the manual engine start holds router lifecycle barrier test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange manual response as the fixture input; why: the manual engine start holds router lifecycle barrier test consumes this named precondition before exercising the behavior. + manual_response = [] + # What: arrange routed lease as the fixture input; why: the manual engine start holds router lifecycle barrier test consumes this named precondition before exercising the behavior. + routed_lease = [] + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_manual_engine_start_holds_router_lifecycle_barrier releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the manual engine start holds router lifecycle barrier test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_manual_engine_start_holds_router_lifecycle_barrier; why: test_manual_engine_start_holds_router_lifecycle_barrier consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the manual engine start holds router lifecycle barrier scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_manual_engine_start_holds_router_lifecycle_barrier groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the manual engine start holds router lifecycle barrier test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling threading.Thread and capture manual thread; why: the manual engine start holds router lifecycle barrier test asserts the response, state, or failure produced by this call. + manual_thread = threading.Thread(target=lambda: manual_response.append(client.post( + # What: arrange the model field as manual and gguf; why: test_manual_engine_start_holds_router_lifecycle_barrier sends this field through manual thread so the router selects the canonical model or alias for upstream dispatch. + "/engine/start", json={"model": "manual.gguf", "port": 1930} + # What: arrange the threading.Thread call with target; why: test_manual_engine_start_holds_router_lifecycle_barrier groups the supplied clauses as one threading.Thread call before its value is consumed. + ))) + # What: act by calling manual_thread.start with the declared inputs; why: the manual engine start holds router lifecycle barrier scenario observes the manual_thread.start return value during assert entered wait. + manual_thread.start() + # What: assert that entered wait 1; why: this assertion protects the manual engine start holds router lifecycle barrier regression after the test's arranged inputs and exercised call. + assert entered.wait(1) + + # What: define the acquire_routed test helper around captured fixture state; why: the manual engine start holds router lifecycle barrier scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def acquire_routed(): + # What: act by calling router.acquire and capture lease; why: the manual engine start holds router lifecycle barrier test asserts the response, state, or failure produced by this call. + lease = router.acquire("low") + # What: act by calling routed_lease.append with lease; why: the manual engine start holds router lifecycle barrier scenario observes the routed_lease.append return value during the enclosing return. + routed_lease.append(lease) + + # What: act by calling threading.Thread and capture routed thread; why: the manual engine start holds router lifecycle barrier test asserts the response, state, or failure produced by this call. + routed_thread = threading.Thread(target=acquire_routed) + # What: act by calling routed_thread.start with the declared inputs; why: the manual engine start holds router lifecycle barrier scenario observes the routed_thread.start return value during for value in range. + routed_thread.start() + # What: act across range to perform status and router; why: the manual engine start holds router lifecycle barrier scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on status and router before the computed value; why: the manual engine start holds router lifecycle barrier scenario admits the computed value only for this predicate and excludes the opposite state. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the manual engine start holds router lifecycle barrier scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling operation.wait with 0 01; why: the manual engine start holds router lifecycle barrier scenario observes the operation.wait return value during assert router status queued requests. + threading.Event().wait(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the manual engine start holds router lifecycle barrier regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + # What: assert that manager calls equals manual start manual gguf; why: this assertion protects the manual engine start holds router lifecycle barrier regression after the test's arranged inputs and exercised call. + assert manager.calls == [("manual-start", "manual.gguf")] + # What: act by calling finish_manual.set with the declared inputs; why: the manual engine start holds router lifecycle barrier scenario observes the finish_manual.set return value during manual thread join. + finish_manual.set() + # What: act by calling manual_thread.join with 2; why: the manual engine start holds router lifecycle barrier scenario observes the manual_thread.join return value during routed thread join. + manual_thread.join(2) + # What: act by calling routed_thread.join with 2; why: the manual engine start holds router lifecycle barrier scenario observes the routed_thread.join return value during assert not manual thread is alive and not routed thread is alive. + routed_thread.join(2) + # What: assert that not manual thread is alive and not routed thread is alive; why: this assertion protects the manual engine start holds router lifecycle barrier regression after the test's arranged inputs and exercised call. + assert not manual_thread.is_alive() and not routed_thread.is_alive() + + # What: assert that manual response 0 status code equals 200; why: this assertion protects the manual engine start holds router lifecycle barrier regression after the test's arranged inputs and exercised call. + assert manual_response[0].status_code == 200 + # What: assert that manager calls equals manual start manual gguf switch low gguf; why: this assertion protects the manual engine start holds router lifecycle barrier regression after the test's arranged inputs and exercised call. + assert manager.calls == [("manual-start", "manual.gguf"), ("switch", "low.gguf")] + # What: act by calling operation.release with the declared inputs; why: the manual engine start holds router lifecycle barrier scenario observes the operation.release return value during assert router status active requests. + routed_lease.pop().release() + # What: assert that router status active requests equals 0; why: this assertion protects the manual engine start holds router lifecycle barrier regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + + +# What: define the test_manual_lifecycle_claim_rejects_router_ownership_and_requires_matching_token test around local fixtures; why: this test groups the arrange, act, and assertions that protect the manual lifecycle claim rejects router ownership and requires matching token outcome. +def test_manual_lifecycle_claim_rejects_router_ownership_and_requires_matching_token(): + # What: act by calling RoutingCoordinator and capture router; why: the manual lifecycle claim rejects router ownership and requires matching token test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(Manager(), catalog(), object(), ready_fn=ready) + # What: act by calling router.acquire and capture lease; why: the manual lifecycle claim rejects router ownership and requires matching token test asserts the response, state, or failure produced by this call. + lease = router.acquire("low") + # What: arrange with pytest raises RoutingError as conflict for the scenario; why: test raises routing error as conflict requires this concrete input or helper state before exercising the behavior under test. + with pytest.raises(RoutingError) as conflict: + # What: act by calling router.begin_manual_lifecycle with the declared inputs; why: the manual lifecycle claim rejects router ownership and requires matching token scenario observes the router.begin_manual_lifecycle return value during assert conflict value code router owned. + router.begin_manual_lifecycle() + # What: assert that conflict value code equals router owned; why: this assertion protects the manual lifecycle claim rejects router ownership and requires matching token regression after the test's arranged inputs and exercised call. + assert conflict.value.code == "router_owned" + # What: assert that conflict value status code equals 409; why: this assertion protects the manual lifecycle claim rejects router ownership and requires matching token regression after the test's arranged inputs and exercised call. + assert conflict.value.status_code == 409 + # What: act by calling lease.release with the declared inputs; why: the manual lifecycle claim rejects router ownership and requires matching token scenario observes the lease.release return value during with pytest raises routing error match router owns. + lease.release() + # What: arrange with pytest raises RoutingError match router owns for the scenario; why: test raises routing error match router owns requires this concrete input or helper state before exercising the behavior under test. + with pytest.raises(RoutingError, match="router owns"): + # What: act by calling router.begin_manual_lifecycle with the declared inputs; why: the manual lifecycle claim rejects router ownership and requires matching token scenario observes the router.begin_manual_lifecycle return value during router routing coordinator manager catalog object ready fn. + router.begin_manual_lifecycle() + + # What: act by calling RoutingCoordinator and capture router; why: the manual lifecycle claim rejects router ownership and requires matching token test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(Manager(), catalog(), object(), ready_fn=ready) + # What: act by calling router.begin_manual_lifecycle and capture owner; why: the manual lifecycle claim rejects router ownership and requires matching token test asserts the response, state, or failure produced by this call. + owner = router.begin_manual_lifecycle() + # What: arrange with pytest raises ValueError match not owned for the scenario; why: test raises value error match not owned requires this concrete input or helper state before exercising the behavior under test. + with pytest.raises(ValueError, match="not owned"): + # What: act by calling router.end_manual_lifecycle with object; why: the manual lifecycle claim rejects router ownership and requires matching token scenario observes the router.end_manual_lifecycle return value during assert router status switching is. + router.end_manual_lifecycle(object()) + # What: assert that router status switching is true; why: this assertion protects the manual lifecycle claim rejects router ownership and requires matching token regression after the test's arranged inputs and exercised call. + assert router.status()["switching"] is True + # What: act by calling router.begin_manual_lifecycle and capture newer owner; why: the manual lifecycle claim rejects router ownership and requires matching token test asserts the response, state, or failure produced by this call. + newer_owner = router.begin_manual_lifecycle(preempt_manual=True) + # What: act by calling router.end_manual_lifecycle with owner; why: the manual lifecycle claim rejects router ownership and requires matching token scenario observes the router.end_manual_lifecycle return value during assert router status switching is. + router.end_manual_lifecycle(owner) + # What: assert that router status switching is true; why: this assertion protects the manual lifecycle claim rejects router ownership and requires matching token regression after the test's arranged inputs and exercised call. + assert router.status()["switching"] is True + # What: act by calling router.end_manual_lifecycle with newer owner; why: the manual lifecycle claim rejects router ownership and requires matching token scenario observes the router.end_manual_lifecycle return value during assert router status switching is. + router.end_manual_lifecycle(newer_owner) + # What: assert that router status switching is false; why: this assertion protects the manual lifecycle claim rejects router ownership and requires matching token regression after the test's arranged inputs and exercised call. + assert router.status()["switching"] is False + + +# What: define the test_cancelled_manual_start_keeps_barrier_until_executor_finishes test around local fixtures; why: this test groups the arrange, act, and assertions that protect the cancelled manual start keeps barrier until executor finishes outcome. +def test_cancelled_manual_start_keeps_barrier_until_executor_finishes(): + # What: act by calling threading.Event and capture entered; why: the cancelled manual start keeps barrier until executor finishes test asserts the response, state, or failure produced by this call. + entered = threading.Event() + # What: act by calling threading.Event and capture finish manual; why: the cancelled manual start keeps barrier until executor finishes test asserts the response, state, or failure produced by this call. + finish_manual = threading.Event() + + # What: define BlockingManager as the owner of start; why: daemon callers use this class boundary so those methods share one blocking manager state invariant. + class BlockingManager(Manager): + # What: define the start test helper around model and port and args; why: the cancelled manual start keeps barrier until executor finishes scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def start(self, model, port, args): + # What: arrange the exact self calls append manual start model fixture fragment; why: the cancelled manual start keeps barrier until executor finishes scenario feeds this byte-preserved fragment through self.calls.append(("manual-start", model)) before asserting its protocol or parser result. + self.calls.append(("manual-start", model)) + # What: act by calling entered.set with the declared inputs; why: the cancelled manual start keeps barrier until executor finishes scenario observes the entered.set return value during assert finish manual wait. + entered.set() + # What: assert that finish manual wait 2; why: this assertion protects the cancelled manual start keeps barrier until executor finishes regression after the test's arranged inputs and exercised call. + assert finish_manual.wait(2) + # What: act by calling list and capture model and port and args; why: the cancelled manual start keeps barrier until executor finishes test asserts the response, state, or failure produced by this call. + self.model, self.port, self.args = model, port, list(args) + # What: arrange pid from 1; why: the cancelled manual start keeps barrier until executor finishes scenario uses pid during return pid self pid before checking the protected result. + self.pid += 1 + # What: arrange the pid field as pid; why: BlockingManager.start carries pid into return {"pid": self.pid}. + return {"pid": self.pid} + + # What: act by calling BlockingManager and capture manager; why: the cancelled manual start keeps barrier until executor finishes test asserts the response, state, or failure produced by this call. + manager = BlockingManager() + # What: act by calling catalog and capture catalog doc; why: the cancelled manual start keeps barrier until executor finishes test asserts the response, state, or failure produced by this call. + catalog_doc = catalog() + # What: act by calling RoutingCoordinator and capture router; why: the cancelled manual start keeps barrier until executor finishes test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange routed lease as the fixture input; why: the cancelled manual start keeps barrier until executor finishes test consumes this named precondition before exercising the behavior. + routed_lease = [] + + # What: define the scenario test helper around app; why: the cancelled manual start keeps barrier until executor finishes scenario calls this helper to produce or observe the exact behavior checked by its assertions. + async def scenario(app): + # What: act by calling httpx.ASGITransport and capture transport; why: the cancelled manual start keeps barrier until executor finishes test asserts the response, state, or failure produced by this call. + transport = httpx.ASGITransport(app=app) + # What: arrange async with httpx AsyncClient transport transport base url http test as client for the scenario; why: test cancelled manual start keeps barrier until requires this concrete input or helper state before exercising the behavior under test. + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + # What: act by calling asyncio.create_task and capture manual; why: the cancelled manual start keeps barrier until executor finishes test asserts the response, state, or failure produced by this call. + manual = asyncio.create_task(client.post( + # What: arrange the model field as manual and gguf; why: scenario sends this field through manual so the router selects the canonical model or alias for upstream dispatch. + "/engine/start", json={"model": "manual.gguf", "port": 1930} + # What: arrange the grouped source fragment for the scenario; why: test router test cancelled manual start keeps barrier until executor finishes requires this concrete input or helper state before exercising the behavior under test. + )) + # What: act across range to perform is set and entered; why: the cancelled manual start keeps barrier until executor finishes scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on is set and entered before the computed value; why: the cancelled manual start keeps barrier until executor finishes scenario admits the computed value only for this predicate and excludes the opposite state. + if entered.is_set(): + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the cancelled manual start keeps barrier until executor finishes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the cancelled manual start keeps barrier until executor finishes scenario observes the asyncio.sleep return value during assert entered is set. + await asyncio.sleep(0.01) + # What: assert that entered is set; why: this assertion protects the cancelled manual start keeps barrier until executor finishes regression after the test's arranged inputs and exercised call. + assert entered.is_set() + # What: act by calling manual.cancel with the declared inputs; why: the cancelled manual start keeps barrier until executor finishes scenario observes the manual.cancel return value during def acquire routed. + manual.cancel() + + # What: define the acquire_routed test helper around captured fixture state; why: the cancelled manual start keeps barrier until executor finishes scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def acquire_routed(): + # What: arrange the exact routed lease append router acquire low fixture fragment; why: the cancelled manual start keeps barrier until executor finishes scenario feeds this byte-preserved fragment through routed_lease.append(router.acquire("low")) before asserting its protocol or parser result. + routed_lease.append(router.acquire("low")) + + # What: act by calling threading.Thread and capture routed thread; why: the cancelled manual start keeps barrier until executor finishes test asserts the response, state, or failure produced by this call. + routed_thread = threading.Thread(target=acquire_routed) + # What: act by calling routed_thread.start with the declared inputs; why: the cancelled manual start keeps barrier until executor finishes scenario observes the routed_thread.start return value during for value in range. + routed_thread.start() + # What: act across range to perform status and router; why: the cancelled manual start keeps barrier until executor finishes scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on status and router before the computed value; why: the cancelled manual start keeps barrier until executor finishes scenario admits the computed value only for this predicate and excludes the opposite state. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the cancelled manual start keeps barrier until executor finishes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the cancelled manual start keeps barrier until executor finishes scenario observes the asyncio.sleep return value during assert router status queued requests. + await asyncio.sleep(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the cancelled manual start keeps barrier until executor finishes regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + # What: assert that manual done is false; why: this assertion protects the cancelled manual start keeps barrier until executor finishes regression after the test's arranged inputs and exercised call. + assert not manual.done() + # What: assert that manager calls equals manual start manual gguf; why: this assertion protects the cancelled manual start keeps barrier until executor finishes regression after the test's arranged inputs and exercised call. + assert manager.calls == [("manual-start", "manual.gguf")] + # What: act by calling manual.cancel with the declared inputs; why: the cancelled manual start keeps barrier until executor finishes scenario observes the manual.cancel return value during await asyncio sleep. + manual.cancel() + # What: act by calling asyncio.sleep with 0 05; why: the cancelled manual start keeps barrier until executor finishes scenario observes the asyncio.sleep return value during assert not manual done. + await asyncio.sleep(0.05) + # What: assert that manual done is false; why: this assertion protects the cancelled manual start keeps barrier until executor finishes regression after the test's arranged inputs and exercised call. + assert not manual.done() + # What: act by calling finish_manual.set with the declared inputs; why: the cancelled manual start keeps barrier until executor finishes scenario observes the finish_manual.set return value during with pytest raises asyncio cancelled error. + finish_manual.set() + # What: assert the pytest.raises failure context; why: the cancelled manual start keeps barrier until executor finishes scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(asyncio.CancelledError): + # What: arrange the await manual portion of the enclosing predicate; why: this clause remains in the cancelled manual start keeps barrier until executor finishes scenario\'s enclosing expression so its grouping and evaluation order stay intact. + await manual + # What: act by calling routed_thread.join with 2; why: the cancelled manual start keeps barrier until executor finishes scenario observes the routed_thread.join return value during assert not routed thread is alive. + routed_thread.join(2) + # What: assert that routed thread is alive is false; why: this assertion protects the cancelled manual start keeps barrier until executor finishes regression after the test's arranged inputs and exercised call. + assert not routed_thread.is_alive() + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_cancelled_manual_start_keeps_barrier_until_executor_finishes releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the cancelled manual start keeps barrier until executor finishes test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_cancelled_manual_start_keeps_barrier_until_executor_finishes; why: test_cancelled_manual_start_keeps_barrier_until_executor_finishes consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the cancelled manual start keeps barrier until executor finishes scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_cancelled_manual_start_keeps_barrier_until_executor_finishes groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling asyncio.run with scenario and app; why: the cancelled manual start keeps barrier until executor finishes scenario observes the asyncio.run return value during assert manager calls manual start manual gguf switch low gguf. + asyncio.run(scenario(app)) + + # What: assert that manager calls equals manual start manual gguf switch low gguf; why: this assertion protects the cancelled manual start keeps barrier until executor finishes regression after the test's arranged inputs and exercised call. + assert manager.calls == [("manual-start", "manual.gguf"), ("switch", "low.gguf")] + # What: act by calling operation.release with the declared inputs; why: the cancelled manual start keeps barrier until executor finishes scenario observes the operation.release return value during assert router status active requests. + routed_lease.pop().release() + # What: assert that router status active requests equals 0; why: this assertion protects the cancelled manual start keeps barrier until executor finishes regression after the test's arranged inputs and exercised call. + assert router.status()["activeRequests"] == 0 + + +# What: define the test_cancelled_manual_profile_switch_completes_failed_readiness_rollback test around local fixtures; why: this test groups the arrange, act, and assertions that protect the cancelled manual profile switch completes failed readiness rollback outcome. +def test_cancelled_manual_profile_switch_completes_failed_readiness_rollback(): + # What: act by calling threading.Event and capture readiness entered; why: the cancelled manual profile switch completes failed readiness rollback test asserts the response, state, or failure produced by this call. + readiness_entered = threading.Event() + # What: act by calling threading.Event and capture finish readiness; why: the cancelled manual profile switch completes failed readiness rollback test asserts the response, state, or failure produced by this call. + finish_readiness = threading.Event() + + # What: define RecoveringManager as the owner of switch_for_readiness and recover_switch; why: daemon callers use this class boundary so those methods share one recovering manager state invariant. + class RecoveringManager(Manager): + # What: define the switch_for_readiness test helper around model and port and args and force; why: the cancelled manual profile switch completes failed readiness rollback scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def switch_for_readiness(self, model, port, args, force=False): + # What: arrange the exact self calls append switch model fixture fragment; why: the cancelled manual profile switch completes failed readiness rollback scenario feeds this byte-preserved fragment through self.calls.append(("switch", model)) before asserting its protocol or parser result. + self.calls.append(("switch", model)) + # What: act by calling list and capture previous; why: the cancelled manual profile switch completes failed readiness rollback test asserts the response, state, or failure produced by this call. + previous = self.model, self.port, list(self.args) + # What: act by calling list and capture model and port and args; why: the cancelled manual profile switch completes failed readiness rollback test asserts the response, state, or failure produced by this call. + self.model, self.port, self.args = model, port, list(args) + # What: arrange pid from 1; why: the cancelled manual profile switch completes failed readiness rollback scenario uses pid during return pid self pid previous before checking the protected result. + self.pid += 1 + # What: arrange the pid field as pid; why: RecoveringManager.switch_for_readiness carries pid into return {"pid": self.pid}, previous. + return {"pid": self.pid}, previous + + # What: define the recover_switch test helper around ticket and force; why: the cancelled manual profile switch completes failed readiness rollback scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def recover_switch(self, ticket, force=False): + # What: arrange the exact self calls append recover ticket fixture fragment; why: the cancelled manual profile switch completes failed readiness rollback scenario feeds this byte-preserved fragment through self.calls.append(("recover", ticket[0])) before asserting its protocol or parser result. + self.calls.append(("recover", ticket[0])) + # What: arrange model and port and args as ticket; why: the cancelled manual profile switch completes failed readiness rollback test consumes this named precondition before exercising the behavior. + self.model, self.port, self.args = ticket + # What: arrange pid from 1; why: the cancelled manual profile switch completes failed readiness rollback scenario uses pid during return launched pid self pid port self port before checking the protected result. + self.pid += 1 + # What: arrange the launched field as true; why: RecoveringManager.recover_switch carries launched into return {"launched": True, "pid": self.pid, "port": self.port}. + return {"launched": True, "pid": self.pid, "port": self.port} + + # What: define Probe as the owner of fresh_health; why: daemon callers use this class boundary so those methods share one probe state invariant. + class Probe: + # What: arrange the def fresh health self port test helper boundary; why: test router test cancelled manual profile switch completes failed readiness rollback uses this local double to isolate the behavior checked by its assertions. + def fresh_health(self, port): + # What: act on port before set and readiness entered; why: the cancelled manual profile switch completes failed readiness rollback scenario admits set and readiness entered only for this predicate and excludes the opposite state. + if port == 1923: + # What: act by calling readiness_entered.set with the declared inputs; why: the cancelled manual profile switch completes failed readiness rollback scenario observes the readiness_entered.set return value during assert finish readiness wait. + readiness_entered.set() + # What: assert that finish readiness wait 2; why: this assertion protects the cancelled manual profile switch completes failed readiness rollback regression after the test's arranged inputs and exercised call. + assert finish_readiness.wait(2) + # What: arrange the helper response as reachable True status error maintenance serving; why: test router test cancelled manual profile switch completes failed readiness rollback feeds this result into the behavior whose outcome is asserted. + return {"reachable": True, "status": "error", "maintenance": "serving"} + # What: arrange the helper response as reachable True status ok maintenance serving; why: test router test cancelled manual profile switch completes failed readiness rollback feeds this result into the behavior whose outcome is asserted. + return {"reachable": True, "status": "ok", "maintenance": "serving"} + + # What: act by calling RecoveringManager and capture manager; why: the cancelled manual profile switch completes failed readiness rollback test asserts the response, state, or failure produced by this call. + manager = RecoveringManager() + # What: arrange model and port and args as legacy and gguf and 1922; why: the cancelled manual profile switch completes failed readiness rollback test consumes this named precondition before exercising the behavior. + manager.model, manager.port, manager.args = "legacy.gguf", 1922, [] + # What: act by calling ModelCatalog and capture catalog doc; why: the cancelled manual profile switch completes failed readiness rollback test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog({ + # What: arrange the high field as model profile and high and high and gguf and 1923; why: test_cancelled_manual_profile_switch_completes_failed_readiness_rollback carries high through catalog doc into router routing coordinator manager catalog doc probe ready fn ready. + "high": ModelProfile("high", "high.gguf", (), port=1923, ready_timeout_s=1), + # What: arrange the ModelCatalog call with model profile; why: test_cancelled_manual_profile_switch_completes_failed_readiness_rollback groups the supplied clauses as one ModelCatalog call before its value is consumed. + }) + # What: act by calling Probe and capture probe; why: the cancelled manual profile switch completes failed readiness rollback test asserts the response, state, or failure produced by this call. + probe = Probe() + # What: act by calling RoutingCoordinator and capture router; why: the cancelled manual profile switch completes failed readiness rollback test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, probe, ready_fn=ready) + + # What: define the scenario test helper around app; why: the cancelled manual profile switch completes failed readiness rollback scenario calls this helper to produce or observe the exact behavior checked by its assertions. + async def scenario(app): + # What: act by calling httpx.ASGITransport and capture transport; why: the cancelled manual profile switch completes failed readiness rollback test asserts the response, state, or failure produced by this call. + transport = httpx.ASGITransport(app=app) + # What: arrange async with httpx AsyncClient transport transport base url http test as client for the scenario; why: test cancelled manual profile switch c requires this concrete input or helper state before exercising the behavior under test. + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + # What: act by calling asyncio.create_task and capture request; why: the cancelled manual profile switch completes failed readiness rollback test asserts the response, state, or failure produced by this call. + request = asyncio.create_task(client.post( + # What: arrange the name field as high; why: scenario carries name through request into if request done. + "/engine/switch-profile", json={"name": "high"} + # What: arrange the grouped source fragment for the scenario; why: test router test cancelled manual profile switch completes failed readiness rollback requires this concrete input or helper state before exercising the behavior under test. + )) + # What: act across range to perform is set and readiness entered; why: the cancelled manual profile switch completes failed readiness rollback scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on is set and readiness entered before the computed value; why: the cancelled manual profile switch completes failed readiness rollback scenario admits the computed value only for this predicate and excludes the opposite state. + if readiness_entered.is_set(): + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the cancelled manual profile switch completes failed readiness rollback scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the cancelled manual profile switch completes failed readiness rollback scenario observes the asyncio.sleep return value during if request done. + await asyncio.sleep(0.01) + # What: act on done and request before response and result and request; why: the cancelled manual profile switch completes failed readiness rollback scenario admits response and result and request only for this predicate and excludes the opposite state. + if request.done(): + # What: act by calling request.result and capture response; why: the cancelled manual profile switch completes failed readiness rollback test asserts the response, state, or failure produced by this call. + response = request.result() + # What: arrange the exact pytest fail f switch profile exited early response status code fixture fragment; why: the cancelled manual profile switch completes failed readiness rollback scenario feeds this byte-preserved fragment through pytest.fail(f"switch-profile exited early: {response.status_code} {. + pytest.fail(f"switch-profile exited early: {response.status_code} {response.text}") + # What: assert that readiness entered is set; why: this assertion protects the cancelled manual profile switch completes failed readiness rollback regression after the test's arranged inputs and exercised call. + assert readiness_entered.is_set() + # What: act by calling request.cancel with the declared inputs; why: the cancelled manual profile switch completes failed readiness rollback scenario observes the request.cancel return value during assert router status switching is. + request.cancel() + # What: assert that router status switching is true; why: this assertion protects the cancelled manual profile switch completes failed readiness rollback regression after the test's arranged inputs and exercised call. + assert router.status()["switching"] is True + # What: act by calling finish_readiness.set with the declared inputs; why: the cancelled manual profile switch completes failed readiness rollback scenario observes the finish_readiness.set return value during with pytest raises asyncio cancelled error. + finish_readiness.set() + # What: assert the pytest.raises failure context; why: the cancelled manual profile switch completes failed readiness rollback scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(asyncio.CancelledError): + # What: arrange the await request portion of the enclosing predicate; why: this clause remains in the cancelled manual profile switch completes failed readiness rollback scenario\'s enclosing expression so its grouping and evaluation order stay intact. + await request + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_cancelled_manual_profile_switch_completes_failed_readiness_rollback releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the cancelled manual profile switch completes failed readiness rollback test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_cancelled_manual_profile_switch_completes_failed_readiness_rollback; why: test_cancelled_manual_profile_switch_completes_failed_readiness_rollback consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=probe, footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the cancelled manual profile switch completes failed readiness rollback scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_cancelled_manual_profile_switch_completes_failed_readiness_rollback groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling asyncio.run with scenario and app; why: the cancelled manual profile switch completes failed readiness rollback scenario observes the asyncio.run return value during assert manager calls switch high gguf recover legacy gguf. + asyncio.run(scenario(app)) + + # What: assert that manager calls equals switch high gguf recover legacy gguf; why: this assertion protects the cancelled manual profile switch completes failed readiness rollback regression after the test's arranged inputs and exercised call. + assert manager.calls == [("switch", "high.gguf"), ("recover", "legacy.gguf")] + # What: assert that manager model equals legacy gguf; why: this assertion protects the cancelled manual profile switch completes failed readiness rollback regression after the test's arranged inputs and exercised call. + assert manager.model == "legacy.gguf" + # What: assert that router status switching is false; why: this assertion protects the cancelled manual profile switch completes failed readiness rollback regression after the test's arranged inputs and exercised call. + assert router.status()["switching"] is False + + +# What: define the test_router_shutdown_drains_active_lease_and_rejects_queued_and_new_admission test around local fixtures; why: this test groups the arrange, act, and assertions that protect the router shutdown drains active lease and rejects queued and new admission outcome. +def test_router_shutdown_drains_active_lease_and_rejects_queued_and_new_admission(): + # What: define ShutdownManager as the owner of shutdown; why: daemon callers use this class boundary so those methods share one shutdown manager state invariant. + class ShutdownManager(Manager): + # What: define the shutdown test helper around timeout and force; why: the router shutdown drains active lease and rejects queued and new admission scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def shutdown(self, timeout=None, force=False): + # What: arrange the exact self calls append shutdown force fixture fragment; why: the router shutdown drains active lease and rejects queued and new admission scenario feeds this byte-preserved fragment through self.calls.append(("shutdown", force)) before asserting its protocol or parser result. + self.calls.append(("shutdown", force)) + # What: arrange model as the fixture input; why: the router shutdown drains active lease and rejects queued and new admission test consumes this named precondition before exercising the behavior. + self.model = None + # What: arrange the stopped field as true; why: ShutdownManager.shutdown carries stopped into return {"stopped": True, "already": False, "accounting": None}. + return {"stopped": True, "already": False, "accounting": None} + + # What: act by calling ShutdownManager and capture manager; why: the router shutdown drains active lease and rejects queued and new admission test asserts the response, state, or failure produced by this call. + manager = ShutdownManager() + # What: act by calling RoutingCoordinator and capture router; why: the router shutdown drains active lease and rejects queued and new admission test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: act by calling router.acquire and capture active; why: the router shutdown drains active lease and rejects queued and new admission test asserts the response, state, or failure produced by this call. + active = router.acquire("low") + # What: arrange queued result as the fixture input; why: the router shutdown drains active lease and rejects queued and new admission test consumes this named precondition before exercising the behavior. + queued_result = {} + + # What: define the acquire_queued test helper around captured fixture state; why: the router shutdown drains active lease and rejects queued and new admission scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def acquire_queued(): + # What: establish the handler boundary for the protected operation; why: acquire_queued routes failures to routing error while preserving cleanup and success flow. + try: + # What: arrange the exact router acquire high fixture fragment; why: the router shutdown drains active lease and rejects queued and new admission scenario feeds this byte-preserved fragment through router.acquire("high") before asserting its protocol or parser result. + router.acquire("high") + # What: handle routing error by queued result error exc; why: acquire_queued converts that failure into this concrete recovery, response, or cleanup behavior. + except RoutingError as exc: + # What: arrange queued result entry as exc; why: the router shutdown drains active lease and rejects queued and new admission test consumes this named precondition before exercising the behavior. + queued_result["error"] = exc + + # What: act by calling threading.Thread and capture queued; why: the router shutdown drains active lease and rejects queued and new admission test asserts the response, state, or failure produced by this call. + queued = threading.Thread(target=acquire_queued) + # What: act by calling queued.start with the declared inputs; why: the router shutdown drains active lease and rejects queued and new admission scenario observes the queued.start return value during for value in range. + queued.start() + # What: act across range to perform status and router; why: the router shutdown drains active lease and rejects queued and new admission scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: arrange if router status queuedRequests == 1 for the scenario; why: test router test router shutdown drains active lease and rejects queued and new admission requires this concrete input or helper state before exercising the behavior under test. + if router.status()["queuedRequests"] == 1: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the router shutdown drains active lease and rejects queued and new admission scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling time.sleep with 0 01; why: the router shutdown drains active lease and rejects queued and new admission scenario observes the time.sleep return value during assert router status queued requests. + time.sleep(0.01) + # What: assert that router status queued requests equals 1; why: this assertion protects the router shutdown drains active lease and rejects queued and new admission regression after the test's arranged inputs and exercised call. + assert router.status()["queuedRequests"] == 1 + + # What: arrange shutdown result as the fixture input; why: the router shutdown drains active lease and rejects queued and new admission test consumes this named precondition before exercising the behavior. + shutdown_result = {} + # What: act by calling threading.Thread and capture shutdown; why: the router shutdown drains active lease and rejects queued and new admission test asserts the response, state, or failure produced by this call. + shutdown = threading.Thread( + # What: arrange target to shutdown_result.setdefault; why: the router shutdown drains active lease and rejects queued and new admission scenario binds this setdefault and shutdown result and shutdown and router and result value to shutdown_result.setdefault's target input. + target=lambda: shutdown_result.setdefault("result", router.shutdown(force=True)) + # What: arrange the threading.Thread call with target; why: test_router_shutdown_drains_active_lease_and_rejects_queued_and_new_admission groups the supplied clauses as one threading.Thread call before its value is consumed. + ) + # What: act by calling shutdown.start with the declared inputs; why: the router shutdown drains active lease and rejects queued and new admission scenario observes the shutdown.start return value during for value in range. + shutdown.start() + # What: act across range to perform status and router; why: the router shutdown drains active lease and rejects queued and new admission scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: arrange if router status shuttingDown for the scenario; why: test router test router shutdown drains active lease and rejects queued and new admission requires this concrete input or helper state before exercising the behavior under test. + if router.status()["shuttingDown"]: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the router shutdown drains active lease and rejects queued and new admission scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling time.sleep with 0 01; why: the router shutdown drains active lease and rejects queued and new admission scenario observes the time.sleep return value during queued join. + time.sleep(0.01) + # What: act by calling queued.join with 2; why: the router shutdown drains active lease and rejects queued and new admission scenario observes the queued.join return value during assert not queued is alive. + queued.join(2) + # What: assert that queued is alive is false; why: this assertion protects the router shutdown drains active lease and rejects queued and new admission regression after the test's arranged inputs and exercised call. + assert not queued.is_alive() + # What: assert that queued result error code equals router shutting down; why: this assertion protects the router shutdown drains active lease and rejects queued and new admission regression after the test's arranged inputs and exercised call. + assert queued_result["error"].code == "router_shutting_down" + # What: assert that shutdown is alive; why: this assertion protects the router shutdown drains active lease and rejects queued and new admission regression after the test's arranged inputs and exercised call. + assert shutdown.is_alive() + # What: assert that router is ready is false; why: this assertion protects the router shutdown drains active lease and rejects queued and new admission regression after the test's arranged inputs and exercised call. + assert router.is_ready() is False + # What: assert that freetoken swap shutting down 1 is present in router prometheus; why: this assertion protects the router shutdown drains active lease and rejects queued and new admission regression after the test's arranged inputs and exercised call. + assert "freetoken_swap_shutting_down 1" in router.prometheus() + # What: assert that manager calls equals start low gguf; why: this assertion protects the router shutdown drains active lease and rejects queued and new admission regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf")] + # What: assert the pytest.raises failure context; why: the router shutdown drains active lease and rejects queued and new admission scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RoutingError) as exc: + # What: arrange the exact router acquire low fixture fragment; why: the router shutdown drains active lease and rejects queued and new admission scenario feeds this byte-preserved fragment through router.acquire("low") before asserting its protocol or parser result. + router.acquire("low") + # What: assert that exc value code equals router shutting down; why: this assertion protects the router shutdown drains active lease and rejects queued and new admission regression after the test's arranged inputs and exercised call. + assert exc.value.code == "router_shutting_down" + + # What: act by calling active.release with the declared inputs; why: the router shutdown drains active lease and rejects queued and new admission scenario observes the active.release return value during shutdown join. + active.release() + # What: act by calling shutdown.join with 2; why: the router shutdown drains active lease and rejects queued and new admission scenario observes the shutdown.join return value during assert not shutdown is alive. + shutdown.join(2) + # What: assert that shutdown is alive is false; why: this assertion protects the router shutdown drains active lease and rejects queued and new admission regression after the test's arranged inputs and exercised call. + assert not shutdown.is_alive() + # What: assert that shutdown result result stopped is true; why: this assertion protects the router shutdown drains active lease and rejects queued and new admission regression after the test's arranged inputs and exercised call. + assert shutdown_result["result"]["stopped"] is True + # What: assert that manager calls equals start low gguf shutdown true; why: this assertion protects the router shutdown drains active lease and rejects queued and new admission regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf"), ("shutdown", True)] + # What: assert that router status active profile is group delimiter; why: this assertion protects the router shutdown drains active lease and rejects queued and new admission regression after the test's arranged inputs and exercised call. + assert router.status()["activeProfile"] is None + + +# What: define the test_failed_router_shutdown_reopens_admission_and_preserves_resident test around local fixtures; why: this test groups the arrange, act, and assertions that protect the failed router shutdown reopens admission and preserves resident outcome. +def test_failed_router_shutdown_reopens_admission_and_preserves_resident(): + # What: define FailingShutdownManager as the owner of shutdown; why: daemon callers use this class boundary so those methods share one failing shutdown manager state invariant. + class FailingShutdownManager(Manager): + # What: define the shutdown test helper around timeout and force; why: the failed router shutdown reopens admission and preserves resident scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def shutdown(self, timeout=None, force=False): + # What: raise RuntimeError for the caller; why: FailingShutdownManager.shutdown stops this rejected path before it can mutate state, dispatch work, or report success. + raise RuntimeError("stop failed") + + # What: act by calling FailingShutdownManager and capture manager; why: the failed router shutdown reopens admission and preserves resident test asserts the response, state, or failure produced by this call. + manager = FailingShutdownManager() + # What: act by calling RoutingCoordinator and capture router; why: the failed router shutdown reopens admission and preserves resident test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: arrange the exact router acquire low release fixture fragment; why: the failed router shutdown reopens admission and preserves resident scenario feeds this byte-preserved fragment through router.acquire("low").release() before asserting its protocol or parser result. + router.acquire("low").release() + + # What: assert the pytest.raises failure context; why: the failed router shutdown reopens admission and preserves resident scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError, match="stop failed"): + # What: act by calling router.shutdown with the declared inputs; why: the failed router shutdown reopens admission and preserves resident scenario observes the router.shutdown return value during assert router status shutting down is. + router.shutdown() + + # What: assert that router status shutting down is false; why: this assertion protects the failed router shutdown reopens admission and preserves resident regression after the test's arranged inputs and exercised call. + assert router.status()["shuttingDown"] is False + # What: assert that router status active profile equals low; why: this assertion protects the failed router shutdown reopens admission and preserves resident regression after the test's arranged inputs and exercised call. + assert router.status()["activeProfile"] == "low" + # What: act by calling router.acquire and capture lease; why: the failed router shutdown reopens admission and preserves resident test asserts the response, state, or failure produced by this call. + lease = router.acquire("low") + # What: act by calling lease.release with the declared inputs; why: the failed router shutdown reopens admission and preserves resident scenario observes the lease.release return value during the enclosing return. + lease.release() + + +# What: define the test_cancelled_daemon_shutdown_finishes_stop_and_requests_process_exit test around local fixtures; why: this test groups the arrange, act, and assertions that protect the cancelled daemon shutdown finishes stop and requests process exit outcome. +def test_cancelled_daemon_shutdown_finishes_stop_and_requests_process_exit(): + # What: act by calling threading.Event and capture entered; why: the cancelled daemon shutdown finishes stop and requests process exit test asserts the response, state, or failure produced by this call. + entered = threading.Event() + # What: act by calling threading.Event and capture finish; why: the cancelled daemon shutdown finishes stop and requests process exit test asserts the response, state, or failure produced by this call. + finish = threading.Event() + + # What: define BlockingShutdownManager as the owner of shutdown; why: daemon callers use this class boundary so those methods share one blocking shutdown manager state invariant. + class BlockingShutdownManager(Manager): + # What: define the shutdown test helper around timeout and force; why: the cancelled daemon shutdown finishes stop and requests process exit scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def shutdown(self, timeout=None, force=False): + # What: act by calling entered.set with the declared inputs; why: the cancelled daemon shutdown finishes stop and requests process exit scenario observes the entered.set return value during assert finish wait. + entered.set() + # What: assert that finish wait 2; why: this assertion protects the cancelled daemon shutdown finishes stop and requests process exit regression after the test's arranged inputs and exercised call. + assert finish.wait(2) + # What: arrange model as the fixture input; why: the cancelled daemon shutdown finishes stop and requests process exit test consumes this named precondition before exercising the behavior. + self.model = None + # What: arrange the stopped field as true; why: BlockingShutdownManager.shutdown carries stopped into return {"stopped": True, "already": False, "accounting": None}. + return {"stopped": True, "already": False, "accounting": None} + + # What: act by calling BlockingShutdownManager and capture manager; why: the cancelled daemon shutdown finishes stop and requests process exit test asserts the response, state, or failure produced by this call. + manager = BlockingShutdownManager() + # What: act by calling RoutingCoordinator and capture router; why: the cancelled daemon shutdown finishes stop and requests process exit test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: arrange exits as the fixture input; why: the cancelled daemon shutdown finishes stop and requests process exit test consumes this named precondition before exercising the behavior. + exits = [] + + # What: define the scenario test helper around app; why: the cancelled daemon shutdown finishes stop and requests process exit scenario calls this helper to produce or observe the exact behavior checked by its assertions. + async def scenario(app): + # What: act by calling exits.append and capture request shutdown; why: the cancelled daemon shutdown finishes stop and requests process exit test asserts the response, state, or failure produced by this call. + app.state.request_shutdown = lambda: exits.append("requested") + # What: act by calling httpx.ASGITransport and capture transport; why: the cancelled daemon shutdown finishes stop and requests process exit test asserts the response, state, or failure produced by this call. + transport = httpx.ASGITransport(app=app) + # What: enter the httpx.AsyncClient managed context before request asyncio create task client post shutdown json force; why: scenario releases this resource or lock after request asyncio create task client post shutdown json force on both success and failure paths. + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + # What: act by calling asyncio.create_task and capture request; why: the cancelled daemon shutdown finishes stop and requests process exit test asserts the response, state, or failure produced by this call. + request = asyncio.create_task(client.post("/shutdown", json={"force": True})) + # What: act across range to perform is set and entered; why: the cancelled daemon shutdown finishes stop and requests process exit scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on is set and entered before the computed value; why: the cancelled daemon shutdown finishes stop and requests process exit scenario admits the computed value only for this predicate and excludes the opposite state. + if entered.is_set(): + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the cancelled daemon shutdown finishes stop and requests process exit scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the cancelled daemon shutdown finishes stop and requests process exit scenario observes the asyncio.sleep return value during assert entered is set. + await asyncio.sleep(0.01) + # What: assert that entered is set; why: this assertion protects the cancelled daemon shutdown finishes stop and requests process exit regression after the test's arranged inputs and exercised call. + assert entered.is_set() + # What: act by calling request.cancel with the declared inputs; why: the cancelled daemon shutdown finishes stop and requests process exit scenario observes the request.cancel return value during request cancel. + request.cancel() + # What: act by calling request.cancel with the declared inputs; why: the cancelled daemon shutdown finishes stop and requests process exit scenario observes the request.cancel return value during await asyncio sleep. + request.cancel() + # What: act by calling asyncio.sleep with 0 05; why: the cancelled daemon shutdown finishes stop and requests process exit scenario observes the asyncio.sleep return value during assert not request done. + await asyncio.sleep(0.05) + # What: assert that request done is false; why: this assertion protects the cancelled daemon shutdown finishes stop and requests process exit regression after the test's arranged inputs and exercised call. + assert not request.done() + # What: act by calling finish.set with the declared inputs; why: the cancelled daemon shutdown finishes stop and requests process exit scenario observes the finish.set return value during with pytest raises asyncio cancelled error. + finish.set() + # What: assert the pytest.raises failure context; why: the cancelled daemon shutdown finishes stop and requests process exit scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(asyncio.CancelledError): + # What: arrange the await request portion of the enclosing predicate; why: this clause remains in the cancelled daemon shutdown finishes stop and requests process exit scenario\'s enclosing expression so its grouping and evaluation order stay intact. + await request + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_cancelled_daemon_shutdown_finishes_stop_and_requests_process_exit releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the cancelled daemon shutdown finishes stop and requests process exit test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_cancelled_daemon_shutdown_finishes_stop_and_requests_process_exit; why: test_cancelled_daemon_shutdown_finishes_stop_and_requests_process_exit consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to catalog; why: the cancelled daemon shutdown finishes stop and requests process exit scenario binds this lifecycle value to catalog's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog(), router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_cancelled_daemon_shutdown_finishes_stop_and_requests_process_exit groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling asyncio.run with scenario and app; why: the cancelled daemon shutdown finishes stop and requests process exit scenario observes the asyncio.run return value during assert exits requested. + asyncio.run(scenario(app)) + + # What: assert that exits equals requested; why: this assertion protects the cancelled daemon shutdown finishes stop and requests process exit regression after the test's arranged inputs and exercised call. + assert exits == ["requested"] + # What: assert that router status shutting down is true; why: this assertion protects the cancelled daemon shutdown finishes stop and requests process exit regression after the test's arranged inputs and exercised call. + assert router.status()["shuttingDown"] is True + + +# What: define the test_daemon_shutdown_latches_before_single_lifecycle_worker_is_available test around local fixtures; why: this test groups the arrange, act, and assertions that protect the daemon shutdown latches before single lifecycle worker is available outcome. +def test_daemon_shutdown_latches_before_single_lifecycle_worker_is_available(): + # What: act by calling threading.Event and capture start entered; why: the daemon shutdown latches before single lifecycle worker is available test asserts the response, state, or failure produced by this call. + start_entered = threading.Event() + # What: act by calling threading.Event and capture finish start; why: the daemon shutdown latches before single lifecycle worker is available test asserts the response, state, or failure produced by this call. + finish_start = threading.Event() + + # What: define BlockingLifecycleManager as the owner of start and shutdown; why: daemon callers use this class boundary so those methods share one blocking lifecycle manager state invariant. + class BlockingLifecycleManager(Manager): + # What: define the start test helper around model and port and args; why: the daemon shutdown latches before single lifecycle worker is available scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def start(self, model, port, args): + # What: arrange the exact self calls append start model fixture fragment; why: the daemon shutdown latches before single lifecycle worker is available scenario feeds this byte-preserved fragment through self.calls.append(("start", model)) before asserting its protocol or parser result. + self.calls.append(("start", model)) + # What: act by calling start_entered.set with the declared inputs; why: the daemon shutdown latches before single lifecycle worker is available scenario observes the start_entered.set return value during assert finish start wait. + start_entered.set() + # What: assert that finish start wait 2; why: this assertion protects the daemon shutdown latches before single lifecycle worker is available regression after the test's arranged inputs and exercised call. + assert finish_start.wait(2) + # What: act by calling list and capture model and port and args; why: the daemon shutdown latches before single lifecycle worker is available test asserts the response, state, or failure produced by this call. + self.model, self.port, self.args = model, port, list(args) + # What: arrange pid from 1; why: the daemon shutdown latches before single lifecycle worker is available scenario uses pid during return pid self pid before checking the protected result. + self.pid += 1 + # What: arrange the pid field as pid; why: BlockingLifecycleManager.start carries pid into return {"pid": self.pid}. + return {"pid": self.pid} + + # What: define the shutdown test helper around timeout and force; why: the daemon shutdown latches before single lifecycle worker is available scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def shutdown(self, timeout=None, force=False): + # What: arrange the exact self calls append shutdown force fixture fragment; why: the daemon shutdown latches before single lifecycle worker is available scenario feeds this byte-preserved fragment through self.calls.append(("shutdown", force)) before asserting its protocol or parser result. + self.calls.append(("shutdown", force)) + # What: arrange model as the fixture input; why: the daemon shutdown latches before single lifecycle worker is available test consumes this named precondition before exercising the behavior. + self.model = None + # What: arrange the stopped field as true; why: BlockingLifecycleManager.shutdown carries stopped into return {"stopped": True, "already": False, "accounting": None}. + return {"stopped": True, "already": False, "accounting": None} + + # What: act by calling BlockingLifecycleManager and capture manager; why: the daemon shutdown latches before single lifecycle worker is available test asserts the response, state, or failure produced by this call. + manager = BlockingLifecycleManager() + # What: act by calling RoutingCoordinator and capture router; why: the daemon shutdown latches before single lifecycle worker is available test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: arrange exits as the fixture input; why: the daemon shutdown latches before single lifecycle worker is available test consumes this named precondition before exercising the behavior. + exits = [] + + # What: define the scenario test helper around app; why: the daemon shutdown latches before single lifecycle worker is available scenario calls this helper to produce or observe the exact behavior checked by its assertions. + async def scenario(app): + # What: act by calling exits.append and capture request shutdown; why: the daemon shutdown latches before single lifecycle worker is available test asserts the response, state, or failure produced by this call. + app.state.request_shutdown = lambda: exits.append("requested") + # What: act by calling httpx.ASGITransport and capture transport; why: the daemon shutdown latches before single lifecycle worker is available test asserts the response, state, or failure produced by this call. + transport = httpx.ASGITransport(app=app) + # What: arrange async with httpx AsyncClient transport transport base url http test as client for the scenario; why: test daemon shutdown latches before si requires this concrete input or helper state before exercising the behavior under test. + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + # What: act by calling asyncio.create_task and capture manual; why: the daemon shutdown latches before single lifecycle worker is available test asserts the response, state, or failure produced by this call. + manual = asyncio.create_task(client.post( + # What: arrange the model field as legacy and gguf; why: scenario sends this field through manual so the router selects the canonical model or alias for upstream dispatch. + "/engine/start", json={"model": "legacy.gguf", "port": 1930} + # What: arrange the grouped source fragment for the scenario; why: test router test daemon shutdown latches before single lifecycle worker is available requires this concrete input or helper state before exercising the behavior under test. + )) + # What: act across range to perform is set and start entered; why: the daemon shutdown latches before single lifecycle worker is available scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on is set and start entered before the computed value; why: the daemon shutdown latches before single lifecycle worker is available scenario admits the computed value only for this predicate and excludes the opposite state. + if start_entered.is_set(): + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the daemon shutdown latches before single lifecycle worker is available scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the daemon shutdown latches before single lifecycle worker is available scenario observes the asyncio.sleep return value during assert start entered is set. + await asyncio.sleep(0.01) + # What: assert that start entered is set; why: this assertion protects the daemon shutdown latches before single lifecycle worker is available regression after the test's arranged inputs and exercised call. + assert start_entered.is_set() + + # What: act by calling asyncio.create_task and capture shutdown; why: the daemon shutdown latches before single lifecycle worker is available test asserts the response, state, or failure produced by this call. + shutdown = asyncio.create_task(client.post("/shutdown", json={})) + # What: act across range to perform status and router; why: the daemon shutdown latches before single lifecycle worker is available scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on status and router before the computed value; why: the daemon shutdown latches before single lifecycle worker is available scenario admits the computed value only for this predicate and excludes the opposite state. + if router.status()["shuttingDown"]: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the daemon shutdown latches before single lifecycle worker is available scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling asyncio.sleep with 0 01; why: the daemon shutdown latches before single lifecycle worker is available scenario observes the asyncio.sleep return value during assert router status shutting down is. + await asyncio.sleep(0.01) + # What: assert that router status shutting down is true; why: this assertion protects the daemon shutdown latches before single lifecycle worker is available regression after the test's arranged inputs and exercised call. + assert router.status()["shuttingDown"] is True + # What: assert that shutdown done is false; why: this assertion protects the daemon shutdown latches before single lifecycle worker is available regression after the test's arranged inputs and exercised call. + assert not shutdown.done() + # What: assert the pytest.raises failure context; why: the daemon shutdown latches before single lifecycle worker is available scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RoutingError) as exc: + # What: arrange the exact router acquire low fixture fragment; why: the daemon shutdown latches before single lifecycle worker is available scenario feeds this byte-preserved fragment through router.acquire("low") before asserting its protocol or parser result. + router.acquire("low") + # What: assert that exc value code equals router shutting down; why: this assertion protects the daemon shutdown latches before single lifecycle worker is available regression after the test's arranged inputs and exercised call. + assert exc.value.code == "router_shutting_down" + + # What: act by calling finish_start.set with the declared inputs; why: the daemon shutdown latches before single lifecycle worker is available scenario observes the finish_start.set return value during assert await manual status code. + finish_start.set() + # What: assert that await manual status code equals 200; why: this assertion protects the daemon shutdown latches before single lifecycle worker is available regression after the test's arranged inputs and exercised call. + assert (await manual).status_code == 200 + # What: arrange response as shutdown; why: the daemon shutdown latches before single lifecycle worker is available test consumes this named precondition before exercising the behavior. + response = await shutdown + # What: assert that response status code equals 200; why: this assertion protects the daemon shutdown latches before single lifecycle worker is available regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_daemon_shutdown_latches_before_single_lifecycle_worker_is_available releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the daemon shutdown latches before single lifecycle worker is available test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_daemon_shutdown_latches_before_single_lifecycle_worker_is_available; why: test_daemon_shutdown_latches_before_single_lifecycle_worker_is_available consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to catalog; why: the daemon shutdown latches before single lifecycle worker is available scenario binds this lifecycle value to catalog's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog(), router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_daemon_shutdown_latches_before_single_lifecycle_worker_is_available groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling asyncio.run with scenario and app; why: the daemon shutdown latches before single lifecycle worker is available scenario observes the asyncio.run return value during assert manager calls start legacy gguf shutdown. + asyncio.run(scenario(app)) + + # What: assert that manager calls equals start legacy gguf shutdown false; why: this assertion protects the daemon shutdown latches before single lifecycle worker is available regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "legacy.gguf"), ("shutdown", False)] + # What: assert that exits equals requested; why: this assertion protects the daemon shutdown latches before single lifecycle worker is available regression after the test's arranged inputs and exercised call. + assert exits == ["requested"] + + +# What: define the test_coordinated_daemon_exit_drains_then_detaches_once_for_readoption test around local fixtures; why: this test groups the arrange, act, and assertions that protect the coordinated daemon exit drains then detaches once for readoption outcome. +def test_coordinated_daemon_exit_drains_then_detaches_once_for_readoption(): + # What: arrange the class DetachingManager Manager test helper boundary; why: test router test coordinated daemon exit drains then detaches once for readoption uses this local double to isolate the behavior checked by its assertions. + class DetachingManager(Manager): + # What: define the detach test helper around captured fixture state; why: the coordinated daemon exit drains then detaches once for readoption scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def detach(self): + # What: arrange the exact self calls append detach self model fixture fragment; why: the coordinated daemon exit drains then detaches once for readoption scenario feeds this byte-preserved fragment through self.calls.append(("detach", self.model)) before asserting its protocol or parser result. + self.calls.append(("detach", self.model)) + + # What: act by calling DetachingManager and capture manager; why: the coordinated daemon exit drains then detaches once for readoption test asserts the response, state, or failure produced by this call. + manager = DetachingManager() + # What: act by calling RoutingCoordinator and capture router; why: the coordinated daemon exit drains then detaches once for readoption test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: act by calling router.acquire and capture active; why: the coordinated daemon exit drains then detaches once for readoption test asserts the response, state, or failure produced by this call. + active = router.acquire("low") + # What: arrange result as the fixture input; why: the coordinated daemon exit drains then detaches once for readoption test consumes this named precondition before exercising the behavior. + result = {} + # What: act by calling threading.Thread and capture exiting; why: the coordinated daemon exit drains then detaches once for readoption test asserts the response, state, or failure produced by this call. + exiting = threading.Thread( + # What: arrange target to result.setdefault; why: the coordinated daemon exit drains then detaches once for readoption scenario binds this setdefault and result and coordinated exit and router and value value to result.setdefault's target input. + target=lambda: result.setdefault( + # What: arrange stop child to router.coordinated_exit; why: the coordinated daemon exit drains then detaches once for readoption scenario binds this false value to router.coordinated_exit's stop child input. + "value", router.coordinated_exit(stop_child=False) + # What: arrange the result.setdefault call with coordinated exit; why: test_coordinated_daemon_exit_drains_then_detaches_once_for_readoption groups the supplied clauses as one result.setdefault call before its value is consumed. + ) + # What: arrange the threading.Thread call with target; why: test_coordinated_daemon_exit_drains_then_detaches_once_for_readoption groups the supplied clauses as one threading.Thread call before its value is consumed. + ) + # What: act by calling exiting.start with the declared inputs; why: the coordinated daemon exit drains then detaches once for readoption scenario observes the exiting.start return value during for value in range. + exiting.start() + # What: act across range to perform status and router; why: the coordinated daemon exit drains then detaches once for readoption scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on status and router before the computed value; why: the coordinated daemon exit drains then detaches once for readoption scenario admits the computed value only for this predicate and excludes the opposite state. + if router.status()["shuttingDown"]: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the coordinated daemon exit drains then detaches once for readoption scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling time.sleep with 0 01; why: the coordinated daemon exit drains then detaches once for readoption scenario observes the time.sleep return value during assert router status shutting down is. + time.sleep(0.01) + # What: assert that router status shutting down is true; why: this assertion protects the coordinated daemon exit drains then detaches once for readoption regression after the test's arranged inputs and exercised call. + assert router.status()["shuttingDown"] is True + # What: assert that exiting is alive; why: this assertion protects the coordinated daemon exit drains then detaches once for readoption regression after the test's arranged inputs and exercised call. + assert exiting.is_alive() + # What: assert that manager calls equals start low gguf; why: this assertion protects the coordinated daemon exit drains then detaches once for readoption regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf")] + + # What: act by calling active.release with the declared inputs; why: the coordinated daemon exit drains then detaches once for readoption scenario observes the active.release return value during exiting join. + active.release() + # What: act by calling exiting.join with 2; why: the coordinated daemon exit drains then detaches once for readoption scenario observes the exiting.join return value during assert not exiting is alive. + exiting.join(2) + # What: assert that exiting is alive is false; why: this assertion protects the coordinated daemon exit drains then detaches once for readoption regression after the test's arranged inputs and exercised call. + assert not exiting.is_alive() + # What: assert that result value is group delimiter; why: this assertion protects the coordinated daemon exit drains then detaches once for readoption regression after the test's arranged inputs and exercised call. + assert result["value"] is None + # What: assert that manager calls equals start low gguf detach low gguf; why: this assertion protects the coordinated daemon exit drains then detaches once for readoption regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf"), ("detach", "low.gguf")] + # What: assert that manager model equals low gguf; why: this assertion protects the coordinated daemon exit drains then detaches once for readoption regression after the test's arranged inputs and exercised call. + assert manager.model == "low.gguf" + # What: assert that router status active profile is group delimiter; why: this assertion protects the coordinated daemon exit drains then detaches once for readoption regression after the test's arranged inputs and exercised call. + assert router.status()["activeProfile"] is None + + # Uvicorn lifespan can run after POST /shutdown already completed. The + # repeated exit hook must not detach or stop the child a second time. + # What: assert that router coordinated exit stop child false is group delimiter; why: this assertion protects the coordinated daemon exit drains then detaches once for readoption regression after the test's arranged inputs and exercised call. + assert router.coordinated_exit(stop_child=False) is None + # What: assert that manager calls equals start low gguf detach low gguf; why: this assertion protects the coordinated daemon exit drains then detaches once for readoption regression after the test's arranged inputs and exercised call. + assert manager.calls == [("start", "low.gguf"), ("detach", "low.gguf")] + + +# What: define the test_coordinated_exit_waits_for_preempted_manual_transaction_token test around local fixtures; why: this test groups the arrange, act, and assertions that protect the coordinated exit waits for preempted manual transaction token outcome. +def test_coordinated_exit_waits_for_preempted_manual_transaction_token(): + # What: define DetachingManager as the owner of detach; why: daemon callers use this class boundary so those methods share one detaching manager state invariant. + class DetachingManager(Manager): + # What: define the detach test helper around captured fixture state; why: the coordinated exit waits for preempted manual transaction token scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def detach(self): + # What: arrange the exact self calls append detach self model fixture fragment; why: the coordinated exit waits for preempted manual transaction token scenario feeds this byte-preserved fragment through self.calls.append(("detach", self.model)) before asserting its protocol or parser result. + self.calls.append(("detach", self.model)) + + # What: act by calling DetachingManager and capture manager; why: the coordinated exit waits for preempted manual transaction token test asserts the response, state, or failure produced by this call. + manager = DetachingManager() + # What: act by calling RoutingCoordinator and capture router; why: the coordinated exit waits for preempted manual transaction token test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # What: act by calling router.begin_manual_lifecycle and capture older; why: the coordinated exit waits for preempted manual transaction token test asserts the response, state, or failure produced by this call. + older = router.begin_manual_lifecycle() + # What: act by calling router.begin_manual_lifecycle and capture newer; why: the coordinated exit waits for preempted manual transaction token test asserts the response, state, or failure produced by this call. + newer = router.begin_manual_lifecycle(preempt_manual=True) + # What: act by calling router.end_manual_lifecycle with newer; why: the coordinated exit waits for preempted manual transaction token scenario observes the router.end_manual_lifecycle return value during assert router status switching is. + router.end_manual_lifecycle(newer) + # What: assert that router status switching is false; why: this assertion protects the coordinated exit waits for preempted manual transaction token regression after the test's arranged inputs and exercised call. + assert router.status()["switching"] is False + + # What: act by calling threading.Thread and capture exiting; why: the coordinated exit waits for preempted manual transaction token test asserts the response, state, or failure produced by this call. + exiting = threading.Thread( + # What: arrange target to router.coordinated_exit; why: the coordinated exit waits for preempted manual transaction token scenario binds this coordinated exit and router and false value to router.coordinated_exit's target input. + target=lambda: router.coordinated_exit(stop_child=False) + # What: arrange the threading.Thread call with target; why: test_coordinated_exit_waits_for_preempted_manual_transaction_token groups the supplied clauses as one threading.Thread call before its value is consumed. + ) + # What: act by calling exiting.start with the declared inputs; why: the coordinated exit waits for preempted manual transaction token scenario observes the exiting.start return value during for value in range. + exiting.start() + # What: act across range to perform status and router; why: the coordinated exit waits for preempted manual transaction token scenario repeats the body only while or for the loop header admits an iteration. + for _ in range(100): + # What: act on status and router before the computed value; why: the coordinated exit waits for preempted manual transaction token scenario admits the computed value only for this predicate and excludes the opposite state. + if router.status()["shuttingDown"]: + # What: arrange the break portion of the enclosing predicate; why: this clause remains in the coordinated exit waits for preempted manual transaction token scenario\'s enclosing expression so its grouping and evaluation order stay intact. + break + # What: act by calling time.sleep with 0 01; why: the coordinated exit waits for preempted manual transaction token scenario observes the time.sleep return value during assert router status shutting down is. + time.sleep(0.01) + # What: assert that router status shutting down is true; why: this assertion protects the coordinated exit waits for preempted manual transaction token regression after the test's arranged inputs and exercised call. + assert router.status()["shuttingDown"] is True + # What: assert that exiting is alive; why: this assertion protects the coordinated exit waits for preempted manual transaction token regression after the test's arranged inputs and exercised call. + assert exiting.is_alive() + # What: assert that manager calls equals group delimiter; why: this assertion protects the coordinated exit waits for preempted manual transaction token regression after the test's arranged inputs and exercised call. + assert manager.calls == [] + + # What: act by calling router.end_manual_lifecycle with older; why: the coordinated exit waits for preempted manual transaction token scenario observes the router.end_manual_lifecycle return value during exiting join. + router.end_manual_lifecycle(older) + # What: act by calling exiting.join with 2; why: the coordinated exit waits for preempted manual transaction token scenario observes the exiting.join return value during assert not exiting is alive. + exiting.join(2) + # What: assert that exiting is alive is false; why: this assertion protects the coordinated exit waits for preempted manual transaction token regression after the test's arranged inputs and exercised call. + assert not exiting.is_alive() + # What: assert that manager calls equals detach; why: this assertion protects the coordinated exit waits for preempted manual transaction token regression after the test's arranged inputs and exercised call. + assert manager.calls == [("detach", None)] + + +# What: define the test_router_management_ui_has_no_embedded_operational_data_and_hardware_is_gated test around local fixtures; why: this test groups the arrange, act, and assertions that protect the router management ui has no embedded operational data and hardware is gated outcome. +def test_router_management_ui_has_no_embedded_operational_data_and_hardware_is_gated(): + # What: act by calling Manager and capture manager; why: the router management ui has no embedded operational data and hardware is gated test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the router management ui has no embedded operational data and hardware is gated test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and private and models and low; why: test_router_management_ui_has_no_embedded_operational_data_and_hardware_is_gated carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": ModelProfile("low", "/private/models/low.gguf", ())}, + # What: arrange settings to RouterSettings; why: the router management ui has no embedded operational data and hardware is gated scenario binds this router settings and router test key value to RouterSettings's settings input. + settings=RouterSettings(api_keys=("router-test-key",)), + # What: arrange the ModelCatalog call with settings; why: test_router_management_ui_has_no_embedded_operational_data_and_hardware_is_gated groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the router management ui has no embedded operational data and hardware is gated test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_router_management_ui_has_no_embedded_operational_data_and_hardware_is_gated releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the router management ui has no embedded operational data and hardware is gated test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange manager to LogRing; why: the router management ui has no embedded operational data and hardware is gated scenario binds this manager value to LogRing's manager input. + manager=manager, ring=LogRing(), probe=object(), + # What: arrange the ram bytes field as 123; why: test_router_management_ui_has_no_embedded_operational_data_and_hardware_is_gated carries ram bytes through app into client test client app. + footprint_fn=lambda pid: {"ramBytes": 123, "vramBytes": 456}, + # What: arrange lifecycle pool to build_app; why: the router management ui has no embedded operational data and hardware is gated scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_router_management_ui_has_no_embedded_operational_data_and_hardware_is_gated groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling TestClient and capture client; why: the router management ui has no embedded operational data and hardware is gated test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: act by calling client.get and capture page; why: the router management ui has no embedded operational data and hardware is gated test asserts the response, state, or failure produced by this call. + page = client.get("/ui/") + # What: assert that client get router hardware status code equals 401; why: this assertion protects the router management ui has no embedded operational data and hardware is gated regression after the test's arranged inputs and exercised call. + assert client.get("/router/hardware").status_code == 401 + # What: act by calling client.get and capture hardware; why: the router management ui has no embedded operational data and hardware is gated test asserts the response, state, or failure produced by this call. + hardware = client.get("/router/hardware", headers={"Authorization": "Bearer router-test-key"}) + # What: assert that page status code equals 200; why: this assertion protects the router management ui has no embedded operational data and hardware is gated regression after the test's arranged inputs and exercised call. + assert page.status_code == 200 + # What: assert that router load is present in page text; why: this assertion protects the router management ui has no embedded operational data and hardware is gated regression after the test's arranged inputs and exercised call. + assert "/router/load" in page.text + # What: assert that router hardware is present in page text; why: this assertion protects the router management ui has no embedded operational data and hardware is gated regression after the test's arranged inputs and exercised call. + assert "/router/hardware" in page.text + # What: assert that router activity limit 25 is present in page text; why: this assertion protects the router management ui has no embedded operational data and hardware is gated regression after the test's arranged inputs and exercised call. + assert "/router/activity?limit=25" in page.text + # What: assert that router performance is present in page text; why: this assertion protects the router management ui has no embedded operational data and hardware is gated regression after the test's arranged inputs and exercised call. + assert "/router/performance" in page.text + # What: assert that router captures is present in page text; why: this assertion protects the router management ui has no embedded operational data and hardware is gated regression after the test's arranged inputs and exercised call. + assert "/router/captures/" in page.text + # What: assert that inner html is absent from page text; why: this assertion protects the router management ui has no embedded operational data and hardware is gated regression after the test's arranged inputs and exercised call. + assert "innerHTML" not in page.text + # What: assert that captures may contain prompts and are is present in page text; why: this assertion protects the router management ui has no embedded operational data and hardware is gated regression after the test's arranged inputs and exercised call. + assert "Captures may contain prompts and are fetched only when selected" in page.text + # What: assert that private models low gguf is absent from page text; why: this assertion protects the router management ui has no embedded operational data and hardware is gated regression after the test's arranged inputs and exercised call. + assert "/private/models/low.gguf" not in page.text + # What: assert that router test key is absent from page text; why: this assertion protects the router management ui has no embedded operational data and hardware is gated regression after the test's arranged inputs and exercised call. + assert "router-test-key" not in page.text + # What: assert the expected hardware json == outcome; why: test router test router management ui has no embedded operational data and hardware is gated protects its regression by requiring this observable result after the exercised behavior. + assert hardware.json() == { + # What: arrange engine running False pid 100 port None for the scenario; why: test router test router management ui has no embedded operational data and hardware is gated requires this concrete input or helper state before exercising the behavior under test. + "engine": {"running": False, "pid": 100, "port": None}, + # What: arrange memory ramBytes 123 vramBytes 456 for the scenario; why: test router test router management ui has no embedded operational data and hardware is gated requires this concrete input or helper state before exercising the behavior under test. + "memory": {"ramBytes": 123, "vramBytes": 456}, + # What: arrange the grouped source fragment for the scenario; why: test router test router management ui has no embedded operational data and hardware is gated requires this concrete input or helper state before exercising the behavior under test. + } + + +# What: define the test_periodic_performance_api_is_authenticated_filterable_and_private test around local fixtures; why: this test groups the arrange, act, and assertions that protect the periodic performance api is authenticated filterable and private outcome. +def test_periodic_performance_api_is_authenticated_filterable_and_private(): + # What: act by calling Manager and capture manager; why: the periodic performance api is authenticated filterable and private test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the periodic performance api is authenticated filterable and private test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and private and models and low; why: test_periodic_performance_api_is_authenticated_filterable_and_private carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": ModelProfile("low", "/private/models/low.gguf", ())}, + # What: arrange settings to RouterSettings; why: the periodic performance api is authenticated filterable and private scenario binds this router settings and router test key value to RouterSettings's settings input. + settings=RouterSettings(api_keys=("router-test-key",)), + # What: arrange the ModelCatalog call with settings; why: test_periodic_performance_api_is_authenticated_filterable_and_private groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the periodic performance api is authenticated filterable and private test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: arrange footprint as ram bytes and vram bytes and pids and ram available and vram available; why: the periodic performance api is authenticated filterable and private test consumes this named precondition before exercising the behavior. + footprint = { + # What: arrange the ram bytes field as 123; why: test_periodic_performance_api_is_authenticated_filterable_and_private carries ram bytes through footprint into footprint fn lambda pid footprint. + "ramBytes": 123, "vramBytes": 456, "pids": [100], + # What: arrange the ram available field as true; why: test_periodic_performance_api_is_authenticated_filterable_and_private carries ram available through footprint into footprint fn lambda pid footprint. + "ramAvailable": True, "vramAvailable": True, + # What: arrange the ram source field as test pss; why: test_periodic_performance_api_is_authenticated_filterable_and_private carries ram source through footprint into footprint fn lambda pid footprint. + "ramSource": "test-pss", "vramSource": "test-gpu", + # What: arrange the footprint mapping with ram bytes and vram bytes and pids and ram available and vram available; why: test_periodic_performance_api_is_authenticated_filterable_and_private groups the supplied clauses as one footprint mapping before its value is consumed. + } + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_periodic_performance_api_is_authenticated_filterable_and_private releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the periodic performance api is authenticated filterable and private test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange manager to LogRing; why: the periodic performance api is authenticated filterable and private scenario binds this manager value to LogRing's manager input. + manager=manager, ring=LogRing(), probe=object(), + # What: arrange the pid input for test_periodic_performance_api_is_authenticated_filterable_and_private; why: test_periodic_performance_api_is_authenticated_filterable_and_private consumes pid during signature binding, so callers must bind it with the other signature inputs. + footprint_fn=lambda pid: footprint, + # What: arrange lifecycle pool to build_app; why: the periodic performance api is authenticated filterable and private scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_periodic_performance_api_is_authenticated_filterable_and_private groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling app.state.performance_monitor.sample_once with the declared inputs; why: the periodic performance api is authenticated filterable and private scenario observes the app.state.performance_monitor.sample_once return value during client test client app. + app.state.performance_monitor.sample_once() + # What: act by calling TestClient and capture client; why: the periodic performance api is authenticated filterable and private test asserts the response, state, or failure produced by this call. + client = TestClient(app) + # What: arrange headers as authorization and bearer and router test key; why: the periodic performance api is authenticated filterable and private test consumes this named precondition before exercising the behavior. + headers = {"Authorization": "Bearer router-test-key"} + # What: assert that client get api performance status code equals 401; why: this assertion protects the periodic performance api is authenticated filterable and private regression after the test's arranged inputs and exercised call. + assert client.get("/api/performance").status_code == 401 + # What: act by calling client.get and capture response; why: the periodic performance api is authenticated filterable and private test asserts the response, state, or failure produced by this call. + response = client.get("/api/performance", headers=headers) + # What: act by calling response.json and capture timestamp; why: the periodic performance api is authenticated filterable and private test asserts the response, state, or failure produced by this call. + timestamp = response.json()["sys_stats"][0]["timestamp"] + # What: act by calling client.get and capture filtered; why: the periodic performance api is authenticated filterable and private test asserts the response, state, or failure produced by this call. + filtered = client.get( + # What: arrange the after field as timestamp; why: test_periodic_performance_api_is_authenticated_filterable_and_private carries after through filtered into assert filtered json sys stats equals. + "/router/performance", params={"after": timestamp}, headers=headers + # What: arrange the client.get call with params and headers; why: test_periodic_performance_api_is_authenticated_filterable_and_private groups the supplied clauses as one client.get call before its value is consumed. + ) + # What: act by calling client.get and capture invalid; why: the periodic performance api is authenticated filterable and private test asserts the response, state, or failure produced by this call. + invalid = client.get( + # What: arrange the after field as not a time; why: test_periodic_performance_api_is_authenticated_filterable_and_private carries after through invalid into assert invalid status code equals 400. + "/api/performance", params={"after": "not-a-time"}, headers=headers + # What: arrange the client.get call with params and headers; why: test_periodic_performance_api_is_authenticated_filterable_and_private groups the supplied clauses as one client.get call before its value is consumed. + ) + # What: assert that response status code equals 200; why: this assertion protects the periodic performance api is authenticated filterable and private regression after the test's arranged inputs and exercised call. + assert response.status_code == 200 + # What: assert that response json gpu stats equals group delimiter; why: this assertion protects the periodic performance api is authenticated filterable and private regression after the test's arranged inputs and exercised call. + assert response.json()["gpu_stats"] == [] + # What: assert the expected response json sys stats 0 == outcome; why: test router test periodic performance api is authenticated filterable and private protects its regression by requiring this observable result after the exercised behavior. + assert response.json()["sys_stats"][0] == { + # What: arrange timestamp timestamp scope engine process tree for the scenario; why: test router test periodic performance api is authenticated filterable and private requires this concrete input or helper state before exercising the behavior under test. + "timestamp": timestamp, "scope": "engine-process-tree", + # What: arrange ram bytes 123 vram bytes 456 for the scenario; why: test router test periodic performance api is authenticated filterable and private requires this concrete input or helper state before exercising the behavior under test. + "ram_bytes": 123, "vram_bytes": 456, + # What: arrange ram available True vram available True for the scenario; why: test router test periodic performance api is authenticated filterable and private requires this concrete input or helper state before exercising the behavior under test. + "ram_available": True, "vram_available": True, + # What: arrange ram source test pss vram source test gpu for the scenario; why: test pss vram source test gpu requires this concrete input or helper state before exercising the behavior under test. + "ram_source": "test-pss", "vram_source": "test-gpu", + # What: arrange the grouped source fragment for the scenario; why: test router test periodic performance api is authenticated filterable and private requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that private not in response text and pids not in response text; why: this assertion protects the periodic performance api is authenticated filterable and private regression after the test's arranged inputs and exercised call. + assert "/private/" not in response.text and "pids" not in response.text + # What: assert that filtered json sys stats equals group delimiter; why: this assertion protects the periodic performance api is authenticated filterable and private regression after the test's arranged inputs and exercised call. + assert filtered.json()["sys_stats"] == [] + # What: assert that invalid status code equals 400; why: this assertion protects the periodic performance api is authenticated filterable and private regression after the test's arranged inputs and exercised call. + assert invalid.status_code == 400 + + +# What: define the test_disabled_performance_api_matches_pinned_unavailable_contract test around local fixtures; why: this test groups the arrange, act, and assertions that protect the disabled performance api matches pinned unavailable contract outcome. +def test_disabled_performance_api_matches_pinned_unavailable_contract(): + # What: act by calling Manager and capture manager; why: the disabled performance api matches pinned unavailable contract test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog and capture catalog doc; why: the disabled performance api matches pinned unavailable contract test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog( + # What: arrange the low field as model profile and low and low and gguf; why: test_disabled_performance_api_matches_pinned_unavailable_contract carries low through catalog doc into router routing coordinator manager catalog doc object ready fn ready. + {"low": ModelProfile("low", "low.gguf", ())}, + # What: arrange settings to RouterSettings; why: the disabled performance api matches pinned unavailable contract scenario binds this router settings and true value to RouterSettings's settings input. + settings=RouterSettings(performance_disabled=True), + # What: arrange the ModelCatalog call with settings; why: test_disabled_performance_api_matches_pinned_unavailable_contract groups the supplied clauses as one ModelCatalog call before its value is consumed. + ) + # What: act by calling RoutingCoordinator and capture router; why: the disabled performance api matches pinned unavailable contract test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_disabled_performance_api_matches_pinned_unavailable_contract releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the disabled performance api matches pinned unavailable contract test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_disabled_performance_api_matches_pinned_unavailable_contract; why: test_disabled_performance_api_matches_pinned_unavailable_contract consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the disabled performance api matches pinned unavailable contract scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_disabled_performance_api_matches_pinned_unavailable_contract groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: act by calling operation.get and capture response; why: the disabled performance api matches pinned unavailable contract test asserts the response, state, or failure produced by this call. + response = TestClient(app).get("/api/performance") + # What: assert that response status code equals 503; why: this assertion protects the disabled performance api matches pinned unavailable contract regression after the test's arranged inputs and exercised call. + assert response.status_code == 503 + # What: assert that response json equals enabled false; why: this assertion protects the disabled performance api matches pinned unavailable contract regression after the test's arranged inputs and exercised call. + assert response.json() == {"enabled": False} + + +# What: define the test_catalog_watcher_applies_only_valid_idle_replacements test around tmp path; why: this test groups the arrange, act, and assertions that protect the catalog watcher applies only valid idle replacements outcome. +def test_catalog_watcher_applies_only_valid_idle_replacements(tmp_path): + # What: arrange path as tmp path and models and toml; why: the catalog watcher applies only valid idle replacements test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text models a nmodel a gguf n encoding fixture fragment; why: the catalog watcher applies only valid idle replacements scenario feeds this byte-preserved fragment through path.write_text("[models.a]\nmodel = 'a.gguf'\n", encoding="utf-8") before asserting its protocol or parser. + path.write_text("[models.a]\nmodel = 'a.gguf'\n", encoding="utf-8") + # What: act by calling Manager and capture manager; why: the catalog watcher applies only valid idle replacements test asserts the response, state, or failure produced by this call. + manager = Manager() + # What: act by calling ModelCatalog.load and capture catalog doc; why: the catalog watcher applies only valid idle replacements test asserts the response, state, or failure produced by this call. + catalog_doc = ModelCatalog.load(str(path)) + # What: act by calling RoutingCoordinator and capture router; why: the catalog watcher applies only valid idle replacements test asserts the response, state, or failure produced by this call. + router = RoutingCoordinator(manager, catalog_doc, object(), ready_fn=ready) + + # What: define the wait_for test helper around client and result; why: the catalog watcher applies only valid idle replacements scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def wait_for(client, result): + # What: act by calling time.monotonic and capture deadline; why: the catalog watcher applies only valid idle replacements test asserts the response, state, or failure produced by this call. + deadline = time.monotonic() + 2 + # What: act across deadline and monotonic and time to perform result and get and json and client; why: the catalog watcher applies only valid idle replacements scenario repeats the body only while or for the loop header admits an iteration. + while time.monotonic() < deadline: + # What: act on result and get and json and client before the computed value; why: the catalog watcher applies only valid idle replacements scenario admits the computed value only for this predicate and excludes the opposite state. + if client.get("/router/status").json()["catalogWatch"].get("lastResult") == result: + # What: return the named fixture input from the wait_for test helper; why: the catalog watcher applies only valid idle replacements scenario uses this helper result in its subsequent act or assertion. + return + # What: act by calling time.sleep with 0 02; why: the catalog watcher applies only valid idle replacements scenario observes the time.sleep return value during raise assertion error f catalog watcher did. + time.sleep(0.02) + # What: raise AssertionError for the caller; why: wait_for stops this rejected path before it can mutate state, dispatch work, or report success. + raise AssertionError(f"catalog watcher did not report {result}") + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app; why: test_catalog_watcher_applies_only_valid_idle_replacements releases this resource or lock after app build app on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the catalog watcher applies only valid idle replacements test asserts the response, state, or failure produced by this call. + app = build_app( + # What: arrange the pid input for test_catalog_watcher_applies_only_valid_idle_replacements; why: test_catalog_watcher_applies_only_valid_idle_replacements consumes pid during signature binding, so callers must bind it with the other signature inputs. + manager=manager, ring=LogRing(), probe=object(), footprint_fn=lambda pid: {}, + # What: arrange lifecycle pool to build_app; why: the catalog watcher applies only valid idle replacements scenario binds this lifecycle value to build_app's lifecycle pool input. + lifecycle_pool=lifecycle, proxy_pool=proxy, catalog=catalog_doc, router=router, + # What: arrange catalog path to str; why: the catalog watcher applies only valid idle replacements scenario binds this str and path value to str's catalog path input. + catalog_path=str(path), catalog_watch_interval_s=0.01, + # What: arrange the build_app call with manager and ring and probe and footprint fn and lifecycle pool; why: test_catalog_watcher_applies_only_valid_idle_replacements groups the supplied clauses as one build_app call before its value is consumed. + ) + # What: enter the TestClient managed context before path write text; why: test_catalog_watcher_applies_only_valid_idle_replacements releases this resource or lock after path write text on both success and failure paths. + with TestClient(app) as client: + # What: act by calling path.write_text with router and performance disabled and true and models; why: the catalog watcher applies only valid idle replacements scenario observes the path.write_text return value during router nperformance disabled true n n models b. + path.write_text( + # What: arrange the exact router nperformance disabled true n n models b fixture fragment; why: the catalog watcher applies only valid idle replacements scenario feeds this byte-preserved fragment through "[router]\nperformance_disabled = true\n\n[models.b]\nmodel = 'b.gguf'\n before asserting its prot. + "[router]\nperformance_disabled = true\n\n[models.b]\nmodel = 'b.gguf'\n", + # What: arrange the exact encoding utf 8 fixture fragment; why: the catalog watcher applies only valid idle replacements scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the path.write_text call with encoding; why: test_catalog_watcher_applies_only_valid_idle_replacements groups the supplied clauses as one path.write_text call before its value is consumed. + ) + # What: arrange the exact wait for client reloaded fixture fragment; why: the catalog watcher applies only valid idle replacements scenario feeds this byte-preserved fragment through wait_for(client, "reloaded") before asserting its protocol or parser result. + wait_for(client, "reloaded") + # What: assert that model name for model in client get equals b; why: this assertion protects the catalog watcher applies only valid idle replacements regression after the test's arranged inputs and exercised call. + assert [model["name"] for model in client.get("/router/models").json()["data"]] == ["b"] + # What: assert that app state performance monitor current enabled is false; why: this assertion protects the catalog watcher applies only valid idle replacements regression after the test's arranged inputs and exercised call. + assert app.state.performance_monitor.current()["enabled"] is False + # What: arrange the exact path write text models b nmodel n encoding utf 8 fixture fragment; why: the catalog watcher applies only valid idle replacements scenario feeds this byte-preserved fragment through path.write_text("[models.b]\nmodel = [\n", encoding="utf-8") before asserting its protocol or parser. + path.write_text("[models.b]\nmodel = [\n", encoding="utf-8") + # What: arrange the exact wait for client invalid catalog fixture fragment; why: the catalog watcher applies only valid idle replacements scenario feeds this byte-preserved fragment through wait_for(client, "invalid_catalog") before asserting its protocol or parser result. + wait_for(client, "invalid_catalog") + # What: assert that model name for model in client get equals b; why: this assertion protects the catalog watcher applies only valid idle replacements regression after the test's arranged inputs and exercised call. + assert [model["name"] for model in client.get("/router/models").json()["data"]] == ["b"] + # What: assert that app state catalog watch stop is set; why: this assertion protects the catalog watcher applies only valid idle replacements regression after the test's arranged inputs and exercised call. + assert app.state.catalog_watch_stop.is_set() +# What: verify that explicit upstream closure ends a blocked chunk iterator without leaking the transport's internal cancellation exception; why: production router cancellation closes Python HTTP responses from another thread. +def test_upstream_response_chunks_suppresses_only_close_induced_read_error(): + # What: define a raw transport that blocks until close and then raises the observed Python HTTP failure; why: the regression must reproduce the cancellation race deterministically. + class ClosingRaw: + # What: initialize synchronization signals; why: the test must close only after the iterator is actively blocked inside read. + def __init__(self): + # What: signal entry into read; why: the controlling thread needs a deterministic race boundary. + self.read_started = threading.Event() + # What: signal explicit transport closure; why: the blocked reader should wake only after cancellation. + self.closed = threading.Event() + + # What: block a raw read until cancellation and then raise the real low-level symptom; why: concurrent HTTPResponse.close invalidates an internal file pointer in this way. + def read(self, size): + # What: announce that iteration reached the transport; why: close must not race ahead of the read call under test. + self.read_started.set() + # What: wait for explicit closure; why: the failure must be causally tied to cancellation. + self.closed.wait(1) + # What: raise the close-induced internal-state error; why: UpstreamResponse should convert it to clean end-of-stream only after explicit close. + raise AttributeError("closed response has no file pointer") + + # What: release the blocked reader; why: this models the router cancellation path closing the upstream response. + def close(self): + # What: publish transport closure; why: the reader can now produce the expected cancellation symptom. + self.closed.set() + + # What: create the deterministic raw transport and wrapped response; why: the production iterator and close methods must be exercised together. + raw = ClosingRaw() + # What: wrap the raw transport in the production response object; why: cancellation behavior belongs to this ownership boundary. + response = UpstreamResponse(200, {"Content-Type": "text/event-stream"}, raw) + # What: collect unexpected iterator failures; why: the test thread must report any exception back to the assertion thread. + failures = [] + + # What: consume the response iterator and capture unexpected errors; why: generator execution occurs in a separate thread during the cancellation race. + def consume(): + # What: guard iterator consumption for observable test failure capture; why: thread exceptions alone may not fail the test reliably. + try: + # What: exhaust the production chunk iterator; why: a close-induced read error should appear as a normal empty completion. + list(response.chunks()) + # What: capture any propagated exception; why: explicit cancellation must not escape as an ASGI-visible failure. + except Exception as exc: # noqa: BLE001 -- the assertion must retain any stream failure. + # What: retain the unexpected exception; why: the main test thread can assert exact clean termination. + failures.append(exc) + + # What: start iterator consumption concurrently; why: the defect requires close to interrupt an active raw read. + worker = threading.Thread(target=consume) + # What: begin the blocked read; why: cancellation cannot be tested before the iterator runs. + worker.start() + # What: require the raw read to start; why: this prevents a false pass caused by closing before iteration. + assert raw.read_started.wait(1) + # What: explicitly close the wrapped response; why: this is the production cancellation action that should suppress only its own read error. + response.close() + # What: wait for iterator termination; why: the assertion must observe the completed cancellation path. + worker.join(2) + # What: require the iterator thread to exit; why: cancellation must not leave a blocked streaming worker. + assert not worker.is_alive() + # What: require clean termination without propagated error; why: expected client cancellation must not emit an internal server traceback. + assert failures == [] + + +# What: verify dynamic allocation returns a base port whose adjacent distributed-store port is also available; why: FreeToken serve binds both ports during every model activation. +def test_allocate_loopback_port_validates_adjacent_distributed_store_port(): + # What: request a production dynamic port pair; why: the allocator must satisfy the real local socket contract. + port = allocate_loopback_port() + # What: require room for the companion listener; why: a terminal TCP port cannot support FreeToken's distributed store. + assert 1 <= port < 65535 + # What: open temporary verification listeners for both returned ports; why: immediate exclusive binds demonstrate that neither port was occupied at selection time. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as service_sock, socket.socket( + # What: create the adjacent verification socket with matching IPv4 stream semantics; why: the check must mirror the child's listener family and type. + socket.AF_INET, socket.SOCK_STREAM + # What: complete the paired verification context; why: both sockets are released after the assertion path. + ) as distributed_sock: + # What: bind the returned service port; why: the base listener must be available for the HTTP server. + service_sock.bind(("127.0.0.1", port)) + # What: bind the adjacent companion port; why: the distributed process-group store must not encounter EADDRINUSE. + distributed_sock.bind(("127.0.0.1", port + 1)) + +def test_finish_detach_honors_drain_timeout_and_rolls_back_shutdown() -> None: + # Define a detach-capable fake because finish_detach resolves the manager method before draining. + class DetachingManager(Manager): + # Record detach without touching a real child so the regression stays CPU-only. + def detach(self) -> None: + # Preserve the active fake model in the call log so an unexpected detach is observable. + self.calls.append(("detach", self.model)) + + # Create the fake manager that records lifecycle calls without launching a model. + manager = DetachingManager() + # Build the coordinator with existing CPU-only catalog and readiness fixtures. + router = RoutingCoordinator(manager, catalog(), object(), ready_fn=ready) + # Acquire one lease so the shutdown drain has an intentionally blocked owner. + lease = router.acquire("low") + # Reserve shutdown exactly as the public lifecycle endpoint does before finishing. + owner = router.begin_shutdown() + # Require the bounded non-forced drain to fail rather than waiting forever. + with pytest.raises(TimeoutError, match="ownership to drain"): + # Use a short deterministic bound because no other thread releases the lease here. + router.finish_detach(owner, drain_timeout=0.01) + # Confirm timeout rollback reopened routing instead of leaving a shutdown latch. + assert router.status()["shuttingDown"] is False + # Confirm no detach occurred because the active lease never drained. + assert ("detach", "low.gguf") not in manager.calls + # Release the arranged lease so the test leaves no ownership behind. + lease.release() diff --git a/tests/daemon/test_swap_qualification.py b/tests/daemon/test_swap_qualification.py new file mode 100644 index 0000000000..6f9a5aa1d5 --- /dev/null +++ b/tests/daemon/test_swap_qualification.py @@ -0,0 +1,2055 @@ +"""CPU tests of cancellation evidence gates, not real-model qualification.""" +# What: document cpu tests of cancellation evidence gates in the test_swap_qualification docstring; why: introspection and maintainers read this exact docstring fragment to understand test swap qualification behavior without executing it. + +# What: import importlib util for qualifier using importlib and util; why: qualifier uses importlib util spec from file location, making that imported dependency available to its named operation. +# What: import base64 for do get using base64; why: do_GET uses base64 b64encode, making that imported dependency available to its named operation. +import base64 +import importlib.util + +# What: import io for test cancellation requires terminal abort without restart using io; why: test_cancellation_requires_terminal_abort_without_restart uses io bytes io, making that imported dependency available to its named operation. +import io + +# What: import json for test native periodic performance gate rejects unavailable or identifying rows using json; why: test_native_periodic_performance_gate_rejects_unavailable_or_identifying_rows uses json loads, making that imported dependency available to its named operation. +import json + +# What: import socket for test native router benchmark requires final engine listener to close using socket; why: test_native_router_benchmark_requires_final_engine_listener_to_close uses socket socket, making that imported dependency available to its named operation. +import socket + +# What: import threading for test cancellation closes real local http stream using threading; why: test_cancellation_closes_real_local_http_stream uses threading event, making that imported dependency available to its named operation. +import threading + +# What: import time for fake canary using time; why: fake_canary uses time sleep, making that imported dependency available to its named operation. +import time + +# What: arrange from http server import BaseHTTPRequestHandler ThreadingHTTPServer for the scenario; why: test swap qualification requires this concrete input or helper state before exercising the behavior under test. +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +# What: import path for qualifier using pathlib and path; why: qualifier uses path, making that imported dependency available to its named operation. +from pathlib import Path + +# What: import pytest for module initialization using pytest; why: module initialization uses pytest fixture, making that imported dependency available to its named operation. +import pytest + +# What: arrange from freetoken daemon catalog import ModelCatalog for the scenario; why: test swap qualification requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.catalog import ModelCatalog + + +# What: apply pytest.fixture behavior to qualifier; why: Python attaches this named decorator's registration or descriptor semantics to qualifier. +@pytest.fixture +# What: define the qualifier test helper around captured fixture state; why: the qualifier scenario calls this helper to produce or observe the exact behavior checked by its assertions. +def qualifier(): + # What: act by calling Path and capture path; why: the swap qualification test asserts the response, state, or failure produced by this call. + path = Path(__file__).parents[2] / "benchmarks/swap/qualify.py" + # What: act by calling importlib.util.spec_from_file_location and capture spec; why: the swap qualification test asserts the response, state, or failure produced by this call. + spec = importlib.util.spec_from_file_location("swap_qualifier", path) + # What: act by calling importlib.util.module_from_spec and capture module; why: the swap qualification test asserts the response, state, or failure produced by this call. + module = importlib.util.module_from_spec(spec) + # What: act by calling spec.loader.exec_module with module; why: the qualifier scenario observes the spec.loader.exec_module return value during return module. + spec.loader.exec_module(module) + # What: return module from the qualifier test helper; why: the qualifier scenario uses this helper result in its subsequent act or assertion. + return module + + +# What: apply pytest.fixture behavior to native_router_qualifier; why: Python attaches this named decorator's registration or descriptor semantics to native_router_qualifier. +@pytest.fixture +# What: define the native_router_qualifier test helper around captured fixture state; why: the native router qualifier scenario calls this helper to produce or observe the exact behavior checked by its assertions. +def native_router_qualifier(): + # What: act by calling Path and capture path; why: the swap qualification test asserts the response, state, or failure produced by this call. + path = Path(__file__).parents[2] / "benchmarks/swap/qualify_native_router.py" + # What: act by calling importlib.util.spec_from_file_location and capture spec; why: the swap qualification test asserts the response, state, or failure produced by this call. + spec = importlib.util.spec_from_file_location("native_router_qualifier", path) + # What: act by calling importlib.util.module_from_spec and capture module; why: the swap qualification test asserts the response, state, or failure produced by this call. + module = importlib.util.module_from_spec(spec) + # What: act by calling spec.loader.exec_module with module; why: the native router qualifier scenario observes the spec.loader.exec_module return value during return module. + spec.loader.exec_module(module) + # What: return module from the native_router_qualifier test helper; why: the native router qualifier scenario uses this helper result in its subsequent act or assertion. + return module + + +# What: define the stats test helper around active and instance and completed; why: the stats scenario calls this helper to produce or observe the exact behavior checked by its assertions. +def stats(active, *, instance="same", completed=3): + # What: arrange the instance id field as instance; why: stats carries instance id into return {"instance_id": instance, "requests": {"active": active, "complet. + return {"instance_id": instance, "requests": {"active": active, "completed": completed}} + + +# What: parameterize test_maintenance_qualifiers_require_exact_hostname_without_disclosure with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test maintenance qualifiers require exact hostname without disclosure. +@pytest.mark.parametrize("expected", ["", "other-host", "approved-host\x00suffix"]) +# What: define the test_maintenance_qualifiers_require_exact_hostname_without_disclosure test around qualifier and native router qualifier and expected; why: this test groups the arrange, act, and assertions that protect the maintenance qualifiers require exact hostname without disclosure outcome. +def test_maintenance_qualifiers_require_exact_hostname_without_disclosure( + # What: arrange the qualifier input for test_maintenance_qualifiers_require_exact_hostname_without_disclosure; why: test_maintenance_qualifiers_require_exact_hostname_without_disclosure consumes qualifier during for module in qualifier native router qualifier, so callers must bind it with the other signature input. + qualifier, native_router_qualifier, expected +# What: arrange the grouped source fragment for the scenario; why: test maintenance qualifiers require exact hostname without disclosure requires this concrete input or helper state before exercising the behavior under test. +): + # What: act across qualifier and native router qualifier to perform exc and raises and runtime error and require expected hostname and expected; why: the maintenance qualifiers require exact hostname without disclosure scenario repeats the body only while or for the loop header admits an iteration. + for module in (qualifier, native_router_qualifier): + # What: assert the pytest.raises failure context; why: the maintenance qualifiers require exact hostname without disclosure scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError, match="operator-supplied expected hostname") as exc: + # What: arrange the exact module require expected hostname expected actual approved host fixture fragment; why: the maintenance qualifiers require exact hostname without disclosure scenario feeds this byte-preserved fragment through module.require_expected_hostname(expected, actual="approved-host") before. + module.require_expected_hostname(expected, actual="approved-host") + # What: assert that approved host is absent from str exc value; why: this assertion protects the maintenance qualifiers require exact hostname without disclosure regression after the test's arranged inputs and exercised call. + assert "approved-host" not in str(exc.value) + # What: assert that expected not in str exc value or expected equals; why: this assertion protects the maintenance qualifiers require exact hostname without disclosure regression after the test's arranged inputs and exercised call. + assert expected not in str(exc.value) or expected == "" + + # What: assert the expected qualifier require expected hostname outcome; why: test swap qualification test maintenance qualifiers require exact hostname without disclosure protects its regression by requiring this observable result after the exercised behavior. + assert qualifier.require_expected_hostname( + # What: arrange approved host actual approved host for the scenario; why: test swap qualification test maintenance qualifiers require exact hostname without disclosure requires this concrete input or helper state before exercising the behavior under test. + "approved-host", actual="approved-host" + # What: arrange == approved host for the scenario; why: test swap qualification test maintenance qualifiers require exact hostname without disclosure requires this concrete input or helper state before exercising the behavior under test. + ) == "approved-host" + # What: assert the expected native router qualifier require expected hostname outcome; why: test swap qualification test maintenance qualifiers require exact hostname without disclosure protects its regression by requiring this observable result after the exercised behavior. + assert native_router_qualifier.require_expected_hostname( + # What: arrange approved host actual approved host for the scenario; why: test swap qualification test maintenance qualifiers require exact hostname without disclosure requires this concrete input or helper state before exercising the behavior under test. + "approved-host", actual="approved-host" + # What: arrange == approved host for the scenario; why: test swap qualification test maintenance qualifiers require exact hostname without disclosure requires this concrete input or helper state before exercising the behavior under test. + ) == "approved-host" + + +# What: define the test_maintenance_entrypoints_check_hostname_before_side_effects test around qualifier and native router qualifier and monkeypatch and tmp path; why: this test groups the arrange, act, and assertions that protect the maintenance entrypoints check hostname before side effects outcome. +def test_maintenance_entrypoints_check_hostname_before_side_effects( + # What: arrange the qualifier input for test_maintenance_entrypoints_check_hostname_before_side_effects; why: test_maintenance_entrypoints_check_hostname_before_side_effects consumes qualifier during qualifier, so callers must bind it with the other signature inputs. + qualifier, native_router_qualifier, monkeypatch, tmp_path +# What: arrange the grouped source fragment for the scenario; why: test maintenance entrypoints check hostname before side effects requires this concrete input. +): + # What: arrange cases as qualifier and native router qualifier and source and source and python; why: the maintenance entrypoints check hostname before side effects test consumes this named precondition before exercising the behavior. + cases = ( + # What: arrange the cases collection with qualifier and source and source and python and python and; why: test_maintenance_entrypoints_check_hostname_before_side_effects groups the supplied clauses as one cases collection before its value. + ( + # What: arrange the qualifier portion of cases; why: the maintenance entrypoints check hostname before side effects scenario uses this clause to evaluate cases as one grouped value. + qualifier, + # What: arrange the cases collection with source and source and python and python; why: test_maintenance_entrypoints_check_hostname_before_side_effects groups the supplied clauses as one cases collection before its value. + [ + # What: arrange the source source python python llama swap llama swap portion of cases; why: the maintenance entrypoints check hostname before side effects scenario uses this clause to evaluate cases as one grouped value. + "--source", "source", "--python", "python", "--llama-swap", "llama-swap", + # What: arrange the model a a model b b portion of cases; why: the maintenance entrypoints check hostname before side effects scenario uses this clause to evaluate cases as one grouped value. + "--model-a", "a", "--model-b", "b", + # What: arrange the cases collection with source and source and python and python; why: test_maintenance_entrypoints_check_hostname_before_side_effects groups the supplied clauses as one cases collection before its value. + ], + # What: arrange the cases collection with qualifier and source and source and python and python and; why: test_maintenance_entrypoints_check_hostname_before_side_effects groups the supplied clauses as one cases collection before its value. + ), + # What: arrange the cases collection with native router qualifier and source and source and python and; why: test_maintenance_entrypoints_check_hostname_before_side_effects groups the supplied clauses as one cases collection before its value. + ( + # What: arrange the native router qualifier portion of cases; why: the maintenance entrypoints check hostname before side effects scenario uses this clause to evaluate cases as one grouped value. + native_router_qualifier, + # What: arrange the cases collection with source and source and python and python; why: test_maintenance_entrypoints_check_hostname_before_side_effects groups the supplied clauses as one cases collection before its value. + [ + # What: arrange the source source python python portion of cases; why: the maintenance entrypoints check hostname before side effects scenario uses this clause to evaluate cases as one grouped value. + "--source", "source", "--python", "python", + # What: arrange the model a a model b b portion of cases; why: the maintenance entrypoints check hostname before side effects scenario uses this clause to evaluate cases as one grouped value. + "--model-a", "a", "--model-b", "b", + # What: arrange the cases collection with source and source and python and python; why: test_maintenance_entrypoints_check_hostname_before_side_effects groups the supplied clauses as one cases collection before its value. + ], + # What: arrange the cases collection with native router qualifier and source and source and python and; why: test_maintenance_entrypoints_check_hostname_before_side_effects groups the supplied clauses as one cases collection before its value. + ), + # What: arrange the grouped source fragment for the scenario; why: test maintenance entrypoints check hostname before side effects requires this concrete input. + ) + # What: arrange the exact monkeypatch setattr native router qualifier sys platform linux fixture fragment; why: the maintenance entrypoints check hostname before side effects scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier.sys, "platform", "linux") before asserting i. + monkeypatch.setattr(native_router_qualifier.sys, "platform", "linux") + # What: act across enumerate and cases to perform artifacts and tmp path and index; why: the maintenance entrypoints check hostname before side effects scenario repeats the body only while or for the loop header admits an iteration. + for index, (module, specific) in enumerate(cases): + # What: arrange artifacts as tmp path and index and must not exist; why: the maintenance entrypoints check hostname before side effects test consumes this named precondition before exercising the behavior. + artifacts = tmp_path / f"must-not-exist-{index}" + # What: act by calling str and capture argv; why: the maintenance entrypoints check hostname before side effects test asserts the response, state, or failure produced by this call. + argv = [ + # What: act by calling str with artifacts; why: the maintenance entrypoints check hostname before side effects scenario observes the str return value during protected service protected protected url http protected. + "qualifier", *specific, "--artifacts", str(artifacts), + # What: arrange the protected service protected protected url http protected portion of argv; why: the maintenance entrypoints check hostname before side effects scenario uses this clause to evaluate argv as one grouped value. + "--protected-service", "protected", "--protected-url", "http://protected", + # What: arrange the expected hostname expected host allow maintenance portion of argv; why: the maintenance entrypoints check hostname before side effects scenario uses this clause to evaluate argv as one grouped value. + "--expected-hostname", "expected-host", "--allow-maintenance", + # What: arrange the argv collection with qualifier and specific and artifacts and str and artifacts; why: test_maintenance_entrypoints_check_hostname_before_side_effects groups the supplied clauses as one argv collection before its value is consumed. + ] + # What: arrange the exact monkeypatch setattr module sys argv argv fixture fragment; why: the maintenance entrypoints check hostname before side effects scenario feeds this byte-preserved fragment through monkeypatch.setattr(module.sys, "argv", argv) before asserting its protocol or parser result. + monkeypatch.setattr(module.sys, "argv", argv) + # What: arrange the exact monkeypatch setattr module socket gethostname lambda different host fixture fragment; why: the maintenance entrypoints check hostname before side effects scenario feeds this byte-preserved fragment through monkeypatch.setattr(module.socket, "gethostname", lambda: "different-hos before. + monkeypatch.setattr(module.socket, "gethostname", lambda: "different-host") + # What: assert the pytest.raises failure context; why: the maintenance entrypoints check hostname before side effects scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError, match="operator-supplied expected hostname"): + # What: act by calling module.main with the declared inputs; why: the maintenance entrypoints check hostname before side effects scenario observes the module.main return value during assert not artifacts exists. + module.main() + # What: assert that artifacts exists is false; why: this assertion protects the maintenance entrypoints check hostname before side effects regression after the test's arranged inputs and exercised call. + assert not artifacts.exists() + + +# What: parameterize test_cancellation_requires_terminal_abort_without_restart with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test cancellation requires terminal abort without restart. +@pytest.mark.parametrize("outcome", ["abort", "restart", "completion", "already-done", "timeout"]) +# What: define the test_cancellation_requires_terminal_abort_without_restart test around qualifier and monkeypatch and outcome; why: this test groups the arrange, act, and assertions that protect the cancellation requires terminal abort without restart outcome. +def test_cancellation_requires_terminal_abort_without_restart(qualifier, monkeypatch, outcome): + # What: act by calling io.BytesIO and capture stream; why: the cancellation requires terminal abort without restart test asserts the response, state, or failure produced by this call. + stream = io.BytesIO(b'data: {"choices":[{"delta":{"content":"1"}}]}\n\n') + # What: arrange the exact monkeypatch setattr qualifier urllib request urlopen lambda a k fixture fragment; why: the cancellation requires terminal abort without restart scenario feeds this byte-preserved fragment through monkeypatch.setattr(qualifier.urllib.request, "urlopen", lambda *a, **k: before asserting its. + monkeypatch.setattr(qualifier.urllib.request, "urlopen", lambda *a, **k: stream) + # What: act by calling stats and capture during; why: the cancellation requires terminal abort without restart test asserts the response, state, or failure produced by this call. + during = stats(0 if outcome == "already-done" else 1) + # What: act by calling stats and capture after; why: the cancellation requires terminal abort without restart test asserts the response, state, or failure produced by this call. + after = stats(1 if outcome == "timeout" else 0, + # What: arrange instance to stats; why: the cancellation requires terminal abort without restart scenario binds this outcome and new and same and restart value to stats's instance input. + instance="new" if outcome == "restart" else "same", + # What: arrange completed to stats; why: the cancellation requires terminal abort without restart scenario binds this outcome and 4 and 3 and completion value to stats's completed input. + completed=4 if outcome == "completion" else 3) + # What: act by calling iter and capture snapshots; why: the cancellation requires terminal abort without restart test asserts the response, state, or failure produced by this call. + snapshots = iter([stats(0), during, after]) + # What: arrange the exact monkeypatch setattr qualifier http lambda a k fixture fragment; why: the cancellation requires terminal abort without restart scenario feeds this byte-preserved fragment through monkeypatch.setattr(qualifier, "http", lambda *a, **k: json.dumps(next(s before asserting its protocol or parse. + monkeypatch.setattr(qualifier, "http", lambda *a, **k: json.dumps(next(snapshots)).encode()) + # What: act on outcome before prefix and evidence and cancellation canary and qualifier; why: the cancellation requires terminal abort without restart scenario admits prefix and evidence and cancellation canary and qualifier only for this predicate and excludes the opposite state. + if outcome == "abort": + # What: act by calling qualifier.cancellation_canary and capture prefix and evidence; why: the cancellation requires terminal abort without restart test asserts the response, state, or failure produced by this call. + prefix, evidence = qualifier.cancellation_canary("http://test", "model-a", seconds=0) + # What: assert that evidence passed; why: this assertion protects the cancellation requires terminal abort without restart regression after the test's arranged inputs and exercised call. + assert evidence["passed"] + # What: assert that b content 1 is present in prefix; why: this assertion protects the cancellation requires terminal abort without restart regression after the test's arranged inputs and exercised call. + assert b'"content":"1"' in prefix + # What: act on outcome before raises and timeout error and cancellation canary and pytest and qualifier; why: the cancellation requires terminal abort without restart scenario admits raises and timeout error and cancellation canary and pytest and qualifier only for this predicate and excludes the opposite state. + elif outcome == "timeout": + # What: arrange with pytest raises TimeoutError match terminal abort for the scenario; why: test raises timeout error match terminal abort requires this concrete input or helper state before exercising the behavior under test. + with pytest.raises(TimeoutError, match="terminal abort"): + # What: arrange the exact qualifier cancellation canary http test model a seconds fixture fragment; why: the cancellation requires terminal abort without restart scenario feeds this byte-preserved fragment through qualifier.cancellation_canary("http://test", "model-a", seconds=0) before asserting its proto. + qualifier.cancellation_canary("http://test", "model-a", seconds=0) + # What: select the remaining branch that performs with pytest raises runtime error; why: test_cancellation_requires_terminal_abort_without_restart covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: arrange with pytest raises RuntimeError for the scenario; why: test raises assertion error requires this concrete input or helper state before exercising the behavior under test. + with pytest.raises(RuntimeError): + # What: arrange the exact qualifier cancellation canary http test model a seconds fixture fragment; why: the cancellation requires terminal abort without restart scenario feeds this byte-preserved fragment through qualifier.cancellation_canary("http://test", "model-a", seconds=0) before asserting its proto. + qualifier.cancellation_canary("http://test", "model-a", seconds=0) + # What: assert that stream closed; why: this assertion protects the cancellation requires terminal abort without restart regression after the test's arranged inputs and exercised call. + assert stream.closed + + +# What: parameterize test_completed_or_empty_stream_is_not_cancellation with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test completed or empty stream is not cancellation. +@pytest.mark.parametrize("body", [b"data: [DONE]\n\n", b"", b": heartbeat\n\n"]) +# What: define the test_completed_or_empty_stream_is_not_cancellation test around qualifier and monkeypatch and body; why: this test groups the arrange, act, and assertions that protect the completed or empty stream is not cancellation outcome. +def test_completed_or_empty_stream_is_not_cancellation(qualifier, monkeypatch, body): + # What: act by calling io.BytesIO and capture stream; why: the completed or empty stream is not cancellation test asserts the response, state, or failure produced by this call. + stream = io.BytesIO(body) + # What: arrange the exact monkeypatch setattr qualifier urllib request urlopen lambda a k fixture fragment; why: the completed or empty stream is not cancellation scenario feeds this byte-preserved fragment through monkeypatch.setattr(qualifier.urllib.request, "urlopen", lambda *a, **k: before asserting its protoc. + monkeypatch.setattr(qualifier.urllib.request, "urlopen", lambda *a, **k: stream) + # What: arrange the exact monkeypatch setattr qualifier http lambda a k fixture fragment; why: the completed or empty stream is not cancellation scenario feeds this byte-preserved fragment through monkeypatch.setattr(qualifier, "http", lambda *a, **k: json.dumps(stats( before asserting its protocol or parser resul. + monkeypatch.setattr(qualifier, "http", lambda *a, **k: json.dumps(stats(0)).encode()) + # What: assert the pytest.raises failure context; why: the completed or empty stream is not cancellation scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError): + # What: arrange the exact qualifier cancellation canary http test model a fixture fragment; why: the completed or empty stream is not cancellation scenario feeds this byte-preserved fragment through qualifier.cancellation_canary("http://test", "model-a") before asserting its protocol or parser result. + qualifier.cancellation_canary("http://test", "model-a") + # What: assert that stream closed; why: this assertion protects the completed or empty stream is not cancellation regression after the test's arranged inputs and exercised call. + assert stream.closed + + +# What: define the test_cancellation_closes_real_local_http_stream test around qualifier; why: this test groups the arrange, act, and assertions that protect the cancellation closes real local http stream outcome. +def test_cancellation_closes_real_local_http_stream(qualifier): + """Exercise the HTTP transport too, using a CPU-only streaming backend.""" + # What: document exercise the http transport too using in the test_cancellation_closes_real_local_http_stream docstring; why: introspection and maintainers read this exact docstring fragment to understand test cancellation closes real local http stream behavior without executing it. + # What: arrange state as active and 0; why: the cancellation closes real local http stream test consumes this named precondition before exercising the behavior. + state = {"active": 0} + # What: act by calling threading.Event and capture disconnected; why: the cancellation closes real local http stream test asserts the response, state, or failure produced by this call. + disconnected = threading.Event() + + # What: define Handler as the owner of log_message and do_GET and do_POST; why: daemon callers use this class boundary so those methods share one handler state invariant. + class Handler(BaseHTTPRequestHandler): + # What: define the log_message test helper around captured fixture state; why: the cancellation closes real local http stream scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def log_message(self, *args): + # What: ignore the anticipated exception handled by this branch; why: log_message continues its retry or cleanup path instead of re-raising that transient failure. + pass + + # What: define the do_GET test helper around captured fixture state; why: the cancellation closes real local http stream scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def do_GET(self): + # What: act by calling operation.encode and capture body; why: the cancellation closes real local http stream test asserts the response, state, or failure produced by this call. + body = json.dumps(stats(state["active"])).encode() + # What: act by calling self.send_response with 200; why: the cancellation closes real local http stream scenario observes the self.send_response return value during self send header content length str len body. + self.send_response(200) + # What: arrange the exact self send header content length str len body fixture fragment; why: the cancellation closes real local http stream scenario feeds this byte-preserved fragment through self.send_header("Content-Length", str(len(body))) before asserting its protocol or parser result. + self.send_header("Content-Length", str(len(body))) + # What: act by calling self.end_headers with the declared inputs; why: the cancellation closes real local http stream scenario observes the self.end_headers return value during self wfile write body. + self.end_headers() + # What: act by calling self.wfile.write with body; why: the cancellation closes real local http stream scenario observes the self.wfile.write return value during the enclosing return. + self.wfile.write(body) + + # What: define the do_POST test helper around captured fixture state; why: the cancellation closes real local http stream scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def do_POST(self): + # What: arrange the exact self rfile read int self headers content length fixture fragment; why: the cancellation closes real local http stream scenario feeds this byte-preserved fragment through self.rfile.read(int(self.headers["Content-Length"])) before asserting its protocol or parser result. + self.rfile.read(int(self.headers["Content-Length"])) + # What: arrange state entry as 1; why: the cancellation closes real local http stream test consumes this named precondition before exercising the behavior. + state["active"] = 1 + # What: act by calling self.send_response with 200; why: the cancellation closes real local http stream scenario observes the self.send_response return value during self send header content type text event stream. + self.send_response(200) + # What: arrange the exact self send header content type text event stream fixture fragment; why: the cancellation closes real local http stream scenario feeds this byte-preserved fragment through self.send_header("Content-Type", "text/event-stream") before asserting its protocol or parser result. + self.send_header("Content-Type", "text/event-stream") + # What: act by calling self.end_headers with the declared inputs; why: the cancellation closes real local http stream scenario observes the self.end_headers return value during try. + self.end_headers() + # What: establish the handler boundary for the protected operation; why: Handler.do_POST routes failures to broken pipe error and connection reset error and connection aborted error while preserving cleanup and success flow. + try: + # What: act by calling time.monotonic and capture deadline; why: the cancellation closes real local http stream test asserts the response, state, or failure produced by this call. + deadline = time.monotonic() + 5 + # What: act across deadline and monotonic and time to perform write and wfile; why: the cancellation closes real local http stream scenario repeats the body only while or for the loop header admits an iteration. + while time.monotonic() < deadline: + # What: act by calling self.wfile.write with the named fixture input; why: the cancellation closes real local http stream scenario observes the self.wfile.write return value during self wfile flush. + self.wfile.write(b'data: {"choices":[{"delta":{"content":"1"}}]}\n\n') + # What: act by calling self.wfile.flush with the declared inputs; why: the cancellation closes real local http stream scenario observes the self.wfile.flush return value during time sleep. + self.wfile.flush() + # What: act by calling time.sleep with 0 01; why: the cancellation closes real local http stream scenario observes the time.sleep return value during except broken pipe error connection reset error connection aborted error. + time.sleep(0.01) + # What: handle broken pipe error and connection reset error and connection aborted error by disconnected set; why: Handler.do_POST converts that failure into this concrete recovery, response, or cleanup behavior. + except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError): + # What: act by calling disconnected.set with the declared inputs; why: the cancellation closes real local http stream scenario observes the disconnected.set return value during state active. + disconnected.set() + # What: arrange state entry as 0; why: the cancellation closes real local http stream test consumes this named precondition before exercising the behavior. + state["active"] = 0 + + # What: act by calling ThreadingHTTPServer and capture server; why: the cancellation closes real local http stream test asserts the response, state, or failure produced by this call. + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + # What: act by calling threading.Thread and capture worker; why: the cancellation closes real local http stream test asserts the response, state, or failure produced by this call. + worker = threading.Thread(target=server.serve_forever, daemon=True) + # What: act by calling worker.start with the declared inputs; why: the cancellation closes real local http stream scenario observes the worker.start return value during try. + worker.start() + # What: establish the handler boundary for the protected operation; why: test_cancellation_closes_real_local_http_stream routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: act by calling qualifier.cancellation_canary and capture and evidence; why: the cancellation closes real local http stream test asserts the response, state, or failure produced by this call. + _, evidence = qualifier.cancellation_canary( + # What: arrange seconds to qualifier.cancellation_canary; why: the cancellation closes real local http stream scenario binds this 3 value to qualifier.cancellation_canary's seconds input. + f"http://127.0.0.1:{server.server_port}", "model-a", seconds=3 + # What: arrange the qualifier.cancellation_canary call with seconds; why: test_cancellation_closes_real_local_http_stream groups the supplied clauses as one qualifier.cancellation_canary call before its value is consumed. + ) + # What: assert that evidence passed and disconnected is set; why: this assertion protects the cancellation closes real local http stream regression after the test's arranged inputs and exercised call. + assert evidence["passed"] and disconnected.is_set() + # What: run server shutdown on every exit path; why: test_cancellation_closes_real_local_http_stream performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: act by calling server.shutdown with the declared inputs; why: the cancellation closes real local http stream scenario observes the server.shutdown return value during server server close. + server.shutdown() + # What: act by calling server.server_close with the declared inputs; why: the cancellation closes real local http stream scenario observes the server.server_close return value during worker join. + server.server_close() + # What: act by calling worker.join with 3; why: the cancellation closes real local http stream scenario observes the worker.join return value during the enclosing return. + worker.join(3) + + +# What: define the test_native_router_benchmark_canary_records_first_byte_and_preserves_sse test around native router qualifier and monkeypatch; why: this test groups the arrange, act, and assertions that protect the native router benchmark canary records first byte and preserves sse outcome. +def test_native_router_benchmark_canary_records_first_byte_and_preserves_sse(native_router_qualifier, monkeypatch): + # What: act by calling io.BytesIO and capture stream; why: the native router benchmark canary records first byte and preserves sse test asserts the response, state, or failure produced by this call. + stream = io.BytesIO( + # What: arrange the b data choices delta content n portion of stream; why: the native router benchmark canary records first byte and preserves sse scenario uses this clause to evaluate stream as one grouped value. + b'data: {"choices":[{"delta":{"content":"4"}}]}\n\n' + # What: arrange the b data choices usage completion tokens n portion of stream; why: the native router benchmark canary records first byte and preserves sse scenario uses this clause to evaluate stream as one grouped value. + b'data: {"choices":[],"usage":{"completion_tokens":1}}\n\n' + # What: arrange the b data done n n portion of stream; why: the native router benchmark canary records first byte and preserves sse scenario uses this clause to evaluate stream as one grouped value. + b"data: [DONE]\n\n" + # What: arrange the io.BytesIO call with ordered positional inputs; why: test_native_router_benchmark_canary_records_first_byte_and_preserves_sse groups the supplied clauses as one io.BytesIO call before its value is consumed. + ) + # What: arrange the exact monkeypatch setattr native router qualifier urllib request urlopen lambda a k fixture f; why: the native router benchmark canary records first byte and preserves sse scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen". + monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen", lambda *a, **k: stream) + # What: act by calling iter and capture clock; why: the native router benchmark canary records first byte and preserves sse test asserts the response, state, or failure produced by this call. + clock = iter([10.0, 10.25, 11.25]) + # What: arrange the exact monkeypatch setattr native router qualifier time monotonic lambda next clock fixture fr; why: the native router benchmark canary records first byte and preserves sse scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier.time, "monotonic", lambda. + monkeypatch.setattr(native_router_qualifier.time, "monotonic", lambda: next(clock)) + + # What: act by calling native_router_qualifier.canary and capture raw and observation; why: the native router benchmark canary records first byte and preserves sse test asserts the response, state, or failure produced by this call. + raw, observation = native_router_qualifier.canary("http://test", "model-a", direct=False) + + # What: assert that raw endswith b data done n n; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert raw.endswith(b"data: [DONE]\n\n") + # What: assert that observation route equals native router; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert observation["route"] == "native_router" + # What: assert that observation model equals model a; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert observation["model"] == "model-a" + # What: assert that observation response model is group delimiter; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert observation["responseModel"] is None + # What: assert that observation passed is true; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert observation["passed"] is True + # What: assert that observation first byte seconds is not group delimiter; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert observation["firstByteSeconds"] is not None + # What: assert that observation first token seconds equals observation first byte seconds; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert observation["firstTokenSeconds"] == observation["firstByteSeconds"] + # What: assert that observation duration seconds is at least observation first byte seconds; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert observation["durationSeconds"] >= observation["firstByteSeconds"] + # What: assert that observation decode seconds equals 1 0; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert observation["decodeSeconds"] == 1.0 + # What: assert that observation completion tokens equals 1; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert observation["completionTokens"] == 1 + # What: assert that observation completion tokens per second equals 1 0; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert observation["completionTokensPerSecond"] == 1.0 + # What: assert that observation response bytes equals len raw; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert observation["responseBytes"] == len(raw) + # What: assert that stream closed; why: this assertion protects the native router benchmark canary records first byte and preserves sse regression after the test's arranged inputs and exercised call. + assert stream.closed + + +# What: parameterize test_native_router_loading_feedback_gate with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test native router loading feedback gate. +@pytest.mark.parametrize("expected", [True, False]) +# What: define the test_native_router_loading_feedback_gate test around native router qualifier and expected; why: this test groups the arrange, act, and assertions that protect the native router loading feedback gate outcome. +def test_native_router_loading_feedback_gate(native_router_qualifier, expected): + # What: arrange frames as the fixture input; why: the native router loading feedback gate test consumes this named precondition before exercising the behavior. + frames = [ + # What: arrange the b data choices delta reasoning content freetoken swap portion of frames; why: the native router loading feedback gate scenario uses this clause to evaluate frames as one grouped value. + b'data: {"choices":[{"delta":{"reasoning_content":"freetoken-swap "}}]}', + # What: arrange the b data choices delta reasoning content loading portion of frames; why: the native router loading feedback gate scenario uses this clause to evaluate frames as one grouped value. + b'data: {"choices":[{"delta":{"reasoning_content":"loading model: model-b"}}]}', + # What: arrange the b data choices delta content portion of frames; why: the native router loading feedback gate scenario uses this clause to evaluate frames as one grouped value. + b'data: {"choices":[{"delta":{"content":"4"}}]}', + # What: arrange the b data done portion of frames; why: the native router loading feedback gate scenario uses this clause to evaluate frames as one grouped value. + b"data: [DONE]", + # What: arrange the frames collection with the named fixture input and the named fixture input and the named fixture input and the named fixture input; why: test_native_router_loading_feedback_gate groups the supplied clauses as one frames collection before its value is consumed. + ] + # What: act by calling operation.join and capture raw; why: the native router loading feedback gate test asserts the response, state, or failure produced by this call. + raw = b"\n\n".join(frames[2:] if not expected else frames) + b"\n\n" + # What: assert the expected native router qualifier validate loading feedback raw expected expected == outcome; why: test native router loading feedb protects its regression by requiring this observable result after the exercised behavior. + assert native_router_qualifier.validate_loading_feedback(raw, expected=expected) == { + # What: arrange expected expected observed expected passed True for the scenario; why: test swap qualification test native router loading feedback gate requires this concrete input or helper state before exercising the behavior under test. + "expected": expected, "observed": expected, "passed": True, + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router loading feedback gate requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert the pytest.raises failure context; why: the native router loading feedback gate scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError, match="loading feedback"): + # What: arrange expected to native_router_qualifier.validate_loading_feedback; why: the native router loading feedback gate scenario binds this expected value to native_router_qualifier.validate_loading_feedback's expected input. + native_router_qualifier.validate_loading_feedback(raw, expected=not expected) + + +# What: define the test_native_router_canary_separates_loading_first_byte_from_first_token test around native router qualifier and monkeypatch; why: this test groups the arrange, act, and assertions that protect the native router canary separates loading first byte from first token outcome. +def test_native_router_canary_separates_loading_first_byte_from_first_token( + # What: arrange native router qualifier monkeypatch for the scenario; why: test native router canary separates loading first byte from first token requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch +# What: arrange the grouped source fragment for the scenario; why: test native router canary separates loading first byte from first token requires this concrete input or helper state before exercising the behavior under test. +): + # What: act by calling io.BytesIO and capture stream; why: the native router canary separates loading first byte from first token test asserts the response, state, or failure produced by this call. + stream = io.BytesIO( + # What: arrange the b data choices delta reasoning content freetoken swap portion of stream; why: the native router canary separates loading first byte from first token scenario uses this clause to evaluate stream as one grouped value. + b'data: {"choices":[{"delta":{"reasoning_content":"freetoken-swap loading model: a"}}]}\n\n' + # What: arrange the b data choices delta content n portion of stream; why: the native router canary separates loading first byte from first token scenario uses this clause to evaluate stream as one grouped value. + b'data: {"choices":[{"delta":{"content":"4"}}]}\n\n' + # What: arrange the b data choices usage completion tokens n portion of stream; why: the native router canary separates loading first byte from first token scenario uses this clause to evaluate stream as one grouped value. + b'data: {"choices":[],"usage":{"completion_tokens":1}}\n\n' + # What: arrange the b data done n n portion of stream; why: the native router canary separates loading first byte from first token scenario uses this clause to evaluate stream as one grouped value. + b"data: [DONE]\n\n" + # What: arrange the io.BytesIO call with ordered positional inputs; why: test_native_router_canary_separates_loading_first_byte_from_first_token groups the supplied clauses as one io.BytesIO call before its value is consumed. + ) + # What: arrange the exact monkeypatch setattr native router qualifier urllib request urlopen lambda a k fixture f; why: the native router canary separates loading first byte from first token scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen", l. + monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen", lambda *a, **k: stream) + # What: act by calling iter and capture clock; why: the native router canary separates loading first byte from first token test asserts the response, state, or failure produced by this call. + clock = iter([10.0, 10.1, 15.0, 16.0]) + # What: arrange the exact monkeypatch setattr native router qualifier time monotonic lambda next clock fixture fr; why: the native router canary separates loading first byte from first token scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier.time, "monotonic", lambda: n. + monkeypatch.setattr(native_router_qualifier.time, "monotonic", lambda: next(clock)) + + # What: act by calling native_router_qualifier.canary and capture and observation; why: the native router canary separates loading first byte from first token test asserts the response, state, or failure produced by this call. + _, observation = native_router_qualifier.canary("http://test", "model-a", direct=False) + + # What: assert that observation first byte seconds equals pytest approx 0 1; why: this assertion protects the native router canary separates loading first byte from first token regression after the test's arranged inputs and exercised call. + assert observation["firstByteSeconds"] == pytest.approx(0.1) + # What: assert that observation first token seconds equals 5 0; why: this assertion protects the native router canary separates loading first byte from first token regression after the test's arranged inputs and exercised call. + assert observation["firstTokenSeconds"] == 5.0 + # What: assert that observation decode seconds equals 1 0; why: this assertion protects the native router canary separates loading first byte from first token regression after the test's arranged inputs and exercised call. + assert observation["decodeSeconds"] == 1.0 + # What: assert that observation completion tokens per second equals 1 0; why: this assertion protects the native router canary separates loading first byte from first token regression after the test's arranged inputs and exercised call. + assert observation["completionTokensPerSecond"] == 1.0 + + +# What: define the test_native_router_benchmark_rejects_nonterminal_or_wrong_answer_streams test around native router qualifier and monkeypatch; why: this test groups the arrange, act, and assertions that protect the native router benchmark rejects nonterminal or wrong answer streams outcome. +def test_native_router_benchmark_rejects_nonterminal_or_wrong_answer_streams(native_router_qualifier, monkeypatch): + # What: act by calling io.BytesIO and capture stream; why: the native router benchmark rejects nonterminal or wrong answer streams test asserts the response, state, or failure produced by this call. + stream = io.BytesIO(b'data: {"choices":[{"delta":{"content":"5"}}]}\n\n') + # What: arrange the exact monkeypatch setattr native router qualifier urllib request urlopen lambda a k fixture f; why: the native router benchmark rejects nonterminal or wrong answer streams scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen". + monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen", lambda *a, **k: stream) + + # What: assert the pytest.raises failure context; why: the native router benchmark rejects nonterminal or wrong answer streams scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError): + # What: arrange the exact native router qualifier canary http test model a direct fixture fragment; why: the native router benchmark rejects nonterminal or wrong answer streams scenario feeds this byte-preserved fragment through native_router_qualifier.canary("http://test", "model-a", direct=True) before asser. + native_router_qualifier.canary("http://test", "model-a", direct=True) + # What: assert that stream closed; why: this assertion protects the native router benchmark rejects nonterminal or wrong answer streams regression after the test's arranged inputs and exercised call. + assert stream.closed + + +# What: define the test_native_router_benchmark_rejects_completed_stream_without_usage test around native router qualifier and monkeypatch; why: this test groups the arrange, act, and assertions that protect the native router benchmark rejects completed stream without usage outcome. +def test_native_router_benchmark_rejects_completed_stream_without_usage(native_router_qualifier, monkeypatch): + # What: act by calling io.BytesIO and capture stream; why: the native router benchmark rejects completed stream without usage test asserts the response, state, or failure produced by this call. + stream = io.BytesIO( + # What: arrange the b data choices delta content n portion of stream; why: the native router benchmark rejects completed stream without usage scenario uses this clause to evaluate stream as one grouped value. + b'data: {"choices":[{"delta":{"content":"4"}}]}\n\n' + # What: arrange the b data done n n portion of stream; why: the native router benchmark rejects completed stream without usage scenario uses this clause to evaluate stream as one grouped value. + b"data: [DONE]\n\n" + # What: arrange the io.BytesIO call with ordered positional inputs; why: test_native_router_benchmark_rejects_completed_stream_without_usage groups the supplied clauses as one io.BytesIO call before its value is consumed. + ) + # What: arrange the exact monkeypatch setattr native router qualifier urllib request urlopen lambda a k fixture f; why: the native router benchmark rejects completed stream without usage scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen", l bef. + monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen", lambda *a, **k: stream) + + # What: assert the pytest.raises failure context; why: the native router benchmark rejects completed stream without usage scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError, match="usage missing"): + # What: arrange the exact native router qualifier canary http test model a direct fixture fragment; why: the native router benchmark rejects completed stream without usage scenario feeds this byte-preserved fragment through native_router_qualifier.canary("http://test", "model-a", direct=True) before asserting. + native_router_qualifier.canary("http://test", "model-a", direct=True) + # What: assert that stream closed; why: this assertion protects the native router benchmark rejects completed stream without usage regression after the test's arranged inputs and exercised call. + assert stream.closed + + +# What: define the test_native_router_reload_conflict_canary_preserves_active_identity test around native router qualifier and monkeypatch and tmp path; why: this test groups the arrange, act, and assertions that protect the native router reload conflict canary preserves active identity outcome. +def test_native_router_reload_conflict_canary_preserves_active_identity( + # What: arrange native router qualifier monkeypatch tmp path for the scenario; why: test native router reload conflict canary preserves active identity requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch, tmp_path +# What: arrange the grouped source fragment for the scenario; why: test native router reload conflict canary preserves active identity requires this concrete input or helper state before exercising the behavior under test. +): + # What: define the request_json test helper around url and body; why: the native router reload conflict canary preserves active identity scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def request_json(url, body=None, **kwargs): + # What: act on endswith and url before httperror and url and error and bytes io and urllib; why: the native router reload conflict canary preserves active identity scenario admits httperror and url and error and bytes io and urllib only for this predicate and excludes the opposite state. + if url.endswith("/router/reload"): + # What: arrange the helper to raise raise native router qualifier urllib error HTTPError url 409 conflict io BytesIO; why: test swap qualification test exercises the concrete failure path rather than a successful substitute. + raise native_router_qualifier.urllib.error.HTTPError(url, 409, "conflict", {}, io.BytesIO()) + # What: assert that url endswith router status; why: this assertion protects the native router reload conflict canary preserves active identity regression after the test's arranged inputs and exercised call. + assert url.endswith("/router/status") + # What: arrange the helper response as b activeProfile model a activeIdentityMatchesEngine True; why: test native router reload conflict canary pre feeds this result into the behavior whose outcome is asserted. + return b"{}", {"activeProfile": "model-a", "activeIdentityMatchesEngine": True} + + # What: arrange the exact monkeypatch setattr native router qualifier request json request json fixture fragment; why: the native router reload conflict canary preserves active identity scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier, "request_json", request_jso befo. + monkeypatch.setattr(native_router_qualifier, "request_json", request_json) + # What: arrange catalog as tmp path and models and toml; why: the native router reload conflict canary preserves active identity test consumes this named precondition before exercising the behavior. + catalog = tmp_path / "models.toml" + # What: act by calling native_router_qualifier.reload_conflict_canary and capture observation; why: the native router reload conflict canary preserves active identity test asserts the response, state, or failure produced by this call. + observation = native_router_qualifier.reload_conflict_canary( + # What: arrange the http test catalog private a gguf private portion of observation; why: the native router reload conflict canary preserves active identity scenario uses this clause to evaluate observation as one grouped value. + "http://test", catalog, "/private/a.gguf", "/private/b.gguf" + # What: arrange the native_router_qualifier.reload_conflict_canary call with catalog; why: test_native_router_reload_conflict_canary_preserves_active_identity groups the supplied clauses as one native_router_qualifier.reload_conflict_canary call before its value is consumed. + ) + + # What: assert the expected observation == outcome; why: test swap qualification test native router reload conflict canary preserves active identity protects its regression by requiring this observable result after the exercised behavior. + assert observation == { + # What: arrange activeProfile model a rejectedStatus 409 for the scenario; why: test swap qualification test native router reload conflict canary preserves active identity requires this concrete input or helper state before exercising the behavior under test. + "activeProfile": "model-a", "rejectedStatus": 409, + # What: arrange activeIdentityPreserved True passed True for the scenario; why: test swap qualification test native router reload conflict canary preserves active identity requires this concrete input or helper state before exercising the behavior under test. + "activeIdentityPreserved": True, "passed": True, + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router reload conflict canary preserves active identity requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that priority 1 is present in catalog read text encoding utf 8; why: this assertion protects the native router reload conflict canary preserves active identity regression after the test's arranged inputs and exercised call. + assert "priority = 1" in catalog.read_text(encoding="utf-8") + + +# What: define the test_native_router_failed_switch_canary_requires_rollback_and_restored_completion test around native router qualifier and monkeypatch; why: this test groups the arrange, act, and assertions that protect the native router failed switch canary requires rollback and restored completion outcome. +def test_native_router_failed_switch_canary_requires_rollback_and_restored_completion( + # What: arrange native router qualifier monkeypatch for the scenario; why: test native router failed switch canary requires rollback and restored completion requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch +# What: arrange the grouped source fragment for the scenario; why: test native router failed switch canary requires rollback and restored completion requires this concrete input or helper state. +): + # What: act by calling iter and capture statuses; why: the native router failed switch canary requires rollback and restored completion test asserts the response, state, or failure produced by this call. + statuses = iter(( + # What: arrange the grouped source fragment for the scenario; why: test native router failed switch canary requires rollback and restored completion requires this concrete input or helper. + { + # What: arrange the active profile field as model a; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion carries active profile through statuses into return b next statuses. + "activeProfile": "model-a", "activeIdentityMatchesEngine": True, + # What: arrange activeRequests 0 activationFailures 3 for the scenario; why: test swap qualification test native router failed switch canary requires rollback and restored completion requires this concrete input or helper state before exercising the behavior under test. + "activeRequests": 0, "activationFailures": 3, + # What: arrange the grouped source fragment for the scenario; why: test native router failed switch canary requires rollback and restored completion requires this concrete input or. + }, + # What: arrange the grouped source fragment for the scenario; why: test native router failed switch canary requires rollback and restored completion requires this concrete input or helper. + { + # What: arrange the active profile field as model a; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion carries active profile through statuses into return b next statuses. + "activeProfile": "model-a", "activeIdentityMatchesEngine": True, + # What: arrange activeRequests 0 activationFailures 4 for the scenario; why: test swap qualification test native router failed switch canary requires rollback and restored completion requires this concrete input or helper state before exercising the behavior under test. + "activeRequests": 0, "activationFailures": 4, + # What: arrange the grouped source fragment for the scenario; why: test native router failed switch canary requires rollback and restored completion requires this concrete input or. + }, + # What: arrange the iter call with ordered positional inputs; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion groups the supplied clauses as one iter call before its value is consumed. + )) + # What: act by calling operation.encode and capture failure; why: the native router failed switch canary requires rollback and restored completion test asserts the response, state, or failure produced by this call. + failure = json.dumps({ + # What: arrange the type field as engine not ready; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion carries type through failure into url 503 unavailable io bytes io failure. + "error": {"type": "engine_not_ready", "message": "private failure"}, + # What: arrange the launched field as true; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion carries launched through failure into url 503 unavailable io bytes io failure. + "recovery": {"launched": True}, + # What: arrange the encode portion of failure; why: the native router failed switch canary requires rollback and restored completion scenario uses this clause to evaluate failure as one grouped value. + }).encode() + # What: act by calling iter and capture pending; why: the native router failed switch canary requires rollback and restored completion test asserts the response, state, or failure produced by this call. + pending = iter(( + # What: arrange the receipts field as receipt id and existing receipt; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion carries receipts through pending into if url endswith accounting pending. + {"receipts": [{"receiptId": "existing-receipt"}]}, + # What: arrange the receipts field as receipt id and existing receipt and receipt id and failed switch receipt; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion carries receipts through pending into if url endswith accounting pending. + {"receipts": [ + # What: arrange the receipt id field as existing receipt; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion carries receipt id through pending into if url endswith accounting pending. + {"receiptId": "existing-receipt"}, + # What: arrange the receipt id field as failed switch receipt; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion carries receipt id through pending into if url endswith accounting pending. + {"receiptId": "failed-switch-receipt"}, + # What: arrange the pending mapping with receipts; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion groups the supplied clauses as one pending mapping before its value is consumed. + ]}, + # What: arrange the iter call with ordered positional inputs; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion groups the supplied clauses as one iter call before its value is consumed. + )) + # What: collect readiness URLs observed by the test double; why: the regression must prove rollback waits for the concrete restored engine before routing a completion. + readiness_waits = [] + + # What: define the request_json test helper around url and body; why: the native router failed switch canary requires rollback and restored completion scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def request_json(url, body=None, **kwargs): + # What: act on endswith and url before next and statuses; why: the native router failed switch canary requires rollback and restored completion scenario admits next and statuses only for this predicate and excludes the opposite state. + if url.endswith("/router/status"): + # What: return next and statuses from the request_json test helper; why: the native router failed switch canary requires rollback and restored completion scenario uses this helper result in its subsequent act or assertion. + return b"{}", next(statuses) + # What: act on endswith and url before next and pending; why: the native router failed switch canary requires rollback and restored completion scenario admits next and pending only for this predicate and excludes the opposite state. + if url.endswith("/accounting/pending"): + # What: return next and pending from the request_json test helper; why: the native router failed switch canary requires rollback and restored completion scenario uses this helper result in its subsequent act or assertion. + return b"{}", next(pending) + # What: return the concrete restored engine listener for status requests; why: failed-switch recovery now gates the routed canary on process readiness. + if url.endswith("/engine/status"): + # What: provide a running local engine with a deterministic port; why: the test can verify that readiness targets the exact replacement process. + return b"{}", {"running": True, "port": 49187} + # What: assert that url endswith router load and body equals name model invalid; why: this assertion protects the native router failed switch canary requires rollback and restored completion regression after the test's arranged inputs and exercised call. + assert url.endswith("/router/load") and body == {"name": "model-invalid"} + # What: arrange the helper to raise raise native router qualifier urllib error HTTPError; why: test native router failed switch canary exercises the concrete failure path rather than a successful substitute. + raise native_router_qualifier.urllib.error.HTTPError( + # What: act by calling io.BytesIO with failure; why: the native router failed switch canary requires rollback and restored completion scenario observes the io.BytesIO return value while evaluating url, 503, "unavailable", {}, io.BytesIO(failure). + url, 503, "unavailable", {}, io.BytesIO(failure) + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router failed switch canary requires rollback and. + ) + + # What: arrange the exact monkeypatch setattr native router qualifier request json request json fixture fragment; why: the native router failed switch canary requires rollback and restored completion scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier, "request_json", re. + monkeypatch.setattr(native_router_qualifier, "request_json", request_json) + # What: replace readiness polling with a deterministic recorder; why: the unit test validates the target URL without opening a real listener or sleeping. + monkeypatch.setattr( + # What: bind the qualifier helper selected for replacement; why: the production wait must be intercepted at its module lookup point. + native_router_qualifier, + # What: name the readiness helper being replaced; why: only rollback readiness polling should be converted into a test observation. + "wait_json", + # What: append each requested URL and timeout to the observation list; why: exact endpoint and bounded-wait behavior are part of the recovery contract. + lambda url, *, seconds: readiness_waits.append((url, seconds)), + # What: close the monkeypatch call; why: this completes the deterministic readiness test setup. + ) + # What: act by calling monkeypatch.setattr with native router qualifier and canary and passed and true; why: the native router failed switch canary requires rollback and restored completion scenario observes the monkeypatch.setattr return value during native router qualifier canary. + monkeypatch.setattr( + # What: arrange the exact native router qualifier canary fixture fragment; why: the native router failed switch canary requires rollback and restored completion scenario feeds this byte-preserved fragment through native_router_qualifier, "canary" before asserting its protocol or parser result. + native_router_qualifier, "canary", + # What: arrange the passed field as true; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion carries passed into lambda base, model, *, direct: (b"data: [DONE]\n\n", {"passed": True}). + lambda base, model, *, direct: (b"data: [DONE]\n\n", {"passed": True}), + # What: arrange the monkeypatch.setattr call with native router qualifier; why: test_native_router_failed_switch_canary_requires_rollback_and_restored_completion groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + + # What: act by calling native_router_qualifier.failed_switch_canary and capture failure raw and restored raw and observation; why: the native router failed switch canary requires rollback and restored completion test asserts the response, state, or failure produced by this call. + failure_raw, restored_raw, observation = native_router_qualifier.failed_switch_canary( + # What: arrange http test model invalid model a for the scenario; why: test model invalid model a in test requires this concrete input or helper state before exercising the behavior under test. + "http://test", "model-invalid", "model-a" + # What: arrange the grouped source fragment for the scenario; why: test native router failed switch canary requires rollback and restored completion requires this concrete input or helper. + ) + + # What: assert that failure raw equals failure; why: this assertion protects the native router failed switch canary requires rollback and restored completion regression after the test's arranged inputs and exercised call. + assert failure_raw == failure + # What: assert that restored raw equals b data done n n; why: this assertion protects the native router failed switch canary requires rollback and restored completion regression after the test's arranged inputs and exercised call. + assert restored_raw == b"data: [DONE]\n\n" + # What: require a bounded wait against the restored engine's direct readiness endpoint; why: router metadata alone previously allowed a premature 502 canary. + assert readiness_waits == [("http://127.0.0.1:49187/ready", 600)] + # What: assert the expected observation == outcome; why: test swap qualification test native router failed switch canary requires rollback and restored completion protects its regression by requiring this observable result after the exercised behavior. + assert observation == { + # What: arrange failedProfile model invalid restoredProfile model a for the scenario; why: test swap qualification test native router failed switch canary requires rollback and restored completion requires this concrete input or helper state before exercising the behavior under test. + "failedProfile": "model-invalid", "restoredProfile": "model-a", + # What: arrange failureType engine not ready rollbackLaunched True for the scenario; why: test swap qualification test native router failed switch canary requires rollback and restored completion requires this concrete input or helper state before exercising the behavior under test. + "failureType": "engine_not_ready", "rollbackLaunched": True, + # What: arrange activationFailureIncremented True newAccountingReceiptCount 1 for the scenario; why: test swap qualification test requires this concrete input or helper state before exercising the behavior under test. + "activationFailureIncremented": True, "newAccountingReceiptCount": 1, + # What: arrange restoredCompletionPassed True for the scenario; why: test swap qualification test native router failed switch canary requires rollback and restored completion requires this concrete input or helper state before exercising the behavior under test. + "restoredCompletionPassed": True, + # What: arrange passed True for the scenario; why: test swap qualification test native router failed switch canary requires rollback and restored completion requires this concrete input or helper state before exercising the behavior under test. + "passed": True, + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router failed switch canary requires rollback and. + } + + +# What: define a focused running-engine identity validation test; why: failure cleanup must target only a daemon-proven process and listener. +def test_running_engine_identity_requires_exact_running_process(native_router_qualifier, monkeypatch): + # What: provide a valid running engine status response; why: the helper should preserve the exact process group and listener selected for bounded cleanup. + monkeypatch.setattr( + # What: bind the qualifier module under test; why: request lookup occurs in this module's global namespace. + native_router_qualifier, + # What: replace the JSON request helper; why: the test should not require a live daemon endpoint. + "request_json", + # What: return a deterministic running engine identity; why: successful ownership capture can be asserted without process side effects. + lambda url, **kwargs: (b"{}", {"running": True, "pid": 321, "port": 49187}), + # What: close the first monkeypatch call; why: this completes the valid running-engine test setup before assertions execute. + ) + # What: require exact PID and port preservation; why: cleanup must not broaden or rewrite the manager-owned identity. + assert native_router_qualifier.running_engine_identity("http://test") == (321, 49187) + + # What: replace status with an idle response that still contains stale identifiers; why: non-running metadata must never authorize a signal. + monkeypatch.setattr( + # What: bind the qualifier module for the second case; why: the same helper lookup point is exercised with rejected state. + native_router_qualifier, + # What: replace the JSON request helper again; why: this case models an idle daemon after engine exit. + "request_json", + # What: return stale identifiers with running false; why: the helper must fail closed on lifecycle state before trusting numeric fields. + lambda url, **kwargs: (b"{}", {"running": False, "pid": 321, "port": 49187}), + # What: close the second monkeypatch call; why: this completes the idle-engine rejection setup before its assertion executes. + ) + # What: reject the stale idle identity; why: cleanup must not signal a process that may have been reused by the operating system. + assert native_router_qualifier.running_engine_identity("http://test") is None + + +# What: define the test_native_router_ttl_canary_reloads_temporary_catalog_and_closes_listener test around native router qualifier and monkeypatch and tmp path; why: this test groups the arrange, act, and assertions that protect the native router ttl canary reloads temporary catalog and closes listener outcome. +def test_native_router_ttl_canary_reloads_temporary_catalog_and_closes_listener( + # What: arrange native router qualifier monkeypatch tmp path for the scenario; why: test native router ttl canary reloads temporary requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch, tmp_path +# What: arrange the grouped source fragment for the scenario; why: test native router ttl canary reloads temporary catalog and closes listener requires this concrete input or helper state before exercising the behavior under test. +): + # What: act by calling iter and capture statuses; why: the native router ttl canary reloads temporary catalog and closes listener test asserts the response, state, or failure produced by this call. + statuses = iter(( + # What: arrange the evictions field as 2; why: test_native_router_ttl_canary_reloads_temporary_catalog_and_closes_listener carries evictions through statuses into return b next statuses. + {"evictions": 2, "activeProfile": "model-a"}, + # What: arrange the eviction counter after explicit unload and automatic TTL eviction; why: the final status must include both independently accounted lifecycle events. + {"evictions": 4, "activeProfile": None}, + # What: arrange the iter call with ordered positional inputs; why: test_native_router_ttl_canary_reloads_temporary_catalog_and_closes_listener groups the supplied clauses as one iter call before its value is consumed. + )) + + # What: define the request_json test helper around url and body; why: the native router ttl canary reloads temporary catalog and closes listener scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def request_json(url, body=None, **kwargs): + # What: act on endswith and url before next and statuses; why: the native router ttl canary reloads temporary catalog and closes listener scenario admits next and statuses only for this predicate and excludes the opposite state. + if url.endswith("/router/status"): + # What: return next and statuses from the request_json test helper; why: the native router ttl canary reloads temporary catalog and closes listener scenario uses this helper result in its subsequent act or assertion. + return b"{}", next(statuses) + # What: arrange if url endswith router unload for the scenario; why: test swap qualification test native router ttl canary reloads temporary catalog and closes listener requires this concrete input or helper state before exercising the behavior under test. + if url.endswith("/router/unload"): + # What: arrange the helper response as b unloaded true unloaded True; why: test swap qualification test native router ttl canary reloads temporary catalog and closes listener feeds this result into the behavior whose outcome is asserted. + return b'{"unloaded":true}', { + # What: report successful explicit unload; why: the TTL fixture must proceed to temporary catalog activation. + "unloaded": True, + # What: report the router's post-unload counter; why: explicit unload is an eviction and becomes the baseline for the later automatic TTL increment. + "router": {"evictions": 3}, + # What: close the explicit-unload response mapping; why: the fixture returns one coherent management response. + } + # What: arrange if url endswith router reload for the scenario; why: test swap qualification test native router ttl canary reloads temporary catalog and closes listener requires this concrete input or helper state before exercising the behavior under test. + if url.endswith("/router/reload"): + # What: arrange the helper response as b reloaded true reloaded True; why: test swap qualification test native router ttl canary reloads temporary catalog and closes listener feeds this result into the behavior whose outcome is asserted. + return b'{"reloaded":true}', {"reloaded": True} + # What: assert that url endswith router load and body equals name model a; why: this assertion protects the native router ttl canary reloads temporary catalog and closes listener regression after the test's arranged inputs and exercised call. + assert url.endswith("/router/load") and body == {"name": "model-a"} + # What: arrange the helper response as b profile model a port 24567 profile model a port 24567; why: test native router ttl canary reloads temporary feeds this result into the behavior whose outcome is asserted. + return b'{"profile":"model-a","port":24567}', {"profile": "model-a", "port": 24567} + + # What: arrange closed as the fixture input; why: the native router ttl canary reloads temporary catalog and closes listener test consumes this named precondition before exercising the behavior. + closed = [] + # What: collect routed canaries used to trigger TTL scheduling; why: the test must prove eviction follows a completed request rather than preload alone. + ttl_canaries = [] + # What: arrange the exact monkeypatch setattr native router qualifier request json request json fixture fragment; why: the native router ttl canary reloads temporary catalog and closes listener scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier, "request_json", request_. + monkeypatch.setattr(native_router_qualifier, "request_json", request_json) + # What: arrange monkeypatch setattr native router qualifier require listener closed closed append for the scenario; why: test native router ttl canary reloads temporary catalog and closes listener requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr(native_router_qualifier, "require_listener_closed", closed.append) + # What: replace the routed canary with a deterministic successful trigger; why: the unit test validates lifecycle ordering without running inference. + monkeypatch.setattr( + # What: bind the qualifier module under test; why: ttl_eviction_canary resolves its canary helper from this module. + native_router_qualifier, + # What: select the routed canary helper; why: only the request that releases TTL ownership should be intercepted. + "canary", + # What: record the exact routed request and return success; why: the assertion can prove model and route selection while allowing eviction polling to proceed. + lambda base, model, *, direct: (ttl_canaries.append((base, model, direct)) or b"done", {"passed": True}), + # What: close the canary monkeypatch call; why: this completes deterministic TTL trigger setup. + ) + # What: arrange catalog as tmp path and models and toml; why: the native router ttl canary reloads temporary catalog and closes listener test consumes this named precondition before exercising the behavior. + catalog = tmp_path / "models.toml" + # What: act by calling native_router_qualifier.ttl_eviction_canary and capture observation; why: the native router ttl canary reloads temporary catalog and closes listener test asserts the response, state, or failure produced by this call. + observation = native_router_qualifier.ttl_eviction_canary( + # What: arrange seconds to native_router_qualifier.ttl_eviction_canary; why: the native router ttl canary reloads temporary catalog and closes listener scenario binds this 1 value to native_router_qualifier.ttl_eviction_canary's seconds input. + "http://test", catalog, "/private/a.gguf", "/private/b.gguf", seconds=1 + # What: arrange the native_router_qualifier.ttl_eviction_canary call with seconds; why: test_native_router_ttl_canary_reloads_temporary_catalog_and_closes_listener groups the supplied clauses as one native_router_qualifier.ttl_eviction_canary call before its value is consumed. + ) + + # What: assert that closed equals 24567; why: this assertion protects the native router ttl canary reloads temporary catalog and closes listener regression after the test's arranged inputs and exercised call. + assert closed == [24567] + # What: require one routed model-a completion before eviction polling; why: the router schedules idle TTL only after request ownership is released. + assert ttl_canaries == [("http://test", "model-a", False)] + # What: assert the expected observation == outcome; why: test swap qualification test native router ttl canary reloads temporary catalog and closes listener protects its regression by requiring this observable result after the exercised behavior. + assert observation == { + # What: arrange profile model a ttlSeconds 2 port 24567 for the scenario; why: test swap qualification test native router ttl canary reloads temporary catalog and closes listener requires this concrete input or helper state before exercising the behavior under test. + "profile": "model-a", "ttlSeconds": 2, "port": 24567, + # What: arrange evictionIncremented True listenerClosed True passed True for the scenario; why: test swap qualification test native router ttl canary reloads temporary catalog and closes listener requires this concrete input or helper state before exercising the behavior under test. + "evictionIncremented": True, "listenerClosed": True, "passed": True, + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router ttl canary reloads temporary catalog and closes listener requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that ttl s 2 is present in catalog read text encoding utf 8; why: this assertion protects the native router ttl canary reloads temporary catalog and closes listener regression after the test's arranged inputs and exercised call. + assert "ttl_s = 2" in catalog.read_text(encoding="utf-8") + + +# What: define the test_native_router_persistent_capacity_canary_requires_release_before_switch test around native router qualifier and monkeypatch and tmp path; why: this test groups the arrange, act, and assertions that protect the native router persistent capacity canary requires release before switch outcome. +def test_native_router_persistent_capacity_canary_requires_release_before_switch( + # What: arrange native router qualifier monkeypatch tmp path for the scenario; why: test native router persistent capacity canary requires release before switch requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch, tmp_path +# What: arrange the grouped source fragment for the scenario; why: test native router persistent capacity canary requires release before switch requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange calls as model b and 0; why: the native router persistent capacity canary requires release before switch test consumes this named precondition before exercising the behavior. + calls = {"model-b": 0} + # What: arrange rejection as the fixture input; why: the native router persistent capacity canary requires release before switch test consumes this named precondition before exercising the behavior. + rejection = b'{"error":{"type":"capacity_unavailable"}}' + + # What: define the request_json test helper around url and body; why: the native router persistent capacity canary requires release before switch scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def request_json(url, body=None, **kwargs): + # What: arrange if url endswith router unload for the scenario; why: test swap qualification test native router persistent capacity canary requires release before switch requires this concrete input or helper state before exercising the behavior under test. + if url.endswith("/router/unload"): + # What: arrange the unloaded field as true; why: request_json carries unloaded into return b"{}", {"unloaded": True}. + return b"{}", {"unloaded": True} + # What: arrange if url endswith router reload for the scenario; why: test swap qualification test native router persistent capacity canary requires release before switch requires this concrete input or helper state before exercising the behavior under test. + if url.endswith("/router/reload"): + # What: arrange the reloaded field as true; why: request_json carries reloaded into return b"{}", {"reloaded": True}. + return b"{}", {"reloaded": True} + # What: arrange if url endswith router status for the scenario; why: test swap qualification test native router persistent capacity canary requires release before switch requires this concrete input or helper state before exercising the behavior under test. + if url.endswith("/router/status"): + # What: return active profile and active identity matches engine and persistent and model a and true from the request_json test helper; why: the native router persistent capacity canary requires release before switch scenario uses this helper result in its subsequent act or assertion. + return b"{}", { + # What: arrange activeProfile model a activeIdentityMatchesEngine True for the scenario; why: test swap qualification test requires this concrete input or helper state before exercising the behavior under test. + "activeProfile": "model-a", "activeIdentityMatchesEngine": True, + # What: arrange the persistent field as true; why: request_json carries persistent into "persistent": True. + "persistent": True, + # What: arrange the enclosing predicate collection with the named fixture input and active profile and active identity matches engine and persistent and model a and true; why: request_json groups the supplied clauses as one request_json expression collection before its value is consumed. + } + # What: arrange if url endswith engine status for the scenario; why: test swap qualification test native router persistent capacity canary requires release before switch requires this concrete input or helper state before exercising the behavior under test. + if url.endswith("/engine/status"): + # What: arrange the pid field as 71; why: request_json carries pid into return b"{}", {"pid": 71}. + return b"{}", {"pid": 71} + # What: assert that url endswith router load; why: this assertion protects the native router persistent capacity canary requires release before switch regression after the test's arranged inputs and exercised call. + assert url.endswith("/router/load") + # What: arrange the name field as model a; why: request_json carries name into if body == {"name": "model-a"}. + if body == {"name": "model-a"}: + # What: return profile and pid and router and model a and 71 from the request_json test helper; why: the native router persistent capacity canary requires release before switch scenario uses this helper result in its subsequent act or assertion. + return b"{}", { + # What: arrange the profile field as model a; why: request_json carries profile into "profile": "model-a", "pid": 71. + "profile": "model-a", "pid": 71, + # What: arrange router persistent True activeIdentityMatchesEngine True for the scenario; why: test swap qualification test native router persistent capacity canary requires release before switch requires this concrete input or helper state before exercising the behavior under test. + "router": {"persistent": True, "activeIdentityMatchesEngine": True}, + # What: arrange the enclosing predicate collection with the named fixture input and profile and pid and router and model a and 71; why: request_json groups the supplied clauses as one request_json expression collection before its value is consumed. + } + # What: arrange calls entry from 1; why: the native router persistent capacity canary requires release before switch scenario uses calls entry during if calls model b before checking the protected result. + calls["model-b"] += 1 + # What: act on calls before httperror and url and error and bytes io and rejection; why: the native router persistent capacity canary requires release before switch scenario admits httperror and url and error and bytes io and rejection only for this predicate and excludes the opposite state. + if calls["model-b"] == 1: + # What: arrange the helper to raise raise native router qualifier urllib error HTTPError; why: test swap qualification test exercises the concrete failure path rather than a successful substitute. + raise native_router_qualifier.urllib.error.HTTPError( + # What: act by calling io.BytesIO with rejection; why: the native router persistent capacity canary requires release before switch scenario observes the io.BytesIO return value while evaluating url, 409, "capacity", {}, io.BytesIO(rejection). + url, 409, "capacity", {}, io.BytesIO(rejection) + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router persistent capacity canary requires release before switch requires this concrete input or helper state before exercising the behavior under test. + ) + # What: return profile and pid and router and model b and 72 from the request_json test helper; why: the native router persistent capacity canary requires release before switch scenario uses this helper result in its subsequent act or assertion. + return b"{}", { + # What: arrange the profile field as model b; why: request_json carries profile into "profile": "model-b", "pid": 72. + "profile": "model-b", "pid": 72, + # What: arrange the active identity matches engine field as true; why: request_json carries active identity matches engine into "router": {"activeIdentityMatchesEngine": True}. + "router": {"activeIdentityMatchesEngine": True}, + # What: arrange the enclosing predicate collection with the named fixture input and profile and pid and router and model b and 72; why: request_json groups the supplied clauses as one request_json expression collection before its value is consumed. + } + + # What: arrange the exact monkeypatch setattr native router qualifier request json request json fixture fragment; why: the native router persistent capacity canary requires release before switch scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier, "request_json", request. + monkeypatch.setattr(native_router_qualifier, "request_json", request_json) + # What: arrange catalog as tmp path and models and toml; why: the native router persistent capacity canary requires release before switch test consumes this named precondition before exercising the behavior. + catalog = tmp_path / "models.toml" + # What: act by calling native_router_qualifier.persistent_capacity_canary and capture raw and observation; why: the native router persistent capacity canary requires release before switch test asserts the response, state, or failure produced by this call. + raw, observation = native_router_qualifier.persistent_capacity_canary( + # What: arrange the http test catalog a gguf b gguf portion of raw and observation; why: the native router persistent capacity canary requires release before switch scenario uses this clause to evaluate raw and observation as one grouped value. + "http://test", catalog, "a.gguf", "b.gguf" + # What: arrange the native_router_qualifier.persistent_capacity_canary call with catalog; why: test_native_router_persistent_capacity_canary_requires_release_before_switch groups the supplied clauses as one native_router_qualifier.persistent_capacity_canary call before its value is consumed. + ) + + # What: assert that raw equals rejection; why: this assertion protects the native router persistent capacity canary requires release before switch regression after the test's arranged inputs and exercised call. + assert raw == rejection + # What: assert that observation passed is true; why: this assertion protects the native router persistent capacity canary requires release before switch regression after the test's arranged inputs and exercised call. + assert observation["passed"] is True + # What: assert that observation resident pid preserved is true; why: this assertion protects the native router persistent capacity canary requires release before switch regression after the test's arranged inputs and exercised call. + assert observation["residentPidPreserved"] is True + # What: act by calling ModelCatalog.load and capture parsed; why: the native router persistent capacity canary requires release before switch test asserts the response, state, or failure produced by this call. + parsed = ModelCatalog.load(str(catalog)) + # What: assert that parsed group for model a persistent is true; why: this assertion protects the native router persistent capacity canary requires release before switch regression after the test's arranged inputs and exercised call. + assert parsed.group_for("model-a").persistent is True + + +# What: define the test_native_router_concurrent_canaries_require_same_residency test around native router qualifier and monkeypatch; why: this test groups the arrange, act, and assertions that protect the native router concurrent canaries require same residency outcome. +def test_native_router_concurrent_canaries_require_same_residency(native_router_qualifier, monkeypatch): + # What: act by calling iter and capture snapshots; why: the native router concurrent canaries require same residency test asserts the response, state, or failure produced by this call. + snapshots = iter(( + # What: arrange the active profile field as model a; why: test_native_router_concurrent_canaries_require_same_residency carries active profile through snapshots into monkeypatch setattr native router qualifier request json lambda a k b. + {"activeProfile": "model-a", "activations": 4, "activeRequests": 0}, + # What: arrange the active profile field as model a; why: test_native_router_concurrent_canaries_require_same_residency carries active profile through snapshots into monkeypatch setattr native router qualifier request json lambda a k b. + {"activeProfile": "model-a", "activations": 4, "activeRequests": 0}, + # What: arrange the iter call with ordered positional inputs; why: test_native_router_concurrent_canaries_require_same_residency groups the supplied clauses as one iter call before its value is consumed. + )) + # What: arrange the exact monkeypatch setattr native router qualifier request json lambda a k fixture fragment; why: the native router concurrent canaries require same residency scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier, "request_json", lambda *a, before assert. + monkeypatch.setattr(native_router_qualifier, "request_json", lambda *a, **k: (b"{}", next(snapshots))) + + # What: define the fake_canary test helper around base and model and direct; why: the native router concurrent canaries require same residency scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def fake_canary(base, model, *, direct): + # What: assert that base equals http test and model equals model a and direct; why: this assertion protects the native router concurrent canaries require same residency regression after the test's arranged inputs and exercised call. + assert base == "http://test" and model == "model-a" and direct is False + # What: act by calling time.sleep with 0 01; why: the native router concurrent canaries require same residency scenario observes the time.sleep return value during return b data done n n. + time.sleep(0.01) + # What: arrange the passed field as true; why: fake_canary carries passed into return b"data: [DONE]\n\n", {"passed": True}. + return b"data: [DONE]\n\n", {"passed": True} + + # What: arrange the exact monkeypatch setattr native router qualifier canary fake canary fixture fragment; why: the native router concurrent canaries require same residency scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier, "canary", fake_canary) before asserting its p. + monkeypatch.setattr(native_router_qualifier, "canary", fake_canary) + # What: act by calling native_router_qualifier.concurrent_canaries and capture rows and observation; why: the native router concurrent canaries require same residency test asserts the response, state, or failure produced by this call. + rows, observation = native_router_qualifier.concurrent_canaries("http://test", "model-a", seconds=2) + + # What: assert that len rows equals 2; why: this assertion protects the native router concurrent canaries require same residency regression after the test's arranged inputs and exercised call. + assert len(rows) == 2 + # What: assert the expected observation == outcome; why: test swap qualification test native router concurrent canaries require same residency protects its regression by requiring this observable result after the exercised behavior. + assert observation == { + # What: arrange route native router for the scenario; why: test swap qualification test native router concurrent canaries require same residency requires this concrete input or helper state before exercising the behavior under test. + "route": "native_router", + # What: arrange model model a for the scenario; why: test swap qualification test native router concurrent canaries require same residency requires this concrete input or helper state before exercising the behavior under test. + "model": "model-a", + # What: arrange requests 2 for the scenario; why: test swap qualification test native router concurrent canaries require same residency requires this concrete input or helper state before exercising the behavior under test. + "requests": 2, + # What: arrange activationDelta 0 for the scenario; why: test swap qualification test native router concurrent canaries require same residency requires this concrete input or helper state before exercising the behavior under test. + "activationDelta": 0, + # What: arrange activeRequestsAfter 0 for the scenario; why: test swap qualification test native router concurrent canaries require same residency requires this concrete input or helper state before exercising the behavior under test. + "activeRequestsAfter": 0, + # What: arrange passed True for the scenario; why: test swap qualification test native router concurrent canaries require same residency requires this concrete input or helper state before exercising the behavior under test. + "passed": True, + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router concurrent canaries require same residency requires this concrete input or helper state before exercising the behavior under test. + } + + +# What: define the test_native_router_conflicting_request_canary_queues_then_switches test around native router qualifier and monkeypatch; why: this test groups the arrange, act, and assertions that protect the native router conflicting request canary queues then switches outcome. +def test_native_router_conflicting_request_canary_queues_then_switches( + # What: arrange native router qualifier monkeypatch for the scenario; why: test native router conflicting request canary queues then switches requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch +# What: arrange the grouped source fragment for the scenario; why: test native router conflicting request canary queues then switches requires this concrete input or helper state before exercising the behavior. +): + # What: act by calling threading.Event and capture cancelled; why: the native router conflicting request canary queues then switches test asserts the response, state, or failure produced by this call. + cancelled = threading.Event() + # What: act by calling iter and capture statuses; why: the native router conflicting request canary queues then switches test asserts the response, state, or failure produced by this call. + statuses = iter(( + # What: arrange activeProfile model a activations 10 for the scenario; why: test swap qualification test native router conflicting request canary queues then switches requires this concrete input or helper state before exercising the behavior under test. + {"activeProfile": "model-a", "activations": 10}, + # What: arrange the grouped source fragment for the scenario; why: test native router conflicting request canary queues then switches requires this concrete input or helper state before exercising the. + { + # What: arrange activeProfile model a activations 10 for the scenario; why: test swap qualification test native router conflicting request canary queues then switches requires this concrete input or helper state before exercising the behavior under test. + "activeProfile": "model-a", "activations": 10, + # What: arrange the queued requests field as 1; why: test_native_router_conflicting_request_canary_queues_then_switches carries queued requests through statuses into return b next statuses. + "queuedRequests": 1, "activeRequests": 1, + # What: arrange the active identity matches engine field as true; why: test_native_router_conflicting_request_canary_queues_then_switches carries active identity matches engine through statuses into return b next statuses. + "activeIdentityMatchesEngine": True, + # What: arrange the grouped source fragment for the scenario; why: test native router conflicting request canary queues then switches requires this concrete input or helper state before exercising. + }, + # What: arrange the active profile field as model b; why: test_native_router_conflicting_request_canary_queues_then_switches carries active profile through statuses into return b next statuses. + {"activeProfile": "model-b", "activations": 11, "activeRequests": 0}, + # What: arrange activeProfile model a activations 12 activeRequests 0 for the scenario; why: test swap qualification test native router conflicting request canary queues then switches requires this concrete input or helper state before exercising the behavior under test. + {"activeProfile": "model-a", "activations": 12, "activeRequests": 0}, + # What: arrange the iter call with ordered positional inputs; why: test_native_router_conflicting_request_canary_queues_then_switches groups the supplied clauses as one iter call before its value is consumed. + )) + + # What: define ActiveResponse as the owner of __enter__ and __exit__ and __iter__; why: daemon callers use this class boundary so those methods share one active response state invariant. + class ActiveResponse: + # What: define the __enter__ test helper around captured fixture state; why: the native router conflicting request canary queues then switches scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def __enter__(self): + # What: return no value from the __enter__ test helper; why: the native router conflicting request canary queues then switches scenario uses this helper result in its subsequent act or assertion. + return self + + # What: define the __exit__ test helper around captured fixture state; why: the native router conflicting request canary queues then switches scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def __exit__(self, *args): + # What: ignore the anticipated exception handled by this branch; why: __exit__ continues its retry or cleanup path instead of re-raising that transient failure. + pass + + # What: define the __iter__ test helper around captured fixture state; why: the native router conflicting request canary queues then switches scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def __iter__(self): + # What: arrange the yield b data choices delta content portion of the enclosing predicate; why: this clause remains in the native router conflicting request canary queues then switches scenario\'s enclosing expression so its grouping and evaluation order stay intact. + yield b'data: {"choices":[{"delta":{"content":"1"}}]}\n\n' + # What: act by calling cancelled.wait with 2; why: the native router conflicting request canary queues then switches scenario observes the cancelled.wait return value during the enclosing return. + cancelled.wait(2) + + # What: define the request_json test helper around url and body; why: the native router conflicting request canary queues then switches scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def request_json(url, body=None, **kwargs): + # What: act on endswith and url before next and statuses; why: the native router conflicting request canary queues then switches scenario admits next and statuses only for this predicate and excludes the opposite state. + if url.endswith("/router/status"): + # What: return next and statuses from the request_json test helper; why: the native router conflicting request canary queues then switches scenario uses this helper result in its subsequent act or assertion. + return b"{}", next(statuses) + # What: assert that url endswith router requests native qualification conflict cancel; why: this assertion protects the native router conflicting request canary queues then switches regression after the test's arranged inputs and exercised call. + assert url.endswith("/router/requests/native-qualification-conflict/cancel") + # What: act by calling cancelled.set with the declared inputs; why: the native router conflicting request canary queues then switches scenario observes the cancelled.set return value during return b cancelled id native qualification conflict. + cancelled.set() + # What: arrange the cancelled field as true; why: request_json carries cancelled into return b"{}", {"cancelled": True, "id": "native-qualification-conflict"}. + return b"{}", {"cancelled": True, "id": "native-qualification-conflict"} + + # What: arrange canary calls as the fixture input; why: the native router conflicting request canary queues then switches test consumes this named precondition before exercising the behavior. + canary_calls = [] + + # What: define the fake_canary test helper around base and model and direct; why: the native router conflicting request canary queues then switches scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def fake_canary(base, model, *, direct): + # What: act by calling canary_calls.append with model; why: the native router conflicting request canary queues then switches scenario observes the canary_calls.append return value during if model model b. + canary_calls.append(model) + # What: act on model before wait and cancelled; why: the native router conflicting request canary queues then switches scenario admits wait and cancelled only for this predicate and excludes the opposite state. + if model == "model-b": + # What: act by calling cancelled.wait with 2; why: the native router conflicting request canary queues then switches scenario observes the cancelled.wait return value during return f data model n ndata. + cancelled.wait(2) + # What: arrange the helper response as f data model n ndata DONE n n encode passed True; why: test swap qualification test native router conflicting request canary queues then switches feeds this result into the behavior whose outcome is asserted. + return f"data: {model}\n\ndata: [DONE]\n\n".encode(), {"passed": True} + + # What: arrange the exact monkeypatch setattr native router qualifier request json request json fixture fragment; why: the native router conflicting request canary queues then switches scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier, "request_json", request_jso befor. + monkeypatch.setattr(native_router_qualifier, "request_json", request_json) + # What: arrange the exact monkeypatch setattr native router qualifier urllib request urlopen lambda a k fixture f; why: the native router conflicting request canary queues then switches scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen", l befo. + monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen", lambda *a, **k: ActiveResponse()) + # What: arrange the exact monkeypatch setattr native router qualifier canary fake canary fixture fragment; why: the native router conflicting request canary queues then switches scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier, "canary", fake_canary) before asserting. + monkeypatch.setattr(native_router_qualifier, "canary", fake_canary) + + # What: act by calling native_router_qualifier.conflicting_request_canary and capture active and waiting and restored and observation; why: the native router conflicting request canary queues then switches test asserts the response, state, or failure produced by this call. + active, waiting, restored, observation = native_router_qualifier.conflicting_request_canary( + # What: arrange seconds to native_router_qualifier.conflicting_request_canary; why: the native router conflicting request canary queues then switches scenario binds this 2 value to native_router_qualifier.conflicting_request_canary's seconds input. + "http://test", "model-a", "model-b", seconds=2 + # What: arrange the native_router_qualifier.conflicting_request_canary call with seconds; why: test_native_router_conflicting_request_canary_queues_then_switches groups the supplied clauses as one native_router_qualifier.conflicting_request_canary call before its value is consumed. + ) + + # What: assert that b data done is absent from active; why: this assertion protects the native router conflicting request canary queues then switches regression after the test's arranged inputs and exercised call. + assert b"data: [DONE]" not in active + # What: assert that b model b in waiting and b model a in restored; why: this assertion protects the native router conflicting request canary queues then switches regression after the test's arranged inputs and exercised call. + assert b"model-b" in waiting and b"model-a" in restored + # What: assert that canary calls equals model b model a; why: this assertion protects the native router conflicting request canary queues then switches regression after the test's arranged inputs and exercised call. + assert canary_calls == ["model-b", "model-a"] + # What: assert that observation queued behind active is true; why: this assertion protects the native router conflicting request canary queues then switches regression after the test's arranged inputs and exercised call. + assert observation["queuedBehindActive"] is True + # What: assert that observation activation delta equals 2; why: this assertion protects the native router conflicting request canary queues then switches regression after the test's arranged inputs and exercised call. + assert observation["activationDelta"] == 2 + # What: assert that observation passed is true; why: this assertion protects the native router conflicting request canary queues then switches regression after the test's arranged inputs and exercised call. + assert observation["passed"] is True + + +# What: define the test_native_router_cancellation_canary_requires_idle_without_completion_credit test around native router qualifier; why: this test groups the arrange, act, and assertions that protect the native router cancellation canary requires idle without completion credit outcome. +def test_native_router_cancellation_canary_requires_idle_without_completion_credit(native_router_qualifier): + # What: arrange state as active and cancellations and terminal and 0 and 0; why: the native router cancellation canary requires idle without completion credit test consumes this named precondition before exercising the behavior. + state = {"active": 0, "cancellations": 0, "terminal": 0} + # What: act by calling threading.Event and capture cancelled; why: the native router cancellation canary requires idle without completion credit test asserts the response, state, or failure produced by this call. + cancelled = threading.Event() + + # What: define Handler as the owner of log_message and _json and do_GET and do_POST; why: daemon callers use this class boundary so those methods share one handler state invariant. + class Handler(BaseHTTPRequestHandler): + # What: define the log_message test helper around captured fixture state; why: the native router cancellation canary requires idle without completion credit scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def log_message(self, *args): + # What: ignore the anticipated exception handled by this branch; why: log_message continues its retry or cleanup path instead of re-raising that transient failure. + pass + + # What: define the _json test helper around body; why: the native router cancellation canary requires idle without completion credit scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def _json(self, body): + # What: act by calling operation.encode and capture raw; why: the native router cancellation canary requires idle without completion credit test asserts the response, state, or failure produced by this call. + raw = json.dumps(body).encode() + # What: act by calling self.send_response with 200; why: the native router cancellation canary requires idle without completion credit scenario observes the self.send_response return value during self send header content type application json. + self.send_response(200) + # What: arrange the exact self send header content type application json fixture fragment; why: the native router cancellation canary requires idle without completion credit scenario feeds this byte-preserved fragment through self.send_header("Content-Type", "application/json") before asserting its protoco. + self.send_header("Content-Type", "application/json") + # What: arrange the exact self send header content length str len raw fixture fragment; why: the native router cancellation canary requires idle without completion credit scenario feeds this byte-preserved fragment through self.send_header("Content-Length", str(len(raw))) before asserting its protocol or p. + self.send_header("Content-Length", str(len(raw))) + # What: act by calling self.end_headers with the declared inputs; why: the native router cancellation canary requires idle without completion credit scenario observes the self.end_headers return value during self wfile write raw. + self.end_headers() + # What: act by calling self.wfile.write with raw; why: the native router cancellation canary requires idle without completion credit scenario observes the self.wfile.write return value during the enclosing return. + self.wfile.write(raw) + + # What: define the do_GET test helper around captured fixture state; why: the native router cancellation canary requires idle without completion credit scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def do_GET(self): + # What: assert that self path equals router status; why: this assertion protects the native router cancellation canary requires idle without completion credit regression after the test's arranged inputs and exercised call. + assert self.path == "/router/status" + # What: act by calling self._json with state and active requests and cancellations and terminal streams and active; why: the native router cancellation canary requires idle without completion credit scenario observes the self._json return value during active requests state active. + self._json({ + # What: arrange the active requests field as state and active; why: Handler.do_GET carries active requests into "activeRequests": state["active"]. + "activeRequests": state["active"], + # What: arrange the cancellations field as state and cancellations; why: Handler.do_GET carries cancellations into "cancellations": state["cancellations"]. + "cancellations": state["cancellations"], + # What: arrange the terminal streams field as state and terminal; why: Handler.do_GET carries terminal streams into "terminalStreams": state["terminal"]. + "terminalStreams": state["terminal"], + # What: arrange the self._json call with state; why: Handler.do_GET groups the supplied clauses as one self._json call before its value is consumed. + }) + + # What: define the do_POST test helper around captured fixture state; why: the native router cancellation canary requires idle without completion credit scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def do_POST(self): + # What: arrange the exact self rfile read int self headers get content length fixture fragment; why: the native router cancellation canary requires idle without completion credit scenario feeds this byte-preserved fragment through self.rfile.read(int(self.headers.get("Content-Length", "0"))) before asserti. + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + # What: act on path before state; why: the native router cancellation canary requires idle without completion credit scenario admits state only for this predicate and excludes the opposite state. + if self.path == "/v1/chat/completions": + # What: arrange state entry as 1; why: the native router cancellation canary requires idle without completion credit test consumes this named precondition before exercising the behavior. + state["active"] = 1 + # What: act by calling self.send_response with 200; why: the native router cancellation canary requires idle without completion credit scenario observes the self.send_response return value during self send header content type text event stream. + self.send_response(200) + # What: arrange the exact self send header content type text event stream fixture fragment; why: the native router cancellation canary requires idle without completion credit scenario feeds this byte-preserved fragment through self.send_header("Content-Type", "text/event-stream") before asserting its p. + self.send_header("Content-Type", "text/event-stream") + # What: act by calling self.end_headers with the declared inputs; why: the native router cancellation canary requires idle without completion credit scenario observes the self.end_headers return value during self wfile write b data choices delta content. + self.end_headers() + # What: act by calling self.wfile.write with the named fixture input; why: the native router cancellation canary requires idle without completion credit scenario observes the self.wfile.write return value during self wfile flush. + self.wfile.write(b'data: {"choices":[{"delta":{"content":"1"}}]}\n\n') + # What: act by calling self.wfile.flush with the declared inputs; why: the native router cancellation canary requires idle without completion credit scenario observes the self.wfile.flush return value during cancelled wait. + self.wfile.flush() + # What: act by calling cancelled.wait with 3; why: the native router cancellation canary requires idle without completion credit scenario observes the cancelled.wait return value during state active. + cancelled.wait(3) + # What: arrange state entry as 0; why: the native router cancellation canary requires idle without completion credit test consumes this named precondition before exercising the behavior. + state["active"] = 0 + # What: return no value from the do_POST test helper; why: the native router cancellation canary requires idle without completion credit scenario uses this helper result in its subsequent act or assertion. + return + # What: assert that self path equals router requests native qualification cancel cancel; why: this assertion protects the native router cancellation canary requires idle without completion credit regression after the test's arranged inputs and exercised call. + assert self.path == "/router/requests/native-qualification-cancel/cancel" + # What: arrange state entry from 1; why: the native router cancellation canary requires idle without completion credit scenario uses state entry during the enclosing return or state update before checking the protected result. + state["cancellations"] += 1 + # What: act by calling cancelled.set with the declared inputs; why: the native router cancellation canary requires idle without completion credit scenario observes the cancelled.set return value during self json cancelled id native qualification cancel. + cancelled.set() + # What: arrange the cancelled field as true; why: Handler.do_POST carries cancelled into self._json({"cancelled": True, "id": "native-qualification-cancel"}). + self._json({"cancelled": True, "id": "native-qualification-cancel"}) + + # What: act by calling ThreadingHTTPServer and capture server; why: the native router cancellation canary requires idle without completion credit test asserts the response, state, or failure produced by this call. + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + # What: act by calling threading.Thread and capture worker; why: the native router cancellation canary requires idle without completion credit test asserts the response, state, or failure produced by this call. + worker = threading.Thread(target=server.serve_forever, daemon=True) + # What: act by calling worker.start with the declared inputs; why: the native router cancellation canary requires idle without completion credit scenario observes the worker.start return value during try. + worker.start() + # What: establish the handler boundary for the protected operation; why: test_native_router_cancellation_canary_requires_idle_without_completion_credit routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: act by calling native_router_qualifier.cancellation_canary and capture raw and observation; why: the native router cancellation canary requires idle without completion credit test asserts the response, state, or failure produced by this call. + raw, observation = native_router_qualifier.cancellation_canary( + # What: arrange seconds to native_router_qualifier.cancellation_canary; why: the native router cancellation canary requires idle without completion credit scenario binds this 3 value to native_router_qualifier.cancellation_canary's seconds input. + f"http://127.0.0.1:{server.server_port}", "model-a", seconds=3 + # What: arrange the native_router_qualifier.cancellation_canary call with seconds; why: test_native_router_cancellation_canary_requires_idle_without_completion_credit groups the supplied clauses as one native_router_qualifier.cancellation_canary call before its value is consumed. + ) + # What: assert that b content 1 is present in raw; why: this assertion protects the native router cancellation canary requires idle without completion credit regression after the test's arranged inputs and exercised call. + assert b'"content":"1"' in raw + # What: assert that b data done is absent from raw; why: this assertion protects the native router cancellation canary requires idle without completion credit regression after the test's arranged inputs and exercised call. + assert b"data: [DONE]" not in raw + # What: assert that observation passed is true; why: this assertion protects the native router cancellation canary requires idle without completion credit regression after the test's arranged inputs and exercised call. + assert observation["passed"] is True + # What: assert that observation cancellation incremented is true; why: this assertion protects the native router cancellation canary requires idle without completion credit regression after the test's arranged inputs and exercised call. + assert observation["cancellationIncremented"] is True + # What: assert that observation normal completion credited is false; why: this assertion protects the native router cancellation canary requires idle without completion credit regression after the test's arranged inputs and exercised call. + assert observation["normalCompletionCredited"] is False + # What: assert that state equals active 0 cancellations 1 terminal 0; why: this assertion protects the native router cancellation canary requires idle without completion credit regression after the test's arranged inputs and exercised call. + assert state == {"active": 0, "cancellations": 1, "terminal": 0} + # What: run server shutdown on every exit path; why: test_native_router_cancellation_canary_requires_idle_without_completion_credit performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: act by calling server.shutdown with the declared inputs; why: the native router cancellation canary requires idle without completion credit scenario observes the server.shutdown return value during server server close. + server.shutdown() + # What: act by calling server.server_close with the declared inputs; why: the native router cancellation canary requires idle without completion credit scenario observes the server.server_close return value during worker join. + server.server_close() + # What: act by calling worker.join with 3; why: the native router cancellation canary requires idle without completion credit scenario observes the worker.join return value during the enclosing return. + worker.join(3) + + +# What: define the test_native_router_benchmark_keeps_prometheus_capture_private_bytes test around native router qualifier and monkeypatch; why: this test groups the arrange, act, and assertions that protect the native router benchmark keeps prometheus capture private bytes outcome. +def test_native_router_benchmark_keeps_prometheus_capture_private_bytes(native_router_qualifier, monkeypatch): + # What: act by calling io.BytesIO and capture stream; why: the native router benchmark keeps prometheus capture private bytes test asserts the response, state, or failure produced by this call. + stream = io.BytesIO(b"freetoken_swap_admissions_total 3\n") + # What: arrange the exact monkeypatch setattr native router qualifier urllib request urlopen lambda a k fixture f; why: the native router benchmark keeps prometheus capture private bytes scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen", l bef. + monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen", lambda *a, **k: stream) + + # What: assert that native router qualifier request bytes http test metrics equals b freetoken swap admissions total 3 n; why: this assertion protects the native router benchmark keeps prometheus capture private bytes regression after the test's arranged inputs and exercised call. + assert native_router_qualifier.request_bytes("http://test/metrics") == b"freetoken_swap_admissions_total 3\n" + # What: assert that stream closed; why: this assertion protects the native router benchmark keeps prometheus capture private bytes regression after the test's arranged inputs and exercised call. + assert stream.closed + + +# What: define the test_native_router_credentials_are_scoped_to_the_temporary_origin test around native router qualifier and monkeypatch; why: this test groups the arrange, act, and assertions that protect the native router credentials are scoped to the temporary origin outcome. +def test_native_router_credentials_are_scoped_to_the_temporary_origin( + # What: arrange native router qualifier monkeypatch for the scenario; why: test native router credentials are scoped to the temporary origin requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch +# What: arrange the grouped source fragment for the scenario; why: test native router credentials are scoped to the temporary origin requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange requests as the fixture input; why: the native router credentials are scoped to the temporary origin test consumes this named precondition before exercising the behavior. + requests = [] + + # What: define the urlopen test helper around request; why: the native router credentials are scoped to the temporary origin scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def urlopen(request, **kwargs): + # What: act by calling requests.append with request; why: the native router credentials are scoped to the temporary origin scenario observes the requests.append return value during return io bytes io b. + requests.append(request) + # What: return bytes io and io from the urlopen test helper; why: the native router credentials are scoped to the temporary origin scenario uses this helper result in its subsequent act or assertion. + return io.BytesIO(b'{}') + + # What: arrange the exact monkeypatch setattr native router qualifier urllib request urlopen urlopen fixture frag; why: the native router credentials are scoped to the temporary origin scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen", u befor. + monkeypatch.setattr(native_router_qualifier.urllib.request, "urlopen", urlopen) + # What: arrange the exact native router qualifier configure native auth http native test private key fixture frag; why: the native router credentials are scoped to the temporary origin scenario feeds this byte-preserved fragment through native_router_qualifier.configure_native_auth("http://native.test:1964", befor. + native_router_qualifier.configure_native_auth("http://native.test:1964", "private-key") + + # What: arrange the exact native router qualifier request json http native test router status fixture fragment; why: the native router credentials are scoped to the temporary origin scenario feeds this byte-preserved fragment through native_router_qualifier.request_json("http://native.test:1964/router/sta before a. + native_router_qualifier.request_json("http://native.test:1964/router/status") + # What: arrange the exact native router qualifier request json http protected test health fixture fragment; why: the native router credentials are scoped to the temporary origin scenario feeds this byte-preserved fragment through native_router_qualifier.request_json("http://protected.test:8000/health" before asser. + native_router_qualifier.request_json("http://protected.test:8000/health") + # What: arrange the exact native router qualifier request json http native test v1 models fixture fragment; why: the native router credentials are scoped to the temporary origin scenario feeds this byte-preserved fragment through native_router_qualifier.request_json("http://native.test:24567/v1/models before asser. + native_router_qualifier.request_json("http://native.test:24567/v1/models") + # What: submit one body-bearing management request; why: the helper must distinguish real JSON payloads from empty authenticated reads. + native_router_qualifier.request_json("http://native.test:1964/router/load", {"name": "model-a"}) + + # What: assert that requests 0 get header authorization equals bearer private key; why: this assertion protects the native router credentials are scoped to the temporary origin regression after the test's arranged inputs and exercised call. + assert requests[0].get_header("Authorization") == "Bearer private-key" + # What: assert that requests 1 get header authorization is group delimiter; why: this assertion protects the native router credentials are scoped to the temporary origin regression after the test's arranged inputs and exercised call. + assert requests[1].get_header("Authorization") is None + # What: assert that requests 2 get header authorization is group delimiter; why: this assertion protects the native router credentials are scoped to the temporary origin regression after the test's arranged inputs and exercised call. + assert requests[2].get_header("Authorization") is None + # What: require no content type on the authenticated empty GET; why: claiming an absent body is JSON causes the live router to reject otherwise valid upstream reads. + assert requests[0].get_header("Content-type") is None + # What: require the JSON content type on the body-bearing request; why: management writes still need explicit payload metadata for strict validation. + assert requests[3].get_header("Content-type") == "application/json" + + +# What: define the test_native_router_control_plane_canary_requires_auth_and_captures_evidence test around native router qualifier and tmp path; why: this test groups the arrange, act, and assertions that protect the native router control plane canary requires auth and captures evidence outcome. +def test_native_router_control_plane_canary_requires_auth_and_captures_evidence( + # What: arrange native router qualifier tmp path for the scenario; why: test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, tmp_path +# What: arrange the grouped source fragment for the scenario; why: test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange authorized paths as the fixture input; why: the native router control plane canary requires auth and captures evidence test consumes this named precondition before exercising the behavior. + authorized_paths = [] + + # What: define Handler as the owner of log_message and _send and do_GET; why: daemon callers use this class boundary so those methods share one handler state invariant. + class Handler(BaseHTTPRequestHandler): + # What: define the log_message test helper around captured fixture state; why: the native router control plane canary requires auth and captures evidence scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def log_message(self, *args): + # What: ignore the anticipated exception handled by this branch; why: log_message continues its retry or cleanup path instead of re-raising that transient failure. + pass + + # What: define the _send test helper around body and content type; why: the native router control plane canary requires auth and captures evidence scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def _send(self, body, *, content_type="application/json"): + # What: act by calling self.send_response with 200; why: the native router control plane canary requires auth and captures evidence scenario observes the self.send_response return value during self send header content type content type. + self.send_response(200) + # What: arrange the exact self send header content type content type fixture fragment; why: the native router control plane canary requires auth and captures evidence scenario feeds this byte-preserved fragment through self.send_header("Content-Type", content_type) before asserting its protocol or parser r. + self.send_header("Content-Type", content_type) + # What: arrange the exact self send header content length str len body fixture fragment; why: the native router control plane canary requires auth and captures evidence scenario feeds this byte-preserved fragment through self.send_header("Content-Length", str(len(body))) before asserting its protocol or pa. + self.send_header("Content-Length", str(len(body))) + # What: act by calling self.end_headers with the declared inputs; why: the native router control plane canary requires auth and captures evidence scenario observes the self.end_headers return value during self wfile write body. + self.end_headers() + # What: act by calling self.wfile.write with body; why: the native router control plane canary requires auth and captures evidence scenario observes the self.wfile.write return value during the enclosing return. + self.wfile.write(body) + + # What: define the do_GET test helper around captured fixture state; why: the native router control plane canary requires auth and captures evidence scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def do_GET(self): + # What: act by calling operation.decode and capture accepted; why: the native router control plane canary requires auth and captures evidence test asserts the response, state, or failure produced by this call. + accepted = { + # What: arrange the bearer private key portion of accepted; why: the native router control plane canary requires auth and captures evidence scenario uses this clause to evaluate accepted as one grouped value. + "Bearer private-key", + # What: act by calling operation.decode with the declared inputs; why: the native router control plane canary requires auth and captures evidence scenario observes the operation.decode return value while evaluating "Basic " + base64.b64encode(b"operator:private-key").decode(). + "Basic " + base64.b64encode(b"operator:private-key").decode(), + # What: arrange the accepted collection with bearer and private key and decode and b64encode and base64 and basic; why: Handler.do_GET groups the supplied clauses as one accepted collection before its value is consumed. + } + # What: act on accepted and get and headers before send response; why: the native router control plane canary requires auth and captures evidence scenario admits send response only for this predicate and excludes the opposite state. + if ( + # What: act by calling self.headers.get with authorization; why: the native router control plane canary requires auth and captures evidence scenario observes the self.headers.get return value during and self headers get x api key private key. + self.headers.get("Authorization") not in accepted + # What: act by calling self.headers.get with x api key; why: the native router control plane canary requires auth and captures evidence scenario observes the self.headers.get return value while evaluating and self.headers.get("X-Api-Key") != "private-key". + and self.headers.get("X-Api-Key") != "private-key" + # What: arrange the enclosing predicate with if self headers get authorization not in accepted and self headers get x api key; why: Handler.do_GET groups the supplied clauses as one Handler.do_GET expression before its value is consumed. + ): + # What: act by calling self.send_response with 401; why: the native router control plane canary requires auth and captures evidence scenario observes the self.send_response return value during self send header content length. + self.send_response(401) + # What: arrange the exact self send header content length fixture fragment; why: the native router control plane canary requires auth and captures evidence scenario feeds this byte-preserved fragment through self.send_header("Content-Length", "0") before asserting its protocol or parser result. + self.send_header("Content-Length", "0") + # What: act by calling self.end_headers with the declared inputs; why: the native router control plane canary requires auth and captures evidence scenario observes the self.end_headers return value during return. + self.end_headers() + # What: return no value from the do_GET test helper; why: the native router control plane canary requires auth and captures evidence scenario uses this helper result in its subsequent act or assertion. + return + # What: act by calling authorized_paths.append with path; why: the native router control plane canary requires auth and captures evidence scenario observes the authorized_paths.append return value during if self path router status. + authorized_paths.append(self.path) + # What: arrange if self path == router status for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. + if self.path == "/router/status": + # What: arrange body as active profile and model a; why: the native router control plane canary requires auth and captures evidence test consumes this named precondition before exercising the behavior. + body = {"activeProfile": "model-a"} + # What: act on path before body and path; why: the native router control plane canary requires auth and captures evidence scenario admits body and path only for this predicate and excludes the opposite state. + elif self.path in ("/v1/models", "/models"): + # What: arrange body as path and object and data and list and id; why: the native router control plane canary requires auth and captures evidence test consumes this named precondition before exercising the behavior. + body = { + # What: arrange the object field as list; why: Handler.do_GET carries object through body into body running requests 0. + "object": "list", + # What: arrange the data field as path and id and created and name and meta; why: Handler.do_GET carries data through body into body running requests 0. + "data": [ + # What: arrange the body mapping with id and created and name and meta; why: Handler.do_GET groups the supplied clauses as one body mapping before its value. + { + # What: arrange the id field as model a; why: Handler.do_GET carries id through body into body running requests 0. + "id": "model-a", + # What: arrange the created field as path and 10 and 11 and v1 and models; why: Handler.do_GET carries created through body into body running requests 0. + "created": 10 if self.path == "/v1/models" else 11, + # What: arrange the name field as qualification and model and a; why: Handler.do_GET carries name through body into body running requests 0. + "name": "Qualification model A", + # What: arrange the freetoken field as aliases and tier and type and qualification and model; why: Handler.do_GET carries freetoken through body into body running requests 0. + "meta": {"freetoken": { + # What: arrange the aliases compat model a portion of body; why: the native router control plane canary requires auth and captures evidence scenario uses this clause to evaluate body as one grouped value. + "aliases": ["compat/model-a"], + # What: arrange the tier qualification type model portion of body; why: the native router control plane canary requires auth and captures evidence scenario uses this clause to evaluate body as one grouped value. + "tier": "qualification", "type": "model", + # What: arrange the body mapping with freetoken; why: Handler.do_GET groups the supplied clauses as one body mapping before its value is consumed. + }}, + # What: arrange the body mapping with id and created and name and meta; why: Handler.do_GET groups the supplied clauses as one body mapping before its. + }, + # What: arrange the id field as model b; why: Handler.do_GET carries id through body into body running requests 0. + {"id": "model-b", "created": 10 if self.path == "/v1/models" else 11}, + # What: arrange the body mapping with id and created and name and meta; why: Handler.do_GET groups the supplied clauses as one body mapping before its value. + { + # What: arrange the id field as compat and model a; why: Handler.do_GET carries id through body into body running requests 0. + "id": "compat/model-a", + # What: arrange the created field as path and 10 and 11 and v1 and models; why: Handler.do_GET carries created through body into body running requests 0. + "created": 10 if self.path == "/v1/models" else 11, + # What: arrange the name field as qualification and model and a; why: Handler.do_GET carries name through body into body running requests 0. + "name": "Qualification model A", + # What: arrange the freetoken field as model id and tier and type and model a and qualification; why: Handler.do_GET carries freetoken through body into body running requests 0. + "meta": {"freetoken": { + # What: arrange the model id model a tier qualification portion of body; why: the native router control plane canary requires auth and captures evidence scenario uses this clause to evaluate body as one grouped value. + "modelID": "model-a", "tier": "qualification", + # What: arrange the type alias portion of body; why: the native router control plane canary requires auth and captures evidence scenario uses this clause to evaluate body as one grouped value. + "type": "alias", + # What: arrange the body mapping with freetoken; why: Handler.do_GET groups the supplied clauses as one body mapping before its value is consumed. + }}, + # What: arrange the body mapping with id and created and name and meta; why: Handler.do_GET groups the supplied clauses as one body mapping before its. + }, + # What: arrange the id field as preferred model; why: Handler.do_GET carries id through body into body running requests 0. + {"id": "preferred-model", "created": 10 if self.path == "/v1/models" else 11}, + # What: arrange the grouped source fragment for the scenario; why: this test requires this concrete input or helper state before exercising the behavior under test. + ], + # What: arrange the body mapping with object and data; why: Handler.do_GET groups the supplied clauses as one body mapping before its value is consumed. + } + # What: arrange elif self path == upstream compat model a v1 stats for the scenario; why: test native router control plane canary requires requires this concrete input or helper state before exercising the behavior under test. + elif self.path == "/upstream/compat/model-a/v1/stats": + # What: arrange body as running requests and 0; why: the native router control plane canary requires auth and captures evidence test consumes this named precondition before exercising the behavior. + body = {"running_requests": 0} + # What: arrange elif self path == router models for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. + elif self.path == "/router/models": + # What: arrange body as data and name and resident and check endpoint and use model name; why: the native router control plane canary requires auth and captures evidence test consumes this named precondition before exercising the behavior. + body = {"data": [ + # What: arrange the body mapping with name and resident and check endpoint and use model name; why: Handler.do_GET groups the supplied clauses as one body mapping. + { + # What: arrange the name field as model a; why: Handler.do_GET carries name through body into body. + "name": "model-a", "resident": True, + # What: arrange the check endpoint field as ready; why: Handler.do_GET carries check endpoint through body into body. + "checkEndpoint": "/ready", + # What: arrange the use model name field as model a; why: Handler.do_GET carries use model name through body into body. + "useModelName": "model-a", + # What: arrange the upstream timeout s field as 659; why: Handler.do_GET carries upstream timeout s through body into body. + "upstreamTimeoutS": 659, + # What: arrange the display name field as qualification and model and a; why: Handler.do_GET carries display name through body into body. + "displayName": "Qualification model A", + # What: arrange the tier field as qualification; why: Handler.do_GET carries tier through body into body. + "metadata": {"tier": "qualification", "type": "operator"}, + # What: arrange the body mapping with name and resident and check endpoint and use model name; why: Handler.do_GET groups the supplied clauses as one body. + }, + # What: arrange the body mapping with name and resident and check endpoint; why: Handler.do_GET groups the supplied clauses as one body mapping before its value. + { + # What: arrange the name field as model b; why: Handler.do_GET carries name through body into body. + "name": "model-b", "resident": False, + # What: arrange the check endpoint field as ready; why: Handler.do_GET carries check endpoint through body into body. + "checkEndpoint": "/ready", + # What: arrange the body mapping with name and resident and check endpoint; why: Handler.do_GET groups the supplied clauses as one body mapping before its. + }, + # What: arrange the body mapping with data; why: Handler.do_GET groups the supplied clauses as one body mapping before its value is consumed. + ]} + # What: arrange elif self path == router profiles for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. + elif self.path == "/router/profiles": + # What: arrange body as active profile and active routing profile and routing profiles and data and model a; why: the native router control plane canary requires auth and captures evidence test consumes this named precondition before exercising the behavior. + body = { + # What: arrange the active profile field as model a; why: Handler.do_GET carries active profile through body into body. + "activeProfile": "model-a", + # What: arrange the active routing profile field as the fixture input; why: Handler.do_GET carries active routing profile through body into body. + "activeRoutingProfile": None, + # What: arrange the routing profiles portion of body; why: the native router control plane canary requires auth and captures evidence scenario uses this clause to evaluate body as one grouped value. + "routingProfiles": [{ + # What: arrange the name coding pins portion of body; why: the native router control plane canary requires auth and captures evidence scenario uses this clause to evaluate body as one grouped value. + "name": "coding", "pins": { + # What: arrange the disabled model field as the fixture input; why: Handler.do_GET carries disabled model through body into body. + "disabled-model": None, + # What: arrange the profile model field as preferred model; why: Handler.do_GET carries profile model through body into body. + "profile-model": "preferred-model", + # What: arrange the body mapping with disabled model and profile model; why: Handler.do_GET groups the supplied clauses as one body mapping before its value is consumed. + }, + # What: arrange the body collection with name and pins and coding and disabled model and profile model; why: Handler.do_GET groups the supplied clauses as one body collection before its value is consumed. + }], + # What: arrange the name field as model a; why: Handler.do_GET carries name through body into body. + "data": [{"name": "model-a"}, {"name": "model-b"}], + # What: arrange the body mapping with active profile and active routing profile and routing profiles and data; why: Handler.do_GET groups the supplied clauses as one body mapping before its value is consumed. + } + # What: arrange elif self path == api performance for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. + elif self.path == "/api/performance": + # What: arrange body as enabled and sys stats and gpu stats and true and timestamp; why: the native router control plane canary requires auth and captures evidence test consumes this named precondition before exercising the behavior. + body = { + # What: arrange the enabled field as true; why: Handler.do_GET carries enabled through body into self send json dumps body encode. + "enabled": True, + # What: arrange the sys stats portion of body; why: the native router control plane canary requires auth and captures evidence scenario uses this clause to evaluate body as one grouped value. + "sys_stats": [{ + # What: arrange the timestamp field as t00 and z; why: Handler.do_GET carries timestamp through body into self send json dumps body encode. + "timestamp": "2026-09-15T00:00:00Z", + # What: arrange the scope field as engine process tree; why: Handler.do_GET carries scope through body into self send json dumps body encode. + "scope": "engine-process-tree", + # What: arrange the ram bytes field as 1024; why: Handler.do_GET carries ram bytes through body into self send json dumps body encode. + "ram_bytes": 1024, + # What: arrange the vram bytes field as 2048; why: Handler.do_GET carries vram bytes through body into self send json dumps body encode. + "vram_bytes": 2048, + # What: arrange the ram available field as true; why: Handler.do_GET carries ram available through body into self send json dumps body encode. + "ram_available": True, + # What: arrange the vram available field as true; why: Handler.do_GET carries vram available through body into self send json dumps body encode. + "vram_available": True, + # What: arrange the ram source field as proc smaps rollup pss; why: Handler.do_GET carries ram source through body into self send json dumps body encode. + "ram_source": "proc-smaps-rollup-pss", + # What: arrange the vram source field as amd smi; why: Handler.do_GET carries vram source through body into self send json dumps body encode. + "vram_source": "amd-smi", + # What: arrange the body collection with timestamp and scope and ram bytes and vram bytes and ram available; why: Handler.do_GET groups the supplied clauses as one body collection before its value is consumed. + }], + # What: arrange the gpu stats field as the fixture input; why: Handler.do_GET carries gpu stats through body into self send json dumps body encode. + "gpu_stats": [], + # What: arrange the body mapping with enabled and sys stats and gpu stats; why: Handler.do_GET groups the supplied clauses as one body mapping before its value is consumed. + } + # What: arrange elif self path == metrics for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. + elif self.path == "/metrics": + # What: act by calling self._send with the named fixture input; why: the native router control plane canary requires auth and captures evidence scenario observes the self._send return value during b freetoken swap admissions total n. + self._send( + # What: arrange the b freetoken swap admissions total n portion of the enclosing predicate; why: this clause remains in the native router control plane canary requires auth and captures evidence scenario\'s enclosing expression so its grouping and evaluation order stay intact. + b"freetoken_swap_admissions_total 1\n", + # What: arrange the exact content type text plain version fixture fragment; why: the native router control plane canary requires auth and captures evidence scenario feeds this byte-preserved fragment through content_type="text/plain; version=0.0.4" before asserting its protocol or parser result. + content_type="text/plain; version=0.0.4", + # What: arrange the self._send call with content type; why: Handler.do_GET groups the supplied clauses as one self._send call before its value is consumed. + ) + # What: return no value from the do_GET test helper; why: the native router control plane canary requires auth and captures evidence scenario uses this helper result in its subsequent act or assertion. + return + # What: arrange elif self path == router logs since 0 for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. + elif self.path == "/router/logs?since=0": + # What: act by calling self._send with the named fixture input; why: the native router control plane canary requires auth and captures evidence scenario observes the self._send return value during b event router ndata event startup. + self._send( + # What: arrange the b event router ndata event startup portion of the enclosing predicate; why: this clause remains in the native router control plane canary requires auth and captures evidence scenario\'s enclosing expression so its grouping and evaluation order stay intact. + b'event: router\ndata: {"event":"startup"}\n\n' + # What: arrange the b event router ndata event management loaded portion of the enclosing predicate; why: this clause remains in the native router control plane canary requires auth and captures evidence scenario\'s enclosing expression so its grouping and evaluation order stay intact. + b'event: router\ndata: {"event":"management_loaded"}\n\n', + # What: arrange the exact content type text event stream fixture fragment; why: the native router control plane canary requires auth and captures evidence scenario feeds this byte-preserved fragment through content_type="text/event-stream" before asserting its protocol or parser result. + content_type="text/event-stream", + # What: arrange the self._send call with content type; why: Handler.do_GET groups the supplied clauses as one self._send call before its value is consumed. + ) + # What: return no value from the do_GET test helper; why: the native router control plane canary requires auth and captures evidence scenario uses this helper result in its subsequent act or assertion. + return + # What: select the remaining branch that performs self send error; why: do_GET covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: act by calling self.send_error with 404; why: the native router control plane canary requires auth and captures evidence scenario observes the self.send_error return value during return. + self.send_error(404) + # What: return no value from the do_GET test helper; why: the native router control plane canary requires auth and captures evidence scenario uses this helper result in its subsequent act or assertion. + return + # What: act by calling self._send with encode and dumps and body and json; why: the native router control plane canary requires auth and captures evidence scenario observes the self._send return value during the enclosing return. + self._send(json.dumps(body).encode()) + + # What: act by calling ThreadingHTTPServer and capture server; why: the native router control plane canary requires auth and captures evidence test asserts the response, state, or failure produced by this call. + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + # What: act by calling threading.Thread and capture worker; why: the native router control plane canary requires auth and captures evidence test asserts the response, state, or failure produced by this call. + worker = threading.Thread(target=server.serve_forever, daemon=True) + # What: act by calling worker.start with the declared inputs; why: the native router control plane canary requires auth and captures evidence scenario observes the worker.start return value during base f http server server port. + worker.start() + # What: arrange base as server port and server and http; why: the native router control plane canary requires auth and captures evidence test consumes this named precondition before exercising the behavior. + base = f"http://127.0.0.1:{server.server_port}" + # What: arrange the exact native router qualifier configure native auth base private key fixture fragment; why: the native router control plane canary requires auth and captures evidence scenario feeds this byte-preserved fragment through native_router_qualifier.configure_native_auth(base, "private-key") before as. + native_router_qualifier.configure_native_auth(base, "private-key") + # What: establish the handler boundary for the protected operation; why: test_native_router_control_plane_canary_requires_auth_and_captures_evidence routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: act by calling native_router_qualifier.control_plane_canary and capture observation; why: the native router control plane canary requires auth and captures evidence test asserts the response, state, or failure produced by this call. + observation = native_router_qualifier.control_plane_canary(base, tmp_path) + # What: run server shutdown on every exit path; why: test_native_router_control_plane_canary_requires_auth_and_captures_evidence performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: act by calling server.shutdown with the declared inputs; why: the native router control plane canary requires auth and captures evidence scenario observes the server.shutdown return value during server server close. + server.shutdown() + # What: act by calling server.server_close with the declared inputs; why: the native router control plane canary requires auth and captures evidence scenario observes the server.server_close return value during worker join. + server.server_close() + # What: act by calling worker.join with 3; why: the native router control plane canary requires auth and captures evidence scenario observes the worker.join return value during assert observation passed is. + worker.join(3) + + # What: assert that observation passed is true; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["passed"] is True + # What: assert that observation unauthenticated control rejected is true; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["unauthenticatedControlRejected"] is True + # What: assert that observation unauthenticated inference rejected is true; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["unauthenticatedInferenceRejected"] is True + # What: assert that observation resident profile equals model a; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["residentProfile"] == "model-a" + # What: assert that observation model list alias verified is true; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["modelListAliasVerified"] is True + # What: assert that observation selector listed is true; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["selectorListed"] is True + # What: assert that observation routing profile listed is true; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["routingProfileListed"] is True + # What: assert that observation configured readiness target verified is true; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["configuredReadinessTargetVerified"] is True + # What: assert that observation configured upstream model name verified is true; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["configuredUpstreamModelNameVerified"] is True + # What: assert that observation configured upstream timeout verified is true; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["configuredUpstreamTimeoutVerified"] is True + # What: assert that observation configured model metadata verified is true; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["configuredModelMetadataVerified"] is True + # What: assert that observation namespaced upstream verified is true; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["namespacedUpstreamVerified"] is True + # What: assert that observation api key forms verified equals bearer basic x api key; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["apiKeyFormsVerified"] == ["bearer", "basic", "x-api-key"] + # What: assert that observation periodic performance available is true; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert observation["periodicPerformanceAvailable"] is True + # What: assert the expected authorized paths == outcome; why: test swap qualification test native router control plane canary requires auth and captures evidence protects its regression by requiring this observable result after the exercised behavior. + assert authorized_paths == [ + # What: arrange router status router status for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. + "/router/status", "/router/status", + # What: arrange v1 models models upstream compat model a v1 stats for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. + "/v1/models", "/models", "/upstream/compat/model-a/v1/stats", + # What: arrange router models router profiles api performance metrics for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. + "/router/models", "/router/profiles", "/api/performance", "/metrics", + # What: arrange router logs since 0 for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. + "/router/logs?since=0", + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete. + ] + # What: assert that b management loaded is present in tmp path control router log sse read bytes; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert b"management_loaded" in (tmp_path / "control-router-log.sse").read_bytes() + # What: assert the expected tmp path control metrics prom read bytes startswith outcome; why: test swap qualification test native router control plane canary requires auth and captures evidence protects its regression by requiring this observable result after the exercised behavior. + assert (tmp_path / "control-metrics.prom").read_bytes().startswith( + # What: arrange b freetoken swap admissions total for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete input or helper state before exercising the behavior under test. + b"freetoken_swap_admissions_total" + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router control plane canary requires auth and captures evidence requires this concrete input. + ) + # What: assert that tmp path control auth basic json is file; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert (tmp_path / "control-auth-basic.json").is_file() + # What: assert that tmp path control auth x api key json is file; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert (tmp_path / "control-auth-x-api-key.json").is_file() + # What: assert that tmp path control performance json is file; why: this assertion protects the native router control plane canary requires auth and captures evidence regression after the test's arranged inputs and exercised call. + assert (tmp_path / "control-performance.json").is_file() + + +# What: define the test_native_periodic_performance_gate_rejects_unavailable_or_identifying_rows test around native router qualifier; why: this test groups the arrange, act, and assertions that protect the native periodic performance gate rejects unavailable or identifying rows outcome. +def test_native_periodic_performance_gate_rejects_unavailable_or_identifying_rows( + # What: arrange native router qualifier for the scenario; why: test native periodic performance gate rejects unavailable or identifying rows requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, +# What: arrange the grouped source fragment for the scenario; why: test native periodic performance gate rejects unavailable or identifying rows requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange valid as enabled and sys stats and gpu stats and true and timestamp; why: the native periodic performance gate rejects unavailable or identifying rows test consumes this named precondition before exercising the behavior. + valid = { + # What: arrange the enabled field as true; why: test_native_periodic_performance_gate_rejects_unavailable_or_identifying_rows carries enabled through valid into assert native router qualifier valid periodic performance valid. + "enabled": True, + # What: arrange the sys stats portion of valid; why: the native periodic performance gate rejects unavailable or identifying rows scenario uses this clause to evaluate valid as one grouped value. + "sys_stats": [{ + # What: arrange the timestamp field as t00 and z; why: test_native_periodic_performance_gate_rejects_unavailable_or_identifying_rows carries timestamp through valid into assert native router qualifier valid periodic performance valid. + "timestamp": "2026-09-15T00:00:00Z", "scope": "engine-process-tree", + # What: arrange the ram bytes field as 1; why: test_native_periodic_performance_gate_rejects_unavailable_or_identifying_rows carries ram bytes through valid into assert native router qualifier valid periodic performance valid. + "ram_bytes": 1, "vram_bytes": 2, + # What: arrange the ram available field as true; why: test_native_periodic_performance_gate_rejects_unavailable_or_identifying_rows carries ram available through valid into assert native router qualifier valid periodic performance valid. + "ram_available": True, "vram_available": True, + # What: arrange the ram source field as pss; why: test_native_periodic_performance_gate_rejects_unavailable_or_identifying_rows carries ram source through valid into assert native router qualifier valid periodic performance valid. + "ram_source": "pss", "vram_source": "amd-smi", + # What: arrange the valid collection with timestamp and scope and ram bytes and vram bytes and ram available; why: test_native_periodic_performance_gate_rejects_unavailable_or_identifying_rows groups the supplied clauses as one valid collection before its value is consumed. + }], + # What: arrange the gpu stats field as the fixture input; why: test_native_periodic_performance_gate_rejects_unavailable_or_identifying_rows carries gpu stats through valid into assert native router qualifier valid periodic performance valid. + "gpu_stats": [], + # What: arrange the valid mapping with enabled and sys stats and gpu stats; why: test_native_periodic_performance_gate_rejects_unavailable_or_identifying_rows groups the supplied clauses as one valid mapping before its value is consumed. + } + # What: assert that native router qualifier valid periodic performance valid; why: this assertion protects the native periodic performance gate rejects unavailable or identifying rows regression after the test's arranged inputs and exercised call. + assert native_router_qualifier.valid_periodic_performance(valid) + # What: act across the computed value to perform candidate and loads and json and dumps and valid; why: the native periodic performance gate rejects unavailable or identifying rows scenario repeats the body only while or for the loop header admits an iteration. + for key, value in ( + # What: arrange the ram available vram bytes pids portion of the enclosing predicate; why: this clause remains in the native periodic performance gate rejects unavailable or identifying rows scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("ram_available", False), ("vram_bytes", 0), ("pids", [123]), + # What: arrange the model private path private portion of the enclosing predicate; why: this clause remains in the native periodic performance gate rejects unavailable or identifying rows scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("model", "private"), ("path", "/private"), + # What: arrange the grouped source fragment for the scenario; why: test native periodic performance gate rejects unavailable or identifying rows requires this concrete input or helper state before exercising the behavior under test. + ): + # What: act by calling json.loads and capture candidate; why: the native periodic performance gate rejects unavailable or identifying rows test asserts the response, state, or failure produced by this call. + candidate = json.loads(json.dumps(valid)) + # What: arrange candidate entry entry entry as value; why: the native periodic performance gate rejects unavailable or identifying rows test consumes this named precondition before exercising the behavior. + candidate["sys_stats"][-1][key] = value + # What: assert that native router qualifier valid periodic performance candidate is false; why: this assertion protects the native periodic performance gate rejects unavailable or identifying rows regression after the test's arranged inputs and exercised call. + assert not native_router_qualifier.valid_periodic_performance(candidate) + + +# What: define the test_native_router_benchmark_validates_warm_and_swap_activation_labels test around native router qualifier; why: this test groups the arrange, act, and assertions that protect the native router benchmark validates warm and swap activation labels outcome. +def test_native_router_benchmark_validates_warm_and_swap_activation_labels(native_router_qualifier): + # What: arrange status a as active profile and active requests and activations and model a and 0; why: the native router benchmark validates warm and swap activation labels test consumes this named precondition before exercising the behavior. + status_a = {"activeProfile": "model-a", "activeRequests": 0, "activations": 1} + # What: arrange status b as active profile and active requests and activations and model b and 0; why: the native router benchmark validates warm and swap activation labels test consumes this named precondition before exercising the behavior. + status_b = {"activeProfile": "model-b", "activeRequests": 0, "activations": 2} + # What: assert the expected native router qualifier validate routed trial outcome; why: test swap qualification test native router benchmark validates warm and swap activation labels protects its regression by requiring this observable result after the exercised behavior. + assert native_router_qualifier.validate_routed_trial( + # What: arrange status a alias model a prior activations 1 expected delta 0 for the scenario; why: test swap qualification test native router benchmark validates warm and swap activation labels requires this concrete input or helper state before exercising the behavior under test. + status_a, alias="model-a", prior_activations=1, expected_delta=0 + # What: arrange == 1 for the scenario; why: test swap qualification test native router benchmark validates warm and swap activation labels requires this concrete input or helper state before exercising the behavior under test. + ) == 1 + # What: assert the expected native router qualifier validate routed trial outcome; why: test swap qualification test native router benchmark validates warm and swap activation labels protects its regression by requiring this observable result after the exercised behavior. + assert native_router_qualifier.validate_routed_trial( + # What: arrange status b alias model b prior activations 1 expected delta 1 for the scenario; why: test swap qualification test native router benchmark validates warm and swap activation labels requires this concrete input or helper state before exercising the behavior under test. + status_b, alias="model-b", prior_activations=1, expected_delta=1 + # What: arrange == 2 for the scenario; why: test swap qualification test native router benchmark validates warm and swap activation labels requires this concrete input or helper state before exercising the behavior under test. + ) == 2 + + +# What: define the test_native_router_benchmark_proves_warm_selector_reuses_resident_target test around native router qualifier and monkeypatch and tmp path; why: this test groups the arrange, act, and assertions that protect the native router benchmark proves warm selector reuses resident target outcome. +def test_native_router_benchmark_proves_warm_selector_reuses_resident_target( + # What: arrange native router qualifier monkeypatch tmp path for the scenario; why: test native router benchmark proves warm selector reuses resident target requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch, tmp_path +# What: arrange the grouped source fragment for the scenario; why: test native router benchmark proves warm selector reuses resident target requires this concrete input or helper state before exercising the behavior under test. +): + # What: act by calling iter and capture statuses; why: the native router benchmark proves warm selector reuses resident target test asserts the response, state, or failure produced by this call. + statuses = iter(( + # What: arrange the active profile field as model a; why: test_native_router_benchmark_proves_warm_selector_reuses_resident_target carries active profile through statuses into lambda args kwargs b next statuses. + {"activeProfile": "model-a", "activeRequests": 0, "activations": 3}, + # What: arrange the active profile field as model a; why: test_native_router_benchmark_proves_warm_selector_reuses_resident_target carries active profile through statuses into lambda args kwargs b next statuses. + {"activeProfile": "model-a", "activeRequests": 0, "activations": 3}, + # What: arrange the iter call with ordered positional inputs; why: test_native_router_benchmark_proves_warm_selector_reuses_resident_target groups the supplied clauses as one iter call before its value is consumed. + )) + # What: act by calling monkeypatch.setattr with native router qualifier and request json and next and statuses; why: the native router benchmark proves warm selector reuses resident target scenario observes the monkeypatch.setattr return value during native router qualifier request json. + monkeypatch.setattr( + # What: arrange the exact native router qualifier request json fixture fragment; why: the native router benchmark proves warm selector reuses resident target scenario feeds this byte-preserved fragment through native_router_qualifier, "request_json" before asserting its protocol or parser result. + native_router_qualifier, "request_json", + # What: arrange the args input for test_native_router_benchmark_proves_warm_selector_reuses_resident_target; why: test_native_router_benchmark_proves_warm_selector_reuses_resident_target consumes args during signature binding, so callers must bind it with the other signature inputs. + lambda *args, **kwargs: (b"{}", next(statuses)), + # What: arrange the monkeypatch.setattr call with native router qualifier and next; why: test_native_router_benchmark_proves_warm_selector_reuses_resident_target groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + # What: act by calling monkeypatch.setattr with native router qualifier and canary and model and model and passed and true; why: the native router benchmark proves warm selector reuses resident target scenario observes the monkeypatch.setattr return value during native router qualifier canary. + monkeypatch.setattr( + # What: arrange the exact native router qualifier canary fixture fragment; why: the native router benchmark proves warm selector reuses resident target scenario feeds this byte-preserved fragment through native_router_qualifier, "canary" before asserting its protocol or parser result. + native_router_qualifier, "canary", + # What: arrange the base input for test_native_router_benchmark_proves_warm_selector_reuses_resident_target; why: test_native_router_benchmark_proves_warm_selector_reuses_resident_target consumes base during signature binding, so callers must bind it with the other signature inputs. + lambda base, model, direct: (b"data: private\n\n", { + # What: arrange the model field as model; why: test_native_router_benchmark_proves_warm_selector_reuses_resident_target sends this field through "model": model, "passed": True so the router selects the canonical model or alias for upstream dispatch. + "model": model, "passed": True, + # What: arrange the enclosing predicate collection with the named fixture input and model and model and passed and true; why: test_native_router_benchmark_proves_warm_selector_reuses_resident_target groups the supplied clauses as one }) collection before its value is consumed. + }), + # What: arrange the monkeypatch.setattr call with native router qualifier and model; why: test_native_router_benchmark_proves_warm_selector_reuses_resident_target groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + + # What: act by calling native_router_qualifier.selector_canary and capture result; why: the native router benchmark proves warm selector reuses resident target test asserts the response, state, or failure produced by this call. + result = native_router_qualifier.selector_canary("http://test", tmp_path) + + # What: assert the expected result == outcome; why: test swap qualification test native router benchmark proves warm selector reuses resident target protects its regression by requiring this observable result after the exercised behavior. + assert result == { + # What: arrange strategy warm resolvedProfile model a for the scenario; why: test swap qualification test native router benchmark proves warm selector reuses resident target requires this concrete input or helper state before exercising the behavior under test. + "strategy": "warm", "resolvedProfile": "model-a", + # What: arrange activationDelta 0 passed True for the scenario; why: test swap qualification test native router benchmark proves warm selector reuses resident target requires this concrete input or helper state before exercising the behavior under test. + "activationDelta": 0, "passed": True, + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router benchmark proves warm selector reuses resident target requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that tmp path warm selector sse read bytes equals b data private n n; why: this assertion protects the native router benchmark proves warm selector reuses resident target regression after the test's arranged inputs and exercised call. + assert (tmp_path / "warm-selector.sse").read_bytes() == b"data: private\n\n" + + +# What: define the test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap test around native router qualifier and monkeypatch and tmp path; why: this test groups the arrange, act, and assertions that protect the native router benchmark proves alias rewrites upstream without swap outcome. +def test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap( + # What: arrange native router qualifier monkeypatch tmp path for the scenario; why: test native router benchmark proves alias rewrites upstream without swap requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch, tmp_path +# What: arrange the grouped source fragment for the scenario; why: test native router benchmark proves alias rewrites upstream without swap requires this concrete input or helper state before exercising the behavior under test. +): + # What: act by calling iter and capture statuses; why: the native router benchmark proves alias rewrites upstream without swap test asserts the response, state, or failure produced by this call. + statuses = iter(( + # What: arrange the active profile field as model a; why: test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap carries active profile through statuses into lambda args kwargs b next statuses. + {"activeProfile": "model-a", "activeRequests": 0, "activations": 3}, + # What: arrange the active profile field as model a; why: test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap carries active profile through statuses into lambda args kwargs b next statuses. + {"activeProfile": "model-a", "activeRequests": 0, "activations": 3}, + # What: arrange the iter call with ordered positional inputs; why: test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap groups the supplied clauses as one iter call before its value is consumed. + )) + # What: act by calling monkeypatch.setattr with native router qualifier and request json and next and statuses; why: the native router benchmark proves alias rewrites upstream without swap scenario observes the monkeypatch.setattr return value during native router qualifier request json. + monkeypatch.setattr( + # What: arrange the exact native router qualifier request json fixture fragment; why: the native router benchmark proves alias rewrites upstream without swap scenario feeds this byte-preserved fragment through native_router_qualifier, "request_json" before asserting its protocol or parser result. + native_router_qualifier, "request_json", + # What: arrange the args input for test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap; why: test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap consumes args during signature binding, so callers must bind it with the other signature inputs. + lambda *args, **kwargs: (b"{}", next(statuses)), + # What: arrange the monkeypatch.setattr call with native router qualifier and next; why: test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + # What: arrange monkeypatch setattr for the scenario; why: test native router benchmark proves alias rewrites upstream without swap requires this concrete input or helper state before exercising the behavior under test. + monkeypatch.setattr( + # What: arrange the exact native router qualifier canary fixture fragment; why: the native router benchmark proves alias rewrites upstream without swap scenario feeds this byte-preserved fragment through native_router_qualifier, "canary" before asserting its protocol or parser result. + native_router_qualifier, "canary", + # What: arrange the base input for test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap; why: test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap consumes base during signature binding, so callers must bind it with the other signature inputs. + lambda base, model, direct: (b"data: private-rewrite\n\n", { + # What: arrange the model field as model; why: test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap sends this field through "model": model, "responseModel": "model-a", "passed": True so the router selects the canonical model or alias for upstream dispatch. + "model": model, "responseModel": "model-a", "passed": True, + # What: arrange the grouped source fragment for the scenario; why: test native router benchmark proves alias rewrites upstream without swap requires this concrete input or helper state before exercising the behavior under test. + }), + # What: arrange the monkeypatch.setattr call with native router qualifier and model; why: test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + + # What: act by calling native_router_qualifier.upstream_model_rewrite_canary and capture result; why: the native router benchmark proves alias rewrites upstream without swap test asserts the response, state, or failure produced by this call. + result = native_router_qualifier.upstream_model_rewrite_canary( + # What: arrange the http test tmp path portion of result; why: the native router benchmark proves alias rewrites upstream without swap scenario uses this clause to evaluate result as one grouped value. + "http://test", tmp_path + # What: arrange the native_router_qualifier.upstream_model_rewrite_canary call with tmp path; why: test_native_router_benchmark_proves_alias_rewrites_upstream_without_swap groups the supplied clauses as one native_router_qualifier.upstream_model_rewrite_canary call before its value is consumed. + ) + + # What: assert the expected result == outcome; why: test swap qualification test native router benchmark proves alias rewrites upstream without swap protects its regression by requiring this observable result after the exercised behavior. + assert result == { + # What: arrange requestedModel compat model a for the scenario; why: test swap qualification test native router benchmark proves alias rewrites upstream without swap requires this concrete input or helper state before exercising the behavior under test. + "requestedModel": "compat/model-a", + # What: arrange upstreamResponseModel model a for the scenario; why: test swap qualification test native router benchmark proves alias rewrites upstream without swap requires this concrete input or helper state before exercising the behavior under test. + "upstreamResponseModel": "model-a", + # What: arrange residentProfile model a for the scenario; why: test swap qualification test native router benchmark proves alias rewrites upstream without swap requires this concrete input or helper state before exercising the behavior under test. + "residentProfile": "model-a", + # What: arrange activationDelta 0 for the scenario; why: test swap qualification test native router benchmark proves alias rewrites upstream without swap requires this concrete input or helper state before exercising the behavior under test. + "activationDelta": 0, + # What: arrange passed True for the scenario; why: test swap qualification test native router benchmark proves alias rewrites upstream without swap requires this concrete input or helper state before exercising the behavior under test. + "passed": True, + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router benchmark proves alias rewrites upstream without swap requires this concrete input or. + } + # What: assert the expected tmp path upstream model rewrite sse read bytes == outcome; why: test swap qualification test native router benchmark proves alias rewrites upstream without swap protects its regression by requiring this observable result after the exercised behavior. + assert (tmp_path / "upstream-model-rewrite.sse").read_bytes() == ( + # What: arrange b data private rewrite n n for the scenario; why: test swap qualification test native router benchmark proves alias rewrites upstream without swap requires this concrete input or helper state before exercising the behavior under test. + b"data: private-rewrite\n\n" + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router benchmark proves alias rewrites upstream without swap requires this concrete input or. + ) + + +# What: define the test_native_router_benchmark_proves_profile_selector_composition_and_cleanup test around native router qualifier and monkeypatch and tmp path; why: this test groups the arrange, act, and assertions that protect the native router benchmark proves profile selector composition and cleanup outcome. +def test_native_router_benchmark_proves_profile_selector_composition_and_cleanup( + # What: arrange native router qualifier monkeypatch tmp path for the scenario; why: test native router benchmark proves profile selector composition and cleanup requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch, tmp_path +# What: arrange the grouped source fragment for the scenario; why: test native router benchmark proves profile selector composition and cleanup requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange calls as the fixture input; why: the native router benchmark proves profile selector composition and cleanup test consumes this named precondition before exercising the behavior. + calls = [] + # What: act by calling iter and capture statuses; why: the native router benchmark proves profile selector composition and cleanup test asserts the response, state, or failure produced by this call. + statuses = iter(( + # What: arrange activeProfile model a activeRequests 0 activations 3 for the scenario; why: test swap qualification test native router benchmark proves profile selector composition and cleanup requires this concrete input or helper state before exercising the behavior under test. + {"activeProfile": "model-a", "activeRequests": 0, "activations": 3}, + # What: arrange the statuses mapping with active profile and active routing profile and active requests and; why: test_native_router_benchmark_proves_profile_selector_composition_and_cleanup groups the supplied clauses as one statuses mapping before its. + { + # What: arrange activeProfile model a activeRoutingProfile coding for the scenario; why: test swap qualification test native router benchmark proves profile selector composition and cleanup requires this concrete input or helper state before exercising the behavior under test. + "activeProfile": "model-a", "activeRoutingProfile": "coding", + # What: arrange the active requests field as 0; why: test_native_router_benchmark_proves_profile_selector_composition_and_cleanup carries active requests through statuses into return b next statuses. + "activeRequests": 0, "activations": 3, + # What: arrange the statuses mapping with active profile and active routing profile and active requests and; why: test_native_router_benchmark_proves_profile_selector_composition_and_cleanup groups the supplied clauses as one statuses mapping before. + }, + # What: arrange the iter call with ordered positional inputs; why: test_native_router_benchmark_proves_profile_selector_composition_and_cleanup groups the supplied clauses as one iter call before its value is consumed. + )) + + # What: define the request_json test helper around url and body; why: the native router benchmark proves profile selector composition and cleanup scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def request_json(url, body=None, **kwargs): + # What: act by calling calls.append with url and body and kwargs; why: the native router benchmark proves profile selector composition and cleanup scenario observes the calls.append return value during if url endswith router status. + calls.append((url, body, kwargs)) + # What: act on endswith and url before next and statuses; why: the native router benchmark proves profile selector composition and cleanup scenario admits next and statuses only for this predicate and excludes the opposite state. + if url.endswith("/router/status"): + # What: return next and statuses from the request_json test helper; why: the native router benchmark proves profile selector composition and cleanup scenario uses this helper result in its subsequent act or assertion. + return b"{}", next(statuses) + # What: act on endswith and url before body; why: the native router benchmark proves profile selector composition and cleanup scenario admits body only for this predicate and excludes the opposite state. + if url.endswith("/router/profiles/active"): + # What: arrange the active field as body and name; why: request_json carries active into return b"{}", {"active": body["name"]}. + return b"{}", {"active": body["name"]} + # What: act on endswith and url before the computed value; why: the native router benchmark proves profile selector composition and cleanup scenario admits the computed value only for this predicate and excludes the opposite state. + if url.endswith("/v1/models"): + # What: return data and id and profile model and id and model a from the request_json test helper; why: the native router benchmark proves profile selector composition and cleanup scenario uses this helper result in its subsequent act or assertion. + return b'{"data":[]}', { + # What: arrange the id field as profile model; why: request_json carries id into "data": [{"id": "profile-model"}, {"id": "model-a"}]. + "data": [{"id": "profile-model"}, {"id": "model-a"}], + # What: arrange the enclosing predicate collection with the named fixture input and data and id and profile model and id and model a; why: request_json groups the supplied clauses as one request_json expression collection before its value is consumed. + } + # What: raise AssertionError for the caller; why: request_json stops this rejected path before it can mutate state, dispatch work, or report success. + raise AssertionError(url) + + # What: arrange the exact monkeypatch setattr native router qualifier request json request json fixture fragment; why: the native router benchmark proves profile selector composition and cleanup scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier, "request_json", request. + monkeypatch.setattr(native_router_qualifier, "request_json", request_json) + # What: act by calling monkeypatch.setattr with native router qualifier and canary and model and model and passed and true; why: the native router benchmark proves profile selector composition and cleanup scenario observes the monkeypatch.setattr return value during native router qualifier canary. + monkeypatch.setattr( + # What: arrange the exact native router qualifier canary fixture fragment; why: the native router benchmark proves profile selector composition and cleanup scenario feeds this byte-preserved fragment through native_router_qualifier, "canary" before asserting its protocol or parser result. + native_router_qualifier, "canary", + # What: arrange the base input for test_native_router_benchmark_proves_profile_selector_composition_and_cleanup; why: test_native_router_benchmark_proves_profile_selector_composition_and_cleanup consumes base during signature binding, so callers must bind it with the other signature inputs. + lambda base, model, direct: (b"data: private-profile\n\n", { + # What: arrange the model field as model; why: test_native_router_benchmark_proves_profile_selector_composition_and_cleanup sends this field through "model": model, "passed": True so the router selects the canonical model or alias for upstream dispatch. + "model": model, "passed": True, + # What: arrange the grouped source fragment for the scenario; why: test native router benchmark proves profile selector composition and cleanup requires this concrete input or helper state before exercising the behavior under test. + }), + # What: arrange the monkeypatch.setattr call with native router qualifier and model; why: test_native_router_benchmark_proves_profile_selector_composition_and_cleanup groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + + # What: act by calling native_router_qualifier.routing_profile_canary and capture result; why: the native router benchmark proves profile selector composition and cleanup test asserts the response, state, or failure produced by this call. + result = native_router_qualifier.routing_profile_canary("http://test", tmp_path) + + # What: assert the expected result == outcome; why: test swap qualification test native router benchmark proves profile selector composition and cleanup protects its regression by requiring this observable result after the exercised behavior. + assert result == { + # What: arrange profileActivated True profileCleared True for the scenario; why: test swap qualification test native router benchmark proves profile selector composition and cleanup requires this concrete input or helper state before exercising the behavior under test. + "profileActivated": True, "profileCleared": True, + # What: arrange selectorComposed True resolvedProfile model a for the scenario; why: test swap qualification test native router benchmark proves profile selector composition and cleanup requires this concrete input or helper state before exercising the behavior under test. + "selectorComposed": True, "resolvedProfile": "model-a", + # What: arrange activationDelta 0 passed True for the scenario; why: test swap qualification test native router benchmark proves profile selector composition and cleanup requires this concrete input or helper state before exercising the behavior under test. + "activationDelta": 0, "passed": True, + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router benchmark proves profile selector composition and cleanup requires this concrete input or helper state before exercising the behavior under test. + } + # What: act by calling call.endswith and capture profile calls; why: the native router benchmark proves profile selector composition and cleanup test asserts the response, state, or failure produced by this call. + profile_calls = [call for call in calls if call[0].endswith("/router/profiles/active")] + # What: assert that call 1 for call in profile calls equals name coding name; why: this assertion protects the native router benchmark proves profile selector composition and cleanup regression after the test's arranged inputs and exercised call. + assert [call[1] for call in profile_calls] == [{"name": "coding"}, {"name": None}] + # What: assert that all call 2 method equals put for call in profile calls; why: this assertion protects the native router benchmark proves profile selector composition and cleanup regression after the test's arranged inputs and exercised call. + assert all(call[2]["method"] == "PUT" for call in profile_calls) + # What: assert that tmp path routing profile sse read bytes equals b data private profile n n; why: this assertion protects the native router benchmark proves profile selector composition and cleanup regression after the test's arranged inputs and exercised call. + assert (tmp_path / "routing-profile.sse").read_bytes() == b"data: private-profile\n\n" + # What: assert that tmp path routing profile models json read bytes equals b data; why: this assertion protects the native router benchmark proves profile selector composition and cleanup regression after the test's arranged inputs and exercised call. + assert (tmp_path / "routing-profile-models.json").read_bytes() == b'{"data":[]}' + + +# What: define the test_native_router_benchmark_captures_private_hardware_observation test around native router qualifier and monkeypatch and tmp path; why: this test groups the arrange, act, and assertions that protect the native router benchmark captures private hardware observation outcome. +def test_native_router_benchmark_captures_private_hardware_observation( + # What: arrange native router qualifier monkeypatch tmp path for the scenario; why: test native router benchmark captures private hardware observation requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch, tmp_path +# What: arrange the grouped source fragment for the scenario; why: test native router benchmark captures private hardware observation requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange captured as the fixture input; why: the native router benchmark captures private hardware observation test consumes this named precondition before exercising the behavior. + captured = b'{"engine":{"running":true,"pid":7,"port":1234},"memory":{"ramBytes":3,"vramBytes":4,"ramAvailable":true,"vramAvailable":true,"ramSource":"proc-smaps-rollup-pss","vramSource":"amd-smi"}}' + # What: arrange the exact monkeypatch setattr native router qualifier request json lambda a k fixture fragment; why: the native router benchmark captures private hardware observation scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier, "request_json", lambda *a, before a. + monkeypatch.setattr(native_router_qualifier, "request_json", lambda *a, **k: (captured, { + # What: arrange the running field as true; why: test_native_router_benchmark_captures_private_hardware_observation carries running into "engine": {"running": True, "pid": 7, "port": 1234}. + "engine": {"running": True, "pid": 7, "port": 1234}, + # What: arrange the ram bytes field as 3; why: test_native_router_benchmark_captures_private_hardware_observation carries ram bytes into "memory": {"ramBytes": 3, "vramBytes": 4, "ramAvailable": True. + "memory": {"ramBytes": 3, "vramBytes": 4, "ramAvailable": True, + # What: arrange the vram available field as true; why: test_native_router_benchmark_captures_private_hardware_observation carries vram available into "vramAvailable": True, "ramSource": "proc-smaps-rollup-pss". + "vramAvailable": True, "ramSource": "proc-smaps-rollup-pss", + # What: arrange the vram source field as amd smi; why: test_native_router_benchmark_captures_private_hardware_observation carries vram source into "vramSource": "amd-smi"}. + "vramSource": "amd-smi"}, + # What: arrange the monkeypatch.setattr call with native router qualifier and captured; why: test_native_router_benchmark_captures_private_hardware_observation groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + })) + + # What: act by calling native_router_qualifier.capture_hardware and capture hardware; why: the native router benchmark captures private hardware observation test asserts the response, state, or failure produced by this call. + hardware = native_router_qualifier.capture_hardware("http://test", tmp_path, "warm-a") + + # What: assert that hardware engine pid equals 7; why: this assertion protects the native router benchmark captures private hardware observation regression after the test's arranged inputs and exercised call. + assert hardware["engine"]["pid"] == 7 + # What: assert that tmp path warm a hardware json read bytes equals captured; why: this assertion protects the native router benchmark captures private hardware observation regression after the test's arranged inputs and exercised call. + assert (tmp_path / "warm-a.hardware.json").read_bytes() == captured + + +# What: define the test_native_router_benchmark_validates_same_process_re_adoption test around native router qualifier; why: this test groups the arrange, act, and assertions that protect the native router benchmark validates same process re adoption outcome. +def test_native_router_benchmark_validates_same_process_re_adoption(native_router_qualifier): + # What: arrange before as running and pid and port and adopted and true; why: the native router benchmark validates same process re adoption test consumes this named precondition before exercising the behavior. + before = {"running": True, "pid": 41, "port": 24567, "adopted": False} + # What: arrange after as running and pid and port and adopted and true; why: the native router benchmark validates same process re adoption test consumes this named precondition before exercising the behavior. + after = {"running": True, "pid": 41, "port": 24567, "adopted": True} + # What: arrange router as active profile and active identity matches engine and activations and model a and true; why: the native router benchmark validates same process re adoption test consumes this named precondition before exercising the behavior. + router = { + # What: arrange the active profile field as model a; why: test_native_router_benchmark_validates_same_process_re_adoption carries active profile through router into assert native router qualifier validate re adoption before after router equals. + "activeProfile": "model-a", "activeIdentityMatchesEngine": True, + # What: arrange the activations field as 0; why: test_native_router_benchmark_validates_same_process_re_adoption carries activations through router into assert native router qualifier validate re adoption before after router equals. + "activations": 0, + # What: arrange the router mapping with active profile and active identity matches engine and activations; why: test_native_router_benchmark_validates_same_process_re_adoption groups the supplied clauses as one router mapping before its value is consumed. + } + + # What: assert the expected native router qualifier validate re adoption before after router == outcome; why: test swap qualification test protects its regression by requiring this observable result after the exercised behavior. + assert native_router_qualifier.validate_re_adoption(before, after, router) == { + # What: arrange profile model a samePid True samePort True for the scenario; why: test swap qualification test native router benchmark validates same process re adoption requires this concrete input or helper state before exercising the behavior under test. + "profile": "model-a", "samePid": True, "samePort": True, + # What: arrange managerAdopted True activationDelta 0 for the scenario; why: test swap qualification test native router benchmark validates same process re adoption requires this concrete input or helper state before exercising the behavior under test. + "managerAdopted": True, "activationDelta": 0, + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router benchmark validates same process re adoption requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert the pytest.raises failure context; why: the native router benchmark validates same process re adoption scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError, match="exact adopted residency"): + # What: act by calling native_router_qualifier.validate_re_adoption with before and after and pid and 42 and router; why: the native router benchmark validates same process re adoption scenario observes the native_router_qualifier.validate_re_adoption return value during before after pid router. + native_router_qualifier.validate_re_adoption( + # What: arrange the pid field as 42; why: test_native_router_benchmark_validates_same_process_re_adoption carries pid into before, {**after, "pid": 42}, router. + before, {**after, "pid": 42}, router + # What: arrange the native_router_qualifier.validate_re_adoption call with before and after and router; why: test_native_router_benchmark_validates_same_process_re_adoption groups the supplied clauses as one native_router_qualifier.validate_re_adoption call before its value is consumed. + ) + + +# What: parameterize test_native_router_benchmark_rejects_incomplete_hardware_observation with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test native router benchmark rejects incomplete hardware observation. +@pytest.mark.parametrize( + # What: arrange the hardware portion of the enclosing predicate; why: this clause remains in the native router benchmark rejects incomplete hardware observation scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "hardware", + # What: arrange the grouped source fragment for the scenario; why: test native router benchmark rejects incomplete hardware observation requires this concrete input or helper state before exercising the behavior under test. + [ + # What: arrange the engine field as running and pid and port and false and 7; why: test_native_router_benchmark_rejects_incomplete_hardware_observation carries engine into {"engine": {"running": False, "pid": 7, "port": 1234}, "memory": {"ramBy. + {"engine": {"running": False, "pid": 7, "port": 1234}, "memory": {"ramBytes": 3, "vramBytes": 4}}, + # What: arrange the engine field as running and pid and port and true and 1234; why: test_native_router_benchmark_rejects_incomplete_hardware_observation carries engine into {"engine": {"running": True, "pid": None, "port": 1234}, "memory": {"ram. + {"engine": {"running": True, "pid": None, "port": 1234}, "memory": {"ramBytes": 3, "vramBytes": 4}}, + # What: arrange the engine field as running and pid and port and true and 0; why: test_native_router_benchmark_rejects_incomplete_hardware_observation carries engine into {"engine": {"running": True, "pid": 0, "port": 1234}, "memory": {"ramByt. + {"engine": {"running": True, "pid": 0, "port": 1234}, "memory": {"ramBytes": 3, "vramBytes": 4}}, + # What: arrange engine running True pid 7 port 0 memory ramBytes 3 vramBytes 4 for the scenario; why: test swap qualification test native router benchmark rejects incomplete hardware observation requires this concrete input or helper state before exercising the behavior under test. + {"engine": {"running": True, "pid": 7, "port": 0}, "memory": {"ramBytes": 3, "vramBytes": 4}}, + # What: arrange engine running True pid 7 port 1234 memory ramBytes None vramBytes 4 for the scenario; why: test swap qualification test native router benchmark rejects incomplete hardware observation requires this concrete input or helper state before exercising the behavior under test. + {"engine": {"running": True, "pid": 7, "port": 1234}, "memory": {"ramBytes": None, "vramBytes": 4}}, + # What: arrange the grouped source fragment for the scenario; why: test native router benchmark rejects incomplete hardware observation requires this concrete input or helper state before exercising the behavior under test. + ], +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_native_router_benchmark_rejects_incomplete_hardware_observation groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +) +# What: define the test_native_router_benchmark_rejects_incomplete_hardware_observation test around native router qualifier and monkeypatch and tmp path and hardware; why: this test groups the arrange, act, and assertions that protect the native router benchmark rejects incomplete hardware observation outcome. +def test_native_router_benchmark_rejects_incomplete_hardware_observation( + # What: arrange native router qualifier monkeypatch tmp path hardware for the scenario; why: test native router benchmark rejects incomplete hardware observation requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch, tmp_path, hardware +# What: arrange the grouped source fragment for the scenario; why: test native router benchmark rejects incomplete hardware observation requires this concrete input. +): + # What: arrange the exact monkeypatch setattr native router qualifier request json lambda a k fixture fragment; why: the native router benchmark rejects incomplete hardware observation scenario feeds this byte-preserved fragment through monkeypatch.setattr(native_router_qualifier, "request_json", lambda *a, before. + monkeypatch.setattr(native_router_qualifier, "request_json", lambda *a, **k: (b"{}", hardware)) + # What: assert the pytest.raises failure context; why: the native router benchmark rejects incomplete hardware observation scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError): + # What: arrange the exact native router qualifier capture hardware http test tmp path bad fixture fragment; why: the native router benchmark rejects incomplete hardware observation scenario feeds this byte-preserved fragment through native_router_qualifier.capture_hardware("http://test", tmp_path, "bad") befor. + native_router_qualifier.capture_hardware("http://test", tmp_path, "bad") + + +# What: parameterize test_native_router_benchmark_rejects_unmeasured_hardware_values with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test native router benchmark rejects unmeasured hardware values. +@pytest.mark.parametrize( + # What: arrange the field value portion of the enclosing predicate; why: this clause remains in the native router benchmark rejects unmeasured hardware values scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "field,value", + # What: arrange the vram available ram bytes vram source portion of the enclosing predicate; why: this clause remains in the native router benchmark rejects unmeasured hardware values scenario\'s enclosing expression so its grouping and evaluation order stay intact. + [("vramAvailable", False), ("ramBytes", 0), ("vramSource", None)], +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_native_router_benchmark_rejects_unmeasured_hardware_values groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +) +# What: define the test_native_router_benchmark_rejects_unmeasured_hardware_values test around native router qualifier and monkeypatch and tmp path and field and value; why: this test groups the arrange, act, and assertions that protect the native router benchmark rejects unmeasured hardware values outcome. +def test_native_router_benchmark_rejects_unmeasured_hardware_values( + # What: arrange native router qualifier monkeypatch tmp path field value for the scenario; why: test native router benchmark rejects requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, monkeypatch, tmp_path, field, value +# What: arrange the grouped source fragment for the scenario; why: test native router benchmark rejects unmeasured hardware values requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange memory as ram bytes and vram bytes and ram available and vram available and ram source; why: the native router benchmark rejects unmeasured hardware values test consumes this named precondition before exercising the behavior. + memory = { + # What: arrange the ram bytes field as 3; why: test_native_router_benchmark_rejects_unmeasured_hardware_values carries ram bytes through memory into memory field value. + "ramBytes": 3, + # What: arrange the vram bytes field as 4; why: test_native_router_benchmark_rejects_unmeasured_hardware_values carries vram bytes through memory into memory field value. + "vramBytes": 4, + # What: arrange the ram available field as true; why: test_native_router_benchmark_rejects_unmeasured_hardware_values carries ram available through memory into memory field value. + "ramAvailable": True, + # What: arrange the vram available field as true; why: test_native_router_benchmark_rejects_unmeasured_hardware_values carries vram available through memory into memory field value. + "vramAvailable": True, + # What: arrange the ram source field as proc smaps rollup pss; why: test_native_router_benchmark_rejects_unmeasured_hardware_values carries ram source through memory into memory field value. + "ramSource": "proc-smaps-rollup-pss", + # What: arrange the vram source field as amd smi; why: test_native_router_benchmark_rejects_unmeasured_hardware_values carries vram source through memory into memory field value. + "vramSource": "amd-smi", + # What: arrange the memory mapping with ram bytes and vram bytes and ram available and vram available and ram source; why: test_native_router_benchmark_rejects_unmeasured_hardware_values groups the supplied clauses as one memory mapping before its value is consumed. + } + # What: arrange memory entry as value; why: the native router benchmark rejects unmeasured hardware values test consumes this named precondition before exercising the behavior. + memory[field] = value + # What: arrange hardware as memory and engine and memory and running and pid; why: the native router benchmark rejects unmeasured hardware values test consumes this named precondition before exercising the behavior. + hardware = { + # What: arrange the running field as true; why: test_native_router_benchmark_rejects_unmeasured_hardware_values carries running through hardware into native router qualifier request json lambda args kwargs b hardware. + "engine": {"running": True, "pid": 7, "port": 1234}, + # What: arrange the memory field as memory; why: test_native_router_benchmark_rejects_unmeasured_hardware_values carries memory through hardware into native router qualifier request json lambda args kwargs b hardware. + "memory": memory, + # What: arrange the hardware mapping with engine and memory; why: test_native_router_benchmark_rejects_unmeasured_hardware_values groups the supplied clauses as one hardware mapping before its value is consumed. + } + # What: act by calling monkeypatch.setattr with native router qualifier and request json and hardware; why: the native router benchmark rejects unmeasured hardware values scenario observes the monkeypatch.setattr return value during native router qualifier request json lambda args kwargs b. + monkeypatch.setattr( + # What: arrange the exact native router qualifier request json lambda args kwargs b fixture fragment; why: the native router benchmark rejects unmeasured hardware values scenario feeds this byte-preserved fragment through native_router_qualifier, "request_json", lambda *args, **kwargs: (b"{}", before asserting. + native_router_qualifier, "request_json", lambda *args, **kwargs: (b"{}", hardware) + # What: arrange the monkeypatch.setattr call with native router qualifier and hardware; why: test_native_router_benchmark_rejects_unmeasured_hardware_values groups the supplied clauses as one monkeypatch.setattr call before its value is consumed. + ) + + # What: assert the pytest.raises failure context; why: the native router benchmark rejects unmeasured hardware values scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError): + # What: arrange the exact native router qualifier capture hardware http test tmp path bad fixture fragment; why: the native router benchmark rejects unmeasured hardware values scenario feeds this byte-preserved fragment through native_router_qualifier.capture_hardware("http://test", tmp_path, "bad") before ass. + native_router_qualifier.capture_hardware("http://test", tmp_path, "bad") + + +# What: define the test_native_router_benchmark_requires_final_engine_listener_to_close test around native router qualifier; why: this test groups the arrange, act, and assertions that protect the native router benchmark requires final engine listener to close outcome. +def test_native_router_benchmark_requires_final_engine_listener_to_close(native_router_qualifier): + # What: enter the socket.socket managed context before listener bind; why: test_native_router_benchmark_requires_final_engine_listener_to_close releases this resource or lock after listener bind on both success and failure paths. + with socket.socket() as listener: + # What: arrange the exact listener bind fixture fragment; why: the native router benchmark requires final engine listener to close scenario feeds this byte-preserved fragment through listener.bind(("127.0.0.1", 0)) before asserting its protocol or parser result. + listener.bind(("127.0.0.1", 0)) + # What: act by calling listener.listen with the declared inputs; why: the native router benchmark requires final engine listener to close scenario observes the listener.listen return value during with pytest raises runtime error match listener. + listener.listen() + # What: assert the pytest.raises failure context; why: the native router benchmark requires final engine listener to close scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError, match="listener"): + # What: act by calling native_router_qualifier.require_listener_closed with getsockname and listener and 1; why: the native router benchmark requires final engine listener to close scenario observes the native_router_qualifier.require_listener_closed return value during port listener getsockname. + native_router_qualifier.require_listener_closed(listener.getsockname()[1]) + # What: act by calling listener.getsockname and capture port; why: the native router benchmark requires final engine listener to close test asserts the response, state, or failure produced by this call. + port = listener.getsockname()[1] + # What: act by calling native_router_qualifier.require_listener_closed with port; why: the native router benchmark requires final engine listener to close scenario observes the native_router_qualifier.require_listener_closed return value during the enclosing return. + native_router_qualifier.require_listener_closed(port) + + +# What: define the test_native_router_benchmark_generates_a_valid_dynamic_port_catalog test around native router qualifier and tmp path; why: this test groups the arrange, act, and assertions that protect the native router benchmark generates a valid dynamic port catalog outcome. +def test_native_router_benchmark_generates_a_valid_dynamic_port_catalog(native_router_qualifier, tmp_path): + # What: arrange catalog path as tmp path and models and toml; why: the native router benchmark generates a valid dynamic port catalog test consumes this named precondition before exercising the behavior. + catalog_path = tmp_path / "models.toml" + # What: arrange catalog path write text for the scenario; why: test native router benchmark generates a valid dynamic port catalog requires this concrete input or helper state before exercising the behavior under test. + catalog_path.write_text( + # What: act by calling native_router_qualifier.native_catalog_text with first and gguf and second and gguf; why: the native router benchmark generates a valid dynamic port catalog scenario observes the native_router_qualifier.native_catalog_text return value during first gguf second gguf api key private key. + native_router_qualifier.native_catalog_text( + # What: arrange the exact first gguf second gguf api key private key fixture fragment; why: the native router benchmark generates a valid dynamic port catalog scenario feeds this byte-preserved fragment through "first.gguf", "second.gguf", api_key="private-key" before asserting its protocol or parser resul. + "first.gguf", "second.gguf", api_key="private-key" + # What: arrange the native_router_qualifier.native_catalog_text call with api key; why: test_native_router_benchmark_generates_a_valid_dynamic_port_catalog groups the supplied clauses as one native_router_qualifier.native_catalog_text call before its value is consumed. + ), + # What: arrange the exact encoding utf 8 fixture fragment; why: the native router benchmark generates a valid dynamic port catalog scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the catalog_path.write_text call with encoding; why: test_native_router_benchmark_generates_a_valid_dynamic_port_catalog groups the supplied clauses as one catalog_path.write_text call before its value is consumed. + ) + + # What: act by calling ModelCatalog.load and capture catalog; why: the native router benchmark generates a valid dynamic port catalog test asserts the response, state, or failure produced by this call. + catalog = ModelCatalog.load(str(catalog_path)) + + # What: assert that catalog settings upstream timeout s equals 660; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.settings.upstream_timeout_s == 660 + # What: assert that catalog settings api keys equals private key; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.settings.api_keys == ("private-key",) + # What: assert that catalog settings include aliases in list is true; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.settings.include_aliases_in_list is True + # What: assert that catalog settings send loading state is true; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.settings.send_loading_state is True + # What: assert that catalog get model a model equals first gguf; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.get("model-a").model == "first.gguf" + # What: assert that catalog get model a port equals 0; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.get("model-a").port == 0 + # What: assert that catalog get model a ttl s equals 0; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.get("model-a").ttl_s == 0 + # What: assert that catalog get model a check endpoint equals ready; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.get("model-a").check_endpoint == "/ready" + # What: assert that catalog get model a proxy equals http 127 0 0 1 port; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.get("model-a").proxy == "http://127.0.0.1:${PORT}" + # What: assert that catalog get model a use model name equals model a; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.get("model-a").use_model_name == "model-a" + # What: assert that catalog get model a upstream timeout s equals 659; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.get("model-a").upstream_timeout_s == 659 + # What: assert that catalog get model a display name equals qualification model a; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.get("model-a").display_name == "Qualification model A" + # What: assert the expected catalog get model a metadata == outcome; why: test swap qualification test native router benchmark generates a valid dynamic port catalog protects its regression by requiring this observable result after the exercised behavior. + assert catalog.get("model-a").metadata() == { + # What: arrange tier qualification type operator for the scenario; why: test swap qualification test native router benchmark generates a valid dynamic port catalog requires this concrete input or helper state before exercising the behavior under test. + "tier": "qualification", "type": "operator", + # What: arrange the grouped source fragment for the scenario; why: test swap qualification test native router benchmark generates a valid dynamic port catalog requires this concrete input or helper state before exercising the behavior under test. + } + # What: assert that catalog get compat model a name equals model a; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.get("compat/model-a").name == "model-a" + # What: assert that model a is present in catalog get model a args; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert "model-a" in catalog.get("model-a").args + # What: assert that catalog get model b model equals second gguf; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert catalog.get("model-b").model == "second.gguf" + # What: act by calling catalog.selector and capture selector; why: the native router benchmark generates a valid dynamic port catalog test asserts the response, state, or failure produced by this call. + selector = catalog.selector("preferred-model") + # What: assert that selector is not group delimiter; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert selector is not None + # What: assert that selector strategy equals warm; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert selector.strategy == "warm" + # What: assert that selector targets equals model b model a; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert selector.targets == ("model-b", "model-a") + # What: act by calling catalog.routing_profile and capture routing profile; why: the native router benchmark generates a valid dynamic port catalog test asserts the response, state, or failure produced by this call. + routing_profile = catalog.routing_profile("coding") + # What: assert that routing profile is not group delimiter; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert routing_profile is not None + # What: assert that routing profile replacement profile model equals true preferred model; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert routing_profile.replacement("profile-model") == (True, "preferred-model") + # What: assert that routing profile replacement disabled model equals true; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert routing_profile.replacement("disabled-model") == (True, None) + + # What: arrange startup path as tmp path and startup models and toml; why: the native router benchmark generates a valid dynamic port catalog test consumes this named precondition before exercising the behavior. + startup_path = tmp_path / "startup-models.toml" + # What: arrange startup path write text for the scenario; why: test native router benchmark generates a valid dynamic port catalog requires this concrete input or helper state before exercising the behavior under test. + startup_path.write_text( + # What: act by calling native_router_qualifier.native_catalog_text with first and gguf and second and gguf; why: the native router benchmark generates a valid dynamic port catalog scenario observes the native_router_qualifier.native_catalog_text return value during first gguf second gguf startup. + native_router_qualifier.native_catalog_text( + # What: arrange the exact first gguf second gguf startup fixture fragment; why: the native router benchmark generates a valid dynamic port catalog scenario feeds this byte-preserved fragment through "first.gguf", "second.gguf", startup=True before asserting its protocol or parser result. + "first.gguf", "second.gguf", startup=True + # What: arrange the native_router_qualifier.native_catalog_text call with startup; why: test_native_router_benchmark_generates_a_valid_dynamic_port_catalog groups the supplied clauses as one native_router_qualifier.native_catalog_text call before its value is consumed. + ), + # What: arrange the exact encoding utf 8 fixture fragment; why: the native router benchmark generates a valid dynamic port catalog scenario feeds this byte-preserved fragment through encoding="utf-8" before asserting its protocol or parser result. + encoding="utf-8", + # What: arrange the startup_path.write_text call with encoding; why: test_native_router_benchmark_generates_a_valid_dynamic_port_catalog groups the supplied clauses as one startup_path.write_text call before its value is consumed. + ) + # What: act by calling ModelCatalog.load and capture startup; why: the native router benchmark generates a valid dynamic port catalog test asserts the response, state, or failure produced by this call. + startup = ModelCatalog.load(str(startup_path)) + # What: assert that startup settings preload model equals model a; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert startup.settings.preload_model == "model-a" + # What: assert that startup settings startup routing profile equals coding; why: this assertion protects the native router benchmark generates a valid dynamic port catalog regression after the test's arranged inputs and exercised call. + assert startup.settings.startup_routing_profile == "coding" + + +# What: parameterize test_native_router_benchmark_rejects_mislabeled_routed_trials with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test native router benchmark rejects mislabeled routed trials. +@pytest.mark.parametrize( + # What: arrange the status alias prior delta portion of the enclosing predicate; why: this clause remains in the native router benchmark rejects mislabeled routed trials scenario\'s enclosing expression so its grouping and evaluation order stay intact. + "status,alias,prior,delta", + # What: arrange the grouped source fragment for the scenario; why: test native router benchmark rejects mislabeled routed trials requires this concrete input or helper state before exercising the behavior under test. + [ + # What: arrange activeProfile model a activeRequests 1 activations 1 model a 1 0 for the scenario; why: test swap qualification test native router benchmark rejects mislabeled routed trials requires this concrete input or helper state before exercising the behavior under test. + ({"activeProfile": "model-a", "activeRequests": 1, "activations": 1}, "model-a", 1, 0), + # What: arrange the active profile field as model b; why: test_native_router_benchmark_rejects_mislabeled_routed_trials carries active profile into ({"activeProfile": "model-b", "activeRequests": 0, "activations": 1}, "m. + ({"activeProfile": "model-b", "activeRequests": 0, "activations": 1}, "model-a", 1, 0), + # What: arrange activeProfile model a activeRequests 0 activations 2 model a 1 0 for the scenario; why: test swap qualification test native router benchmark rejects mislabeled routed trials requires this concrete input or helper state before exercising the behavior under test. + ({"activeProfile": "model-a", "activeRequests": 0, "activations": 2}, "model-a", 1, 0), + # What: arrange the grouped source fragment for the scenario; why: test native router benchmark rejects mislabeled routed trials requires this concrete input or helper state before exercising the behavior under test. + ], +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_native_router_benchmark_rejects_mislabeled_routed_trials groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +) +# What: define the test_native_router_benchmark_rejects_mislabeled_routed_trials test around native router qualifier and status and alias and prior and delta; why: this test groups the arrange, act, and assertions that protect the native router benchmark rejects mislabeled routed trials outcome. +def test_native_router_benchmark_rejects_mislabeled_routed_trials( + # What: arrange native router qualifier status alias prior delta for the scenario; why: test native router benchmark rejects mislabeled routed trials requires this concrete input or helper state before exercising the behavior under test. + native_router_qualifier, status, alias, prior, delta +# What: arrange the grouped source fragment for the scenario; why: test native router benchmark rejects mislabeled routed trials requires this concrete input or helper state before exercising the behavior under test. +): + # What: assert the pytest.raises failure context; why: the native router benchmark rejects mislabeled routed trials scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(RuntimeError): + # What: act by calling native_router_qualifier.validate_routed_trial with status; why: the native router benchmark rejects mislabeled routed trials scenario observes the native_router_qualifier.validate_routed_trial return value during status alias alias prior activations prior expected delta. + native_router_qualifier.validate_routed_trial( + # What: arrange alias to native_router_qualifier.validate_routed_trial; why: the native router benchmark rejects mislabeled routed trials scenario binds this alias value to native_router_qualifier.validate_routed_trial's alias input. + status, alias=alias, prior_activations=prior, expected_delta=delta + # What: arrange the grouped source fragment for the scenario; why: test native router benchmark rejects mislabeled routed trials requires this concrete input or helper state before exercising the behavior. + ) + +# What: define protected-service scope coverage; why: maintenance must target the exact manager that owns the workload. +def test_native_router_protected_service_scope_commands_are_exact(native_router_qualifier): + # What: require the established system command; why: existing callers must retain non-interactive root service behavior. + assert native_router_qualifier.protected_service_command("system") == ["sudo", "-n", "systemctl"] + # What: require the user-manager command; why: the target AMD host's protected Nemotron unit lives in the invoking user's scope. + assert native_router_qualifier.protected_service_command("user") == ["systemctl", "--user"] + # What: establish the invalid-scope failure boundary; why: ambiguous ownership must fail before service mutation. + with pytest.raises(ValueError, match="protected service scope"): + # What: submit an unsupported scope; why: the test proves the helper rejects rather than guessing a manager. + native_router_qualifier.protected_service_command("session") + + +# What: define the target AMD host catalog capacity coverage; why: the shipped router must retain the exact memory-safe runtime envelope proven on gfx1150. +def test_native_router_catalog_uses_amd_host_safe_capacity(native_router_qualifier): + # What: render a representative two-model catalog; why: assertions must inspect the actual command arguments emitted to managed engines. + catalog = native_router_qualifier.native_catalog_text("first.gguf", "second-model") + # What: require the bounded sequence length argument; why: the 24 GiB host must not silently return to the failed 4096-token allocation profile. + assert '"--max-seq-len-override", "1024"' in catalog + # What: require the bounded cache capacity argument; why: engine startup must preserve the measured headroom used by both qualified artifacts. + assert '"--num-tokens", "1024"' in catalog + # What: require the naive cache selection; why: recurrent-state hybrid allocation previously exhausted memory for a candidate model. + assert '"--cache-type", "naive"' in catalog + # What: require the measured memory ratio argument; why: the target AMD host qualification depends on the exact successful allocation policy rather than an implicit default. + assert '"--memory-ratio", "0.90"' in catalog diff --git a/tests/daemon/test_swap_regressions.py b/tests/daemon/test_swap_regressions.py new file mode 100644 index 0000000000..3b9f497fdf --- /dev/null +++ b/tests/daemon/test_swap_regressions.py @@ -0,0 +1,343 @@ +"""Swap boundary regressions, runnable without the GPU runtime.""" +# What: document swap boundary regressions runnable without the in the test_swap_regressions docstring; why: introspection and maintainers read this exact docstring fragment to understand test swap regressions behavior without executing it. + +# What: import ast for test readiness http contract using ast; why: test_readiness_http_contract uses ast parse, making that imported dependency available to its named operation. +import ast +# What: import json for test routing profile client uses atomic selection endpoint using json; why: test_routing_profile_client_uses_atomic_selection_endpoint uses json loads, making that imported dependency available to its named operation. +import json +# What: import threading for test profile readiness failure recovery end to end using threading; why: test_profile_readiness_failure_recovery_end_to_end uses threading event, making that imported dependency available to its named operation. +import threading +# What: import thread pool executor for test profile readiness failure recovery end to end using concurrent and futures and thread pool executor; why: test_profile_readiness_failure_recovery_end_to_end uses thread pool executor, making that imported dependency available to its named operation. +from concurrent.futures import ThreadPoolExecutor +# What: import path for test readiness http contract using pathlib and path; why: test_readiness_http_contract uses path, making that imported dependency available to its named operation. +from pathlib import Path + +# What: import pytest for module initialization using pytest; why: module initialization uses pytest mark parametrize, making that imported dependency available to its named operation. +import pytest +# What: import fast api for test readiness http contract using fastapi and fast api; why: test_readiness_http_contract uses fast api, making that imported dependency available to its named operation. +from fastapi import FastAPI +# What: import test client for test readiness http contract using fastapi and testclient and test client; why: test_readiness_http_contract uses test client, making that imported dependency available to its named operation. +from fastapi.testclient import TestClient + +# What: arrange from freetoken daemon catalog import CatalogError ModelCatalog for the scenario; why: test swap regressions requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon.catalog import CatalogError, ModelCatalog +# What: arrange from freetoken daemon import client as daemon client for the scenario; why: test swap regressions requires this concrete input or helper state before exercising the behavior under test. +from freetoken.daemon import client as daemon_client +# What: import wait for ready for test readiness rechecks generation after probe using freetoken and daemon and readiness and wait for ready; why: test_readiness_rechecks_generation_after_probe uses wait for ready, making that imported dependency available to its named operation. +from freetoken.daemon.readiness import wait_for_ready +# What: import serve probe for test fresh health does not reuse previous model cache using freetoken and daemon and proxy and serve probe; why: test_fresh_health_does_not_reuse_previous_model_cache uses serve probe, making that imported dependency available to its named operation. +from freetoken.daemon.proxy import ServeProbe +# What: import build app for test profile readiness failure recovery end to end using freetoken and daemon and app and build app; why: test_profile_readiness_failure_recovery_end_to_end uses build app, making that imported dependency available to its named operation. +from freetoken.daemon.app import build_app +# What: import log ring for test switch launch recovery is 503 not success using freetoken and daemon and logring and log ring; why: test_switch_launch_recovery_is_503_not_success uses log ring, making that imported dependency available to its named operation. +from freetoken.daemon.logring import LogRing +# What: import switch launch error for switch using freetoken and daemon and serve manager and switch launch error; why: switch uses switch launch error, making that imported dependency available to its named operation. +from freetoken.daemon.serve_manager import SwitchLaunchError +# What: arrange from tests daemon test daemon serve manager import Spawner make manager for the scenario; why: test daemon serve manager import spawner make manager in test requires this concrete input or helper state before exercising the behavior under test. +from tests.daemon.test_daemon_serve_manager import Spawner, make_manager + + +# What: parameterize test_profile_readiness_failure_recovery_end_to_end with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test profile readiness failure recovery end to end. +@pytest.mark.parametrize("failure", ["error", "timeout", "operator-stop", "recovery-error"]) +# What: define the test_profile_readiness_failure_recovery_end_to_end test around tmp path and failure; why: this test groups the arrange, act, and assertions that protect the profile readiness failure recovery end to end outcome. +def test_profile_readiness_failure_recovery_end_to_end(tmp_path, failure): + # What: act by calling Spawner and capture sp; why: the profile readiness failure recovery end to end test asserts the response, state, or failure produced by this call. + sp = Spawner() + # What: act by calling make_manager and capture manager and and ring; why: the profile readiness failure recovery end to end test asserts the response, state, or failure produced by this call. + manager, _, ring = make_manager(tmp_path, sp, + # What: act by evaluating signal fn lambda pid sig sp by pid pid die; why: test swap regressions test profile readiness failure recovery end to end captures the behavior or response that its following assertions inspect. + signal_fn=lambda pid, sig: sp.by_pid(pid).die()) + # What: arrange the exact manager start previous original fixture fragment; why: the profile readiness failure recovery end to end scenario feeds this byte-preserved fragment through manager.start("previous", 1922, ["--original"]) before asserting its protocol or parser result. + manager.start("previous", 1922, ["--original"]) + # What: arrange path as tmp path and models and toml; why: the profile readiness failure recovery end to end test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text models bad nmodel replacement nport nready timeout s fixture fragment; why: the profile readiness failure recovery end to end scenario feeds this byte-preserved fragment through path.write_text("[models.bad]\nmodel = 'replacement'\nport = 1923\nready before asserting its p. + path.write_text("[models.bad]\nmodel = 'replacement'\nport = 1923\nready_timeout_s = 1\n", + # What: arrange the exact encoding utf 8 fixture fragment; why: the profile readiness failure recovery end to end scenario feeds this byte-preserved fragment through encoding="utf-8") before asserting its protocol or parser result. + encoding="utf-8") + # What: act by calling threading.Event and capture entered and release; why: the profile readiness failure recovery end to end test asserts the response, state, or failure produced by this call. + entered, release = threading.Event(), threading.Event() + + # What: define Probe as the owner of fresh_health; why: daemon callers use this class boundary so those methods share one probe state invariant. + class Probe: + # What: arrange the def fresh health self port test helper boundary; why: test swap regressions test profile readiness failure recovery end to end uses this local double to isolate the behavior checked by its assertions. + def fresh_health(self, port): + # What: act on port before status and failure; why: the profile readiness failure recovery end to end scenario admits status and failure only for this predicate and excludes the opposite state. + if port == 1922: + # What: arrange status as failure and error and ok and recovery error; why: the profile readiness failure recovery end to end test consumes this named precondition before exercising the behavior. + status = "error" if failure == "recovery-error" else "ok" + # What: act on failure before set and entered; why: the profile readiness failure recovery end to end scenario admits set and entered only for this predicate and excludes the opposite state. + elif failure == "operator-stop": + # What: act by calling entered.set with the declared inputs; why: the profile readiness failure recovery end to end scenario observes the entered.set return value during assert release wait. + entered.set() + # What: assert that release wait 5; why: this assertion protects the profile readiness failure recovery end to end regression after the test's arranged inputs and exercised call. + assert release.wait(5) + # What: arrange status as error; why: the profile readiness failure recovery end to end test consumes this named precondition before exercising the behavior. + status = "error" + # What: select the remaining branch that performs status loading if failure timeout else; why: fresh_health covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: arrange status as failure and loading and error and timeout; why: the profile readiness failure recovery end to end test consumes this named precondition before exercising the behavior. + status = "loading" if failure == "timeout" else "error" + # What: arrange the helper response as reachable True status status maintenance serving; why: test swap regressions test profile readiness failure recovery end to end feeds this result into the behavior whose outcome is asserted. + return {"reachable": True, "status": status, "maintenance": "serving"} + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app manager manager ring ring; why: test_profile_readiness_failure_recovery_end_to_end releases this resource or lock after app build app manager manager ring ring on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the profile readiness failure recovery end to end test asserts the response, state, or failure produced by this call. + app = build_app(manager=manager, ring=ring, probe=Probe(), + # What: arrange footprint fn lambda pid lifecycle pool lifecycle for the scenario; why: test swap regressions test profile readiness failure recovery end to end requires this concrete input or helper state before exercising the behavior under test. + footprint_fn=lambda pid: {}, lifecycle_pool=lifecycle, + # What: arrange proxy pool to ModelCatalog.load; why: the profile readiness failure recovery end to end scenario binds this proxy value to ModelCatalog.load's proxy pool input. + proxy_pool=proxy, catalog=ModelCatalog.load(str(path))) + # What: arrange with TestClient app as client ThreadPoolExecutor 1 as requests for the scenario; why: test client app as client thread pool executor 1 as in test requires this concrete input or helper state before exercising the behavior under test. + with TestClient(app) as client, ThreadPoolExecutor(1) as requests: + # What: act by calling requests.submit and capture response task; why: the profile readiness failure recovery end to end test asserts the response, state, or failure produced by this call. + response_task = requests.submit(client.post, "/engine/switch-profile", json={"name": "bad"}) + # What: act on failure before wait and status code and set and entered and release; why: the profile readiness failure recovery end to end scenario admits wait and status code and set and entered and release only for this predicate and excludes the opposite state. + if failure == "operator-stop": + # What: establish the handler boundary for the protected operation; why: test_profile_readiness_failure_recovery_end_to_end routes failures to the unconditional cleanup block while preserving cleanup and success flow. + try: + # What: assert that entered wait 5; why: this assertion protects the profile readiness failure recovery end to end regression after the test's arranged inputs and exercised call. + assert entered.wait(5) + # The only proxy worker is blocked, but lifecycle remains available. + # What: assert that client post engine stop json status code equals 200; why: this assertion protects the profile readiness failure recovery end to end regression after the test's arranged inputs and exercised call. + assert client.post("/engine/stop", json={}).status_code == 200 + # What: run release set on every exit path; why: test_profile_readiness_failure_recovery_end_to_end performs this cleanup after success, rejection, or exception so resources and accounting cannot remain stranded. + finally: + # What: act by calling release.set with the declared inputs; why: the profile readiness failure recovery end to end scenario observes the release.set return value during response response task result timeout. + release.set() + # What: act by calling response_task.result and capture response; why: the profile readiness failure recovery end to end test asserts the response, state, or failure produced by this call. + response = response_task.result(timeout=10) + # What: assert that response status code equals 503; why: this assertion protects the profile readiness failure recovery end to end regression after the test's arranged inputs and exercised call. + assert response.status_code == 503 + # What: act by calling response.json and capture doc; why: the profile readiness failure recovery end to end test asserts the response, state, or failure produced by this call. + doc = response.json() + # What: assert that doc readiness ready is false; why: this assertion protects the profile readiness failure recovery end to end regression after the test's arranged inputs and exercised call. + assert not doc["readiness"]["ready"] + # What: act on failure before doc; why: the profile readiness failure recovery end to end scenario admits doc only for this predicate and excludes the opposite state. + if failure == "operator-stop": + # What: assert that doc rollback reason equals superseded; why: this assertion protects the profile readiness failure recovery end to end regression after the test's arranged inputs and exercised call. + assert doc["rollback"]["reason"] == "superseded" + # What: assert that manager status running is false; why: this assertion protects the profile readiness failure recovery end to end regression after the test's arranged inputs and exercised call. + assert not manager.status()["running"] + # What: select the remaining branch that performs assert doc rollback launched; why: test_profile_readiness_failure_recovery_end_to_end covers the state excluded by the preceding predicate without conflating the two outcomes. + else: + # What: assert that doc rollback launched; why: this assertion protects the profile readiness failure recovery end to end regression after the test's arranged inputs and exercised call. + assert doc["rollback"]["launched"] + # What: assert that doc rollback readiness ready is failure differs from recovery error; why: this assertion protects the profile readiness failure recovery end to end regression after the test's arranged inputs and exercised call. + assert doc["rollback"]["readiness"]["ready"] is (failure != "recovery-error") + # What: assert that manager status model equals previous; why: this assertion protects the profile readiness failure recovery end to end regression after the test's arranged inputs and exercised call. + assert manager.status()["model"] == "previous" + # What: act by calling manager.stop with the declared inputs; why: the profile readiness failure recovery end to end scenario observes the manager.stop return value during the enclosing return. + manager.stop() + + +# What: parameterize test_switch_launch_recovery_is_503_not_success with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test switch launch recovery is 503 not success. +@pytest.mark.parametrize("route,body", [ + # What: arrange the model field as bad; why: test_switch_launch_recovery_is_503_not_success sends this field through ("/engine/switch", {"model": "bad"}) so the router selects the canonical model or alias for upstream dispatch. + ("/engine/switch", {"model": "bad"}), + # What: arrange the name field as bad; why: test_switch_launch_recovery_is_503_not_success carries name into ("/engine/switch-profile", {"name": "bad"}). + ("/engine/switch-profile", {"name": "bad"}), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_switch_launch_recovery_is_503_not_success groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_switch_launch_recovery_is_503_not_success test around tmp path and route and body; why: this test groups the arrange, act, and assertions that protect the switch launch recovery is 503 not success outcome. +def test_switch_launch_recovery_is_503_not_success(tmp_path, route, body): + # What: arrange path as tmp path and models and toml; why: the switch launch recovery is 503 not success test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text models bad nmodel bad n encoding fixture fragment; why: the switch launch recovery is 503 not success scenario feeds this byte-preserved fragment through path.write_text("[models.bad]\nmodel = 'bad'\n", encoding="utf-8") before asserting its protocol or parser result. + path.write_text("[models.bad]\nmodel = 'bad'\n", encoding="utf-8") + + # What: define Manager as the owner of status and switch; why: daemon callers use this class boundary so those methods share one manager state invariant. + class Manager: + # What: define the status test helper around captured fixture state; why: the switch launch recovery is 503 not success scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def status(self): + # What: arrange the port field as 1922; why: Manager.status carries port into return {"port": 1922}. + return {"port": 1922} + + # What: define the switch test helper around captured fixture state; why: the switch launch recovery is 503 not success scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def switch(self, *args): + # What: raise SwitchLaunchError for the caller; why: Manager.switch stops this rejected path before it can mutate state, dispatch work, or report success. + raise SwitchLaunchError(OSError("failed"), + # What: arrange the attempted field as true; why: Manager.switch carries attempted into {"attempted": True, "launched": True, "pid": 42}, None). + {"attempted": True, "launched": True, "pid": 42}, None) + + # What: arrange switch for readiness as switch; why: the switch launch recovery is 503 not success test consumes this named precondition before exercising the behavior. + switch_for_readiness = switch + + # What: enter the ThreadPoolExecutor and ThreadPoolExecutor managed context before app build app manager manager ring log ring; why: test_switch_launch_recovery_is_503_not_success releases this resource or lock after app build app manager manager ring log ring on both success and failure paths. + with ThreadPoolExecutor(1) as lifecycle, ThreadPoolExecutor(1) as proxy: + # What: act by calling build_app and capture app; why: the switch launch recovery is 503 not success test asserts the response, state, or failure produced by this call. + app = build_app(manager=Manager(), ring=LogRing(), probe=None, + # What: arrange the pid input for test_switch_launch_recovery_is_503_not_success; why: test_switch_launch_recovery_is_503_not_success consumes pid during signature binding, so callers must bind it with the other signature inputs. + footprint_fn=lambda pid: {}, lifecycle_pool=lifecycle, + # What: arrange proxy pool to ModelCatalog.load; why: the switch launch recovery is 503 not success scenario binds this proxy value to ModelCatalog.load's proxy pool input. + proxy_pool=proxy, catalog=ModelCatalog.load(str(path))) + # What: enter the TestClient managed context before response client post route json body; why: test_switch_launch_recovery_is_503_not_success releases this resource or lock after response client post route json body on both success and failure paths. + with TestClient(app) as client: + # What: act by calling client.post and capture response; why: the switch launch recovery is 503 not success test asserts the response, state, or failure produced by this call. + response = client.post(route, json=body) + # What: assert that response status code equals 503; why: this assertion protects the switch launch recovery is 503 not success regression after the test's arranged inputs and exercised call. + assert response.status_code == 503 + # What: assert that response json code equals switch launch failed; why: this assertion protects the switch launch recovery is 503 not success regression after the test's arranged inputs and exercised call. + assert response.json()["code"] == "switch_launch_failed" + # What: assert that response json rollback launched is true; why: this assertion protects the switch launch recovery is 503 not success regression after the test's arranged inputs and exercised call. + assert response.json()["rollback"]["launched"] is True + # What: assert that ready is absent from response json rollback; why: this assertion protects the switch launch recovery is 503 not success regression after the test's arranged inputs and exercised call. + assert "ready" not in response.json()["rollback"] + + +# What: parameterize test_catalog_rejects_owned_option_aliases with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test catalog rejects owned option aliases. +@pytest.mark.parametrize("arg", ["--model-path", "--model-path=other", "--model-p", "--mod=other", "--por=8", "--"]) +# What: define the test_catalog_rejects_owned_option_aliases test around tmp path and arg; why: this test groups the arrange, act, and assertions that protect the catalog rejects owned option aliases outcome. +def test_catalog_rejects_owned_option_aliases(tmp_path, arg): + # What: arrange path as tmp path and models and toml; why: the catalog rejects owned option aliases test consumes this named precondition before exercising the behavior. + path = tmp_path / "models.toml" + # What: arrange the exact path write text f models bad nmodel m nargs fixture fragment; why: the catalog rejects owned option aliases scenario feeds this byte-preserved fragment through path.write_text(f"[models.bad]\nmodel = 'm'\nargs = ['{arg}']\n", encodi before asserting its protocol or parser result. + path.write_text(f"[models.bad]\nmodel = 'm'\nargs = ['{arg}']\n", encoding="utf-8") + # What: assert the pytest.raises failure context; why: the catalog rejects owned option aliases scenario rejects the unsafe input through this exact exception boundary. + with pytest.raises(CatalogError, match="must not set"): + # What: act by calling ModelCatalog.load with str and path; why: the catalog rejects owned option aliases scenario observes the ModelCatalog.load return value during the enclosing return. + ModelCatalog.load(str(path)) + + +# What: define the test_readiness_rechecks_generation_after_probe test around local fixtures; why: this test groups the arrange, act, and assertions that protect the readiness rechecks generation after probe outcome. +def test_readiness_rechecks_generation_after_probe(): + # What: define Manager as the owner of status; why: daemon callers use this class boundary so those methods share one manager state invariant. + class Manager: + # What: arrange pid as 44; why: the readiness rechecks generation after probe test consumes this named precondition before exercising the behavior. + pid = 44 + + # What: define the status test helper around captured fixture state; why: the readiness rechecks generation after probe scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def status(self): + # What: arrange the running field as true; why: Manager.status carries running into return {"running": True, "pid": self.pid}. + return {"running": True, "pid": self.pid} + + # What: act by calling Manager and capture manager; why: the readiness rechecks generation after probe test asserts the response, state, or failure produced by this call. + manager = Manager() + + # What: define Probe as the owner of fresh_health; why: daemon callers use this class boundary so those methods share one probe state invariant. + class Probe: + # What: define an uncached health probe for the active engine port; why: readiness checks must bypass a replaced generation's cached response before accepting the new process. + def fresh_health(self, port): + # What: arrange pid as 45; why: the readiness rechecks generation after probe test consumes this named precondition before exercising the behavior. + manager.pid = 45 + # What: arrange the reachable field as true; why: Probe.fresh_health carries reachable into return {"reachable": True, "status": "ok"}. + return {"reachable": True, "status": "ok"} + + # What: act by calling wait_for_ready and capture result; why: the readiness rechecks generation after probe test asserts the response, state, or failure produced by this call. + result = wait_for_ready(manager, Probe(), pid=44, port=1922, timeout_s=1) + # What: assert that result ready is false; why: this assertion protects the readiness rechecks generation after probe regression after the test's arranged inputs and exercised call. + assert result["ready"] is False + # What: assert that result reason equals superseded; why: this assertion protects the readiness rechecks generation after probe regression after the test's arranged inputs and exercised call. + assert result["reason"] == "superseded" + + +# What: define the test_profile_client_reports_legacy_readiness_failure test around monkeypatch; why: this test groups the arrange, act, and assertions that protect the profile client reports legacy readiness failure outcome. +def test_profile_client_reports_legacy_readiness_failure(monkeypatch): + # What: arrange seen as the fixture input; why: the profile client reports legacy readiness failure test consumes this named precondition before exercising the behavior. + seen = {} + + # What: define the request test helper around captured fixture state; why: the profile client reports legacy readiness failure scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def request(*args, **kwargs): + # What: act by calling seen.update with kwargs; why: the profile client reports legacy readiness failure scenario observes the seen.update return value during return readiness ready reason engine error. + seen.update(kwargs) + # What: arrange the readiness field as ready and reason and false and engine error; why: request carries readiness into return {"readiness": {"ready": False, "reason": "engine-error"}}. + return {"readiness": {"ready": False, "reason": "engine-error"}} + + # What: arrange the exact monkeypatch setattr daemon client request json request fixture fragment; why: the profile client reports legacy readiness failure scenario feeds this byte-preserved fragment through monkeypatch.setattr(daemon_client, "_request_json", request) before asserting its protocol or parser result. + monkeypatch.setattr(daemon_client, "_request_json", request) + # What: assert that daemon client main start profile coding equals 1; why: this assertion protects the profile client reports legacy readiness failure regression after the test's arranged inputs and exercised call. + assert daemon_client.main(["start-profile", "coding"]) == 1 + # What: assert that seen timeout equals daemon client default profile timeout; why: this assertion protects the profile client reports legacy readiness failure regression after the test's arranged inputs and exercised call. + assert seen["timeout"] == daemon_client.DEFAULT_PROFILE_TIMEOUT + + +# What: parameterize test_routing_profile_client_uses_atomic_selection_endpoint with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test routing profile client uses atomic selection endpoint. +@pytest.mark.parametrize("argv,expected_body", [ + # What: arrange the name field as coding; why: test_routing_profile_client_uses_atomic_selection_endpoint carries name into (["activate-routing-profile", "coding"], {"name": "coding"}). + (["activate-routing-profile", "coding"], {"name": "coding"}), + # What: arrange the name field as the fixture input; why: test_routing_profile_client_uses_atomic_selection_endpoint carries name into (["clear-routing-profile"], {"name": None}). + (["clear-routing-profile"], {"name": None}), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_routing_profile_client_uses_atomic_selection_endpoint groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_routing_profile_client_uses_atomic_selection_endpoint test around monkeypatch and capsys and argv and expected body; why: this test groups the arrange, act, and assertions that protect the routing profile client uses atomic selection endpoint outcome. +def test_routing_profile_client_uses_atomic_selection_endpoint( + # What: arrange the monkeypatch input for test_routing_profile_client_uses_atomic_selection_endpoint; why: test_routing_profile_client_uses_atomic_selection_endpoint consumes monkeypatch during monkeypatch setattr daemon client request json request, so callers must bind it with the other signature inputs. + monkeypatch, capsys, argv, expected_body +# What: arrange the grouped source fragment for the scenario; why: test routing profile client uses atomic selection endpoint requires this concrete input or helper state before exercising the behavior under test. +): + # What: arrange seen as the fixture input; why: the routing profile client uses atomic selection endpoint test consumes this named precondition before exercising the behavior. + seen = {} + + # What: define the request test helper around method and url and path; why: the routing profile client uses atomic selection endpoint scenario calls this helper to produce or observe the exact behavior checked by its assertions. + def request(method, url, path, **kwargs): + # What: arrange method to seen.update; why: the routing profile client uses atomic selection endpoint scenario binds this method value to seen.update's method input. + seen.update(method=method, url=url, path=path, **kwargs) + # What: arrange the active field as expected body and name; why: request carries active into return {"active": expected_body["name"]}. + return {"active": expected_body["name"]} + + # What: arrange the exact monkeypatch setattr daemon client request json request fixture fragment; why: the routing profile client uses atomic selection endpoint scenario feeds this byte-preserved fragment through monkeypatch.setattr(daemon_client, "_request_json", request) before asserting its protocol or parser. + monkeypatch.setattr(daemon_client, "_request_json", request) + + # What: assert that daemon client main argv equals 0; why: this assertion protects the routing profile client uses atomic selection endpoint regression after the test's arranged inputs and exercised call. + assert daemon_client.main(argv) == 0 + # What: assert that seen method equals put; why: this assertion protects the routing profile client uses atomic selection endpoint regression after the test's arranged inputs and exercised call. + assert seen["method"] == "PUT" + # What: assert that seen path equals router profiles active; why: this assertion protects the routing profile client uses atomic selection endpoint regression after the test's arranged inputs and exercised call. + assert seen["path"] == "/router/profiles/active" + # What: assert that seen body equals expected body; why: this assertion protects the routing profile client uses atomic selection endpoint regression after the test's arranged inputs and exercised call. + assert seen["body"] == expected_body + # What: assert that json loads capsys readouterr out active equals expected body name; why: this assertion protects the routing profile client uses atomic selection endpoint regression after the test's arranged inputs and exercised call. + assert json.loads(capsys.readouterr().out)["active"] == expected_body["name"] + + +# What: define the test_fresh_health_does_not_reuse_previous_model_cache test around local fixtures; why: this test groups the arrange, act, and assertions that protect the fresh health does not reuse previous model cache outcome. +def test_fresh_health_does_not_reuse_previous_model_cache(): + # What: act by calling iter and capture docs; why: the fresh health does not reuse previous model cache test asserts the response, state, or failure produced by this call. + docs = iter([{"status": "ok", "instance_id": "old"}, {"status": "loading", "instance_id": "new"}]) + # What: act by calling ServeProbe and capture probe; why: the fresh health does not reuse previous model cache test asserts the response, state, or failure produced by this call. + probe = ServeProbe(opener=lambda *_: next(docs), ttl_s=100) + # What: assert that probe health 1922 status equals ok; why: this assertion protects the fresh health does not reuse previous model cache regression after the test's arranged inputs and exercised call. + assert probe.health(1922)["status"] == "ok" + # What: assert that probe fresh health 1922 status equals loading; why: this assertion protects the fresh health does not reuse previous model cache regression after the test's arranged inputs and exercised call. + assert probe.fresh_health(1922)["status"] == "loading" + + +# What: parameterize test_readiness_http_contract with the listed cases; why: pytest reruns the same arrange, act, and assertions for each input protecting test readiness http contract. +@pytest.mark.parametrize("status,maintenance,expected", [ + # What: arrange the loading error portion of the enclosing predicate; why: this clause remains in the readiness http contract scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("loading", None, 503), ("error", None, 503), + # What: arrange the ok draining ok serving portion of the enclosing predicate; why: this clause remains in the readiness http contract scenario\'s enclosing expression so its grouping and evaluation order stay intact. + ("ok", "draining", 503), ("ok", "serving", 200), +# What: arrange the pytest.mark.parametrize call with ordered positional inputs; why: test_readiness_http_contract groups the supplied clauses as one pytest.mark.parametrize call before its value is consumed. +]) +# What: define the test_readiness_http_contract test around status and maintenance and expected; why: this test groups the arrange, act, and assertions that protect the readiness http contract outcome. +def test_readiness_http_contract(status, maintenance, expected): + # Execute the actual handlers, excluding unrelated torch-dependent metrics + # imports. This is a CPU contract test, not a full serving integration test. + # What: act by calling Path and capture source; why: the readiness http contract test asserts the response, state, or failure produced by this call. + source = Path(__file__).parents[2] / "python/freetoken/server/control_api.py" + # What: act by calling ast.parse and capture module; why: the readiness http contract test asserts the response, state, or failure produced by this call. + module = ast.parse(source.read_text(encoding="utf-8")) + # What: act by calling next and capture register; why: the readiness http contract test asserts the response, state, or failure produced by this call. + register = next(n for n in module.body if isinstance(n, ast.FunctionDef) and n.name == "register_control_routes") + # What: act by calling isinstance and capture routes; why: the readiness http contract test asserts the response, state, or failure produced by this call. + routes = [n for n in register.body if isinstance(n, ast.AsyncFunctionDef) and n.name in {"health", "ready"}] + # What: act by calling FastAPI and capture app; why: the readiness http contract test asserts the response, state, or failure produced by this call. + app = FastAPI() + # What: arrange doc as status and maintenance and status and maintenance; why: the readiness http contract test consumes this named precondition before exercising the behavior. + doc = {"status": status, "maintenance": maintenance} + # What: arrange namespace as app and doc and app and build health and get state; why: the readiness http contract test consumes this named precondition before exercising the behavior. + namespace = {"app": app, "build_health": lambda *_: doc, "get_state": lambda: None} + # What: arrange the exact exec compile ast module body routes type ignores fixture fragment; why: the readiness http contract scenario feeds this byte-preserved fragment through exec(compile(ast.Module(body=routes, type_ignores=[]), str(source), "exe before asserting its protocol or parser result. + exec(compile(ast.Module(body=routes, type_ignores=[]), str(source), "exec"), namespace) + # What: enter the TestClient managed context before assert client get health status code; why: test_readiness_http_contract releases this resource or lock after assert client get health status code on both success and failure paths. + with TestClient(app) as client: + # What: assert that client get health status code equals 200; why: this assertion protects the readiness http contract regression after the test's arranged inputs and exercised call. + assert client.get("/health").status_code == 200 + # What: act by calling client.get and capture response; why: the readiness http contract test asserts the response, state, or failure produced by this call. + response = client.get("/ready") + # What: assert that response status code equals expected; why: this assertion protects the readiness http contract regression after the test's arranged inputs and exercised call. + assert response.status_code == expected + # What: assert that response json equals doc; why: this assertion protects the readiness http contract regression after the test's arranged inputs and exercised call. + assert response.json() == doc diff --git a/tests/e2e/test_aime.py b/tests/e2e/test_aime.py index 0a4c3a67bc..f28035395b 100644 --- a/tests/e2e/test_aime.py +++ b/tests/e2e/test_aime.py @@ -18,6 +18,10 @@ FREETOKEN_AIME_SAMPLES pass@N sample count when sampling (default 3) FREETOKEN_AIME_MIN_FREE_GIB free-GPU-memory gate (default 70; raise for a big resident model) FREETOKEN_TEST_MOE_CACHE_SIZE >0 switches to the offload MoE backend (fp8/GLM/MiniMax) + FREETOKEN_TEST_MOE_BACKEND offload-family backend override (default offload; e.g. hybrid) + FREETOKEN_TEST_MOE_CPU_THREADS CPU MoE worker threads (0 = physical cores) + FREETOKEN_TEST_MOE_CACHE_AUTO 1 sizes the MoE cache from free VRAM instead of MOE_CACHE_SIZE + FREETOKEN_TEST_KV_RESERVE KV token floor reserved before auto-cache fills experts FREETOKEN_TEST_MEM_RATIO offload memory ratio (default 0.9) fp8 / offload recipe: point FREETOKEN_TEST_MODEL at the fp8 checkpoint dir and set @@ -164,10 +168,14 @@ def build_llm(model_path: Path) -> LLM: max_extend_tokens=8192, ) cache_size = int(os.environ.get("FREETOKEN_TEST_MOE_CACHE_SIZE", "0")) - if cache_size > 0: + cache_auto = os.environ.get("FREETOKEN_TEST_MOE_CACHE_AUTO") == "1" + if cache_size > 0 or cache_auto: kwargs.update( - moe_backend="offload", + moe_backend=os.environ.get("FREETOKEN_TEST_MOE_BACKEND", "offload"), + moe_cpu_threads=int(os.environ.get("FREETOKEN_TEST_MOE_CPU_THREADS", "0")), + moe_cache_auto=cache_auto, moe_cache_size=cache_size, + kv_reserve_tokens=int(os.environ.get("FREETOKEN_TEST_KV_RESERVE", "8192")), moe_cache_policy="lru", memory_ratio=float(os.environ.get("FREETOKEN_TEST_MEM_RATIO", "0.9")), max_seq_len_override=max_tokens() + 2048, diff --git a/tests/engine/test_attention_backend_matrix.py b/tests/engine/test_attention_backend_matrix.py index f27640b9ff..ef7dc7ff85 100644 --- a/tests/engine/test_attention_backend_matrix.py +++ b/tests/engine/test_attention_backend_matrix.py @@ -24,7 +24,7 @@ ) -def _spec(name, attn_type, *, mla=False, sliding_window=None, index_head_dim=0): +def _spec(name, attn_type, *, mla=False, sliding_window=None, index_head_dim=0, index_ratio=1): return KVCacheGroupSpec( name=name, layer_ids=(0, 1), @@ -34,6 +34,7 @@ def _spec(name, attn_type, *, mla=False, sliding_window=None, index_head_dim=0): mla=mla, index_head_dim=index_head_dim, num_index_layers=2 if index_head_dim else 0, + index_ratio=index_ratio, attn_type=attn_type, ) @@ -67,6 +68,11 @@ def _model_config(kind): elif kind == "bsa": # MiniMax-M3 shape: one FULL-family group, mla=False + index dims -> BSA. specs = (_spec("full", AttnType.BSA, index_head_dim=128),) + elif kind == "qsa": + # Qwen3.8-Flash-Next shape: hybrid-linear + one FULL-family group whose index keys + # are compressed index_ratio:1 -> QSA. + mc.has_linear_attention = True + specs = (_spec("full", AttnType.QSA, index_head_dim=128, index_ratio=4),) elif kind == "linear_hybrid": mc.has_linear_attention = True specs = (_spec("full", AttnType.FULL),) @@ -109,6 +115,7 @@ def _patch_env(monkeypatch, *, major=9, flashinfer=True, sgl=True): ("dsa", "dsa"), # MLA + DSA indexer (GLM-5.2 shape) ("dsv4", "dsv4_sparse"), ("bsa", "m3_sparse"), # MiniMax-M3 block-sparse GQA + ("qsa", "qsa_sparse"), # Qwen3.8-Flash-Next compressed-block sparse ], ) def test_auto_resolves_per_type(monkeypatch, kind, expected): @@ -143,6 +150,39 @@ def test_bsa_rejects_float32_dtype(monkeypatch): _adjust_config(config) +def test_auto_qsa_sets_page_size_64(monkeypatch): + # qsa_sparse registers page_sizes=(64,) (a 4-token compress group must never straddle + # a page); the generic backend page-size coercion takes the default 1 to 64. + from freetoken.engine.engine import _adjust_config + + _patch_env(monkeypatch) + config = _config("qsa", attention_backend="auto") + assert config.page_size == 1 + _adjust_config(config) + assert config.page_size == 64 + + +def test_qsa_coerces_explicit_page_size(monkeypatch): + # Same policy as m3_sparse (page_sizes=(128,)): an unsupported explicit value is + # coerced to the backend's page size with a warning, not rejected. + from freetoken.engine.engine import _adjust_config + + _patch_env(monkeypatch) + config = _config("qsa", attention_backend="auto", page_size=16) + _adjust_config(config) + assert config.page_size == 64 + + +def test_qsa_rejects_float32_dtype(monkeypatch): + from freetoken.engine.engine import _adjust_config + + _patch_env(monkeypatch) + config = _config("qsa", attention_backend="auto") + object.__setattr__(config, "dtype", torch.float32) + with pytest.raises(ValueError, match="16-bit"): + _adjust_config(config) + + def test_auto_dsv4_sets_window_page_size(monkeypatch): from freetoken.engine.engine import _adjust_config @@ -160,9 +200,16 @@ def test_auto_dsv4_sets_window_page_size(monkeypatch): ("full", "dsv4_sparse"), ("full", "m3_sparse"), ("swa", "dsa"), + ("full", "qsa_sparse"), # forward gates: generic backends on the BSA-locked model ("bsa", "fi"), ("bsa", "triton"), + # forward gates: generic and neighbouring sparse backends on the QSA-locked model + ("qsa", "fi"), + ("qsa", "fa"), + ("qsa", "triton"), + ("qsa", "m3_sparse"), + ("bsa", "qsa_sparse"), # forward gates: generic backends on type-locked models ("mla", "fi"), ("mla", "triton"), @@ -196,6 +243,7 @@ def test_illegal_combinations_rejected_at_config_time(monkeypatch, kind, backend ("mla", "dsa"), ("dsa", "dsa"), ("dsv4", "dsv4_sparse"), + ("qsa", "qsa_sparse"), ("swa", "triton"), ("full", "triton"), ("full", "fa,fi"), @@ -211,38 +259,15 @@ def test_legal_explicit_combinations_pass(monkeypatch, kind, backend): @pytest.mark.parametrize("kind", ["mla", "dsa"]) -def test_mla_requires_page_size_one(monkeypatch, kind): +def test_mla_auto_adjusts_page_size(monkeypatch, kind): + """Any --page-size is auto-adjusted (with a warning) to the layout's + required size: 1 for plain latent-KV, 64 for the kpool variant.""" from freetoken.engine.engine import _adjust_config _patch_env(monkeypatch) config = _config(kind, attention_backend="auto", page_size=16) - with pytest.raises(ValueError, match="page-size 1"): - _adjust_config(config) - - -def test_hybrid_linear_opt_out_rejects_backend(monkeypatch): - import dataclasses - - from freetoken.attention import attention_backend_info - from freetoken.engine import engine - from freetoken.engine.engine import _adjust_config - - _patch_env(monkeypatch) - real_info = attention_backend_info - - def _info(name): - info = real_info(name) - return dataclasses.replace(info, hybrid_linear_ok=False) if name == "fa" else info - - monkeypatch.setattr(engine, "attention_backend_info", _info) - # auto skips the opted-out backend ... - config = _config("linear_hybrid", attention_backend="auto") _adjust_config(config) - assert config.attention_backend == "fi" - # ... and an explicit choice of it is rejected - config = _config("linear_hybrid", attention_backend="fa") - with pytest.raises(ValueError, match="hybrid-linear"): - _adjust_config(config) + assert config.page_size == 1 def test_trtllm_page_size_coercion_is_part_aware(monkeypatch): @@ -372,3 +397,10 @@ def dataclasses_replace_groups(mc, groups): import dataclasses return dataclasses.replace(mc, attention_groups=groups) + + +def test_linear_attention_defaults_to_hybrid_radix(): + from freetoken.engine.engine import _resolve_cache_type + + assert _resolve_cache_type(True, "radix") == "hybrid_radix" + assert _resolve_cache_type(True, "naive") == "naive" diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.py index a164f0b4d9..9ac2a4f4cc 100644 --- a/tests/engine/test_cache_budget.py +++ b/tests/engine/test_cache_budget.py @@ -5,7 +5,10 @@ import pytest import torch +import os + from freetoken.engine.cache_budget import expert_bytes_per_slot, plan_cache_budget, resolve_moe_cache_auto +from freetoken.engine.engine import _pin_budget_bytes def test_moe_priority_fills_experts_up_to_total(): @@ -480,3 +483,20 @@ def test_adjust_config_rope_gate_exempts_dsv4(): cfg = _dsv4_adjust_cfg(max_seq_len_override=10_000_000) cfg.model_config.rotary_config = SimpleNamespace(max_position=1024) _adjust_config(cfg) # must not raise + + +# ---- _pin_budget_bytes: host bytes already pinned outside the expert banks ---- + + +def test_reserved_subtracts_from_the_cap(monkeypatch): + monkeypatch.setenv("FREETOKEN_PIN_BUDGET_GB", "2") + assert _pin_budget_bytes() == 2 * 2**30 + assert _pin_budget_bytes(reserved=2**30) == 2**30 + assert _pin_budget_bytes(reserved=4 * 2**30) == 0 + + +def test_uncapped_platform_stays_uncapped(monkeypatch): + monkeypatch.delenv("FREETOKEN_PIN_BUDGET_GB", raising=False) + if hasattr(os, "uname") and "microsoft" in os.uname().release.lower(): + pytest.skip("WSL caps pinning") + assert _pin_budget_bytes(reserved=2**30) is None diff --git a/tests/fixtures/deepseek_expert_index/model.safetensors.index.json b/tests/fixtures/deepseek_expert_index/model.safetensors.index.json new file mode 100644 index 0000000000..116d90759a --- /dev/null +++ b/tests/fixtures/deepseek_expert_index/model.safetensors.index.json @@ -0,0 +1,41 @@ +{ + "metadata": {"total_size": 13369344}, + "weight_map": { + "layers.0.ffn.experts.0.w1.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.0.w1.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.0.w2.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.0.w2.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.0.w3.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.0.w3.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.1.w1.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.1.w1.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.1.w2.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.1.w2.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.1.w3.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.1.w3.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.2.w1.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.2.w1.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.2.w2.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.2.w2.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.2.w3.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.2.w3.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.3.w1.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.3.w1.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.3.w2.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.3.w2.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.3.w3.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.3.w3.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.4.w1.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.4.w1.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.4.w2.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.4.w2.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.4.w3.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.4.w3.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.5.w1.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.5.w1.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.5.w2.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.5.w2.scale": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.5.w3.weight": "model-00002-of-00048.safetensors", + "layers.0.ffn.experts.5.w3.scale": "model-00002-of-00048.safetensors" + } +} diff --git a/tests/kernels/test_fp8_pertensor_linear.py b/tests/kernels/test_fp8_pertensor_linear.py index 6b158f523f..441da408f6 100644 --- a/tests/kernels/test_fp8_pertensor_linear.py +++ b/tests/kernels/test_fp8_pertensor_linear.py @@ -9,6 +9,10 @@ from __future__ import annotations +import subprocess +import sys +import textwrap + import pytest import torch @@ -125,3 +129,71 @@ def test_layer_load_marks_uniform_scale_and_optional_input_scale(): # a reload must not trip over the input_scale it kept from the first load single.load_state_dict({"weight": w8, "weight_scale": flat}) assert single.input_scale is None + + +@pytest.mark.skipif(not e4m3_native(), reason="torch._scaled_mm needs sm_89+") +@pytest.mark.parametrize("M", [1, 4, 64, 300]) +def test_per_part_path_matches_rowwise(M: int, monkeypatch): + """Where row-wise ``_scaled_mm`` is unsafe a fused projection runs one tensor-wise GEMM per + part instead. Same scheme, so the two paths agree up to accumulation order (~7e-4).""" + import freetoken.kernel.triton.fp8_pertensor_linear as mod + + K, part_rows = 2048, [1024, 256, 256] + w8, scale = _quant_parts(part_rows, K, seed=M) + x = torch.randn(M, K, device=DEV, dtype=torch.bfloat16) + input_scale = (x.abs().max().float() / 448.0).reshape(()) + + monkeypatch.setattr(mod, "rowwise_scaled_mm_ok", lambda: True) + y_row = mod.fp8_pertensor_linear(x, w8, scale, None, input_scale, False) + monkeypatch.setattr(mod, "rowwise_scaled_mm_ok", lambda: False) + y_part = mod.fp8_pertensor_linear(x, w8, scale, None, input_scale, False) + rel = ((y_part.float() - y_row.float()).norm() / y_row.float().norm()).item() + assert rel < 2e-3, rel + + +@pytest.mark.skipif(not e4m3_native(), reason="torch._scaled_mm needs sm_89+") +def test_fused_layer_forward_on_a_side_stream_completes(): + """Regression for #182 / #72 / #220: on sm_89 with torch < 2.12 a fused FP8 projection's + row-wise ``_scaled_mm`` issued from a non-default stream stalls the GPU (PyTorch's + CUTLASS row-wise kernel ignored the current stream; fixed upstream in 2.12). The layer + must take a path that completes on every supported build. Runs in a subprocess so a + stall fails the test instead of hanging the session.""" + script = textwrap.dedent(""" + import os, time, torch + from freetoken.kernel.triton.fp8_pertensor_linear import FP8, Fp8PerTensorColMerged + + torch.manual_seed(0) + K, parts = 2048, [8192, 512, 512] # a prefill-sized fused qkv + w8 = (torch.randn(sum(parts), K, device="cuda") * 8).clamp(-448, 448).to(FP8) + scale = torch.cat([torch.full((p,), 0.01 * (i + 1), device="cuda") + for i, p in enumerate(parts)]) + layer = Fp8PerTensorColMerged(K, parts) + layer.load_state_dict({"weight": w8, "weight_scale": scale, + "input_scale": torch.tensor(0.02, device="cuda")}) + x = torch.randn(2010, K, device="cuda", dtype=torch.bfloat16) # #182 shape + torch.cuda.synchronize() + + stream = torch.cuda.Stream() + events = [] + with torch.cuda.stream(stream): + for _ in range(128): + layer.forward(x) + ev = torch.cuda.Event() + ev.record(stream) + events.append(ev) + deadline = time.monotonic() + 30 + done = 0 + while time.monotonic() < deadline: + while done < len(events) and events[done].query(): + done += 1 + if done == len(events): + print("completed", done, flush=True) + os._exit(0) + time.sleep(0.05) + print("stalled at", done, "of", len(events), flush=True) + os._exit(124) # a normal exit would wait on the stuck kernel + """) + proc = subprocess.run( + [sys.executable, "-c", script], capture_output=True, text=True, timeout=300, + ) + assert proc.returncode == 0, f"rc={proc.returncode}\n{proc.stdout}\n{proc.stderr[-2000:]}" diff --git a/tests/kernels/test_gguf_hip_build_flags.py b/tests/kernels/test_gguf_hip_build_flags.py new file mode 100644 index 0000000000..9378d55fe1 --- /dev/null +++ b/tests/kernels/test_gguf_hip_build_flags.py @@ -0,0 +1,41 @@ +"""Unit coverage for the HIP-only GGUF extension compiler configuration. + +These tests do not invoke hipcc. They verify the environment that is prepared +before PyTorch's extension builder computes its target-specific cache key. +""" + +import os +from types import SimpleNamespace + +from freetoken.kernel import gguf + + +def test_hip_gguf_flags_pin_the_active_gfx_target(monkeypatch): + """A single-GPU HIP process derives gfx1151 when no target was configured.""" + monkeypatch.delenv("PYTORCH_ROCM_ARCH", raising=False) + monkeypatch.delenv("FREETOKEN_HIP_GGUF_FAST_MATH", raising=False) + monkeypatch.setattr(gguf.torch.cuda, "is_available", lambda: True) + monkeypatch.setattr( + gguf.torch.cuda, + "get_device_properties", + lambda _index: SimpleNamespace(gcnArchName="gfx1151:sramecc-:xnack-"), + ) + + # The HIP build always records the reviewed one-row launch shape in the + # compiler command, even when the caller did not set an experiment knob. + # Keeping this explicit makes the extension cache key and build evidence + # unambiguous for the default AMD serving path. + assert gguf._hip_gguf_cflags() == ["-O3", "-DGGML_CUDA_MMV_Y=1"] + assert gguf._hip_target_arch() == "gfx1151" + assert os.environ["PYTORCH_ROCM_ARCH"] == "gfx1151" + + +def test_hip_gguf_flags_preserve_an_explicit_multi_target_choice(monkeypatch): + """An explicit multi-target deployment choice is never replaced by auto-detection.""" + monkeypatch.setenv("PYTORCH_ROCM_ARCH", "gfx1100;gfx1151") + + # An explicit multi-target architecture choice must not remove the default + # one-row launch definition from the recorded HIP compile flags. + assert gguf._hip_gguf_cflags() == ["-O3", "-DGGML_CUDA_MMV_Y=1"] + assert gguf._hip_target_arch() == "gfx1100" + assert os.environ["PYTORCH_ROCM_ARCH"] == "gfx1100;gfx1151" diff --git a/tests/kernels/test_gguf_hip_flags.py b/tests/kernels/test_gguf_hip_flags.py new file mode 100644 index 0000000000..790063ccd1 --- /dev/null +++ b/tests/kernels/test_gguf_hip_flags.py @@ -0,0 +1,60 @@ +"""Unit tests for explicit HIP GGUF build-shape selection. + +These tests cover the host-side validation that protects the native GGUF +extension build. They do not compile a HIP module, allocate a GPU tensor, or +need a model checkpoint. Device execution and output parity belong to the +separate isolated candidate gate because they require the target AMD GPU. +""" + +from __future__ import annotations + +import pytest + +from freetoken.kernel import gguf + + +def test_hip_gguf_flags_keep_the_one_row_default(monkeypatch: pytest.MonkeyPatch) -> None: + """Preserve the reviewed one-row MMV launch when no experiment is selected.""" + + # Remove both knobs so the helper cannot inherit a developer-shell setting. + monkeypatch.delenv("FREETOKEN_GGUF_MMV_Y", raising=False) + monkeypatch.delenv("PYTORCH_ROCM_ARCH", raising=False) + # Stub target discovery to keep this test independent of local GPU access. + monkeypatch.setattr(gguf, "_hip_target_arch", lambda: "gfx1151") + + flags = gguf._hip_gguf_cflags() + + # The default must remain explicit in the compile command and target only + # the active AMD architecture discovered by the helper. + assert flags == ["-O3", "-DGGML_CUDA_MMV_Y=1"] + assert gguf.os.environ["PYTORCH_ROCM_ARCH"] == "gfx1151" + + +def test_hip_gguf_flags_allow_the_reviewed_row_grouping_candidates(monkeypatch: pytest.MonkeyPatch) -> None: + """Allow two, four, and eight rows while retaining the explicit default and reject path.""" + + monkeypatch.setenv("FREETOKEN_GGUF_MMV_Y", "2") + monkeypatch.setenv("PYTORCH_ROCM_ARCH", "gfx1151") + + assert gguf._hip_gguf_cflags() == ["-O3", "-DGGML_CUDA_MMV_Y=2"] + + # Four rows are a separately qualified RDNA4 experiment. This assertion + # proves the requested compile-time shape becomes part of the extension key. + monkeypatch.setenv("FREETOKEN_GGUF_MMV_Y", "4") + + assert gguf._hip_gguf_cflags() == ["-O3", "-DGGML_CUDA_MMV_Y=4"] + + # Preserve the prior eight-row RDNA4 screen while adding the intermediate + # geometry, so this candidate branch does not narrow test coverage. + monkeypatch.setenv("FREETOKEN_GGUF_MMV_Y", "8") + + assert gguf._hip_gguf_cflags() == ["-O3", "-DGGML_CUDA_MMV_Y=8"] + + +def test_hip_gguf_flags_reject_an_unreviewed_row_grouping(monkeypatch: pytest.MonkeyPatch) -> None: + """Fail closed rather than compiling an arbitrary MMV workgroup shape.""" + + monkeypatch.setenv("FREETOKEN_GGUF_MMV_Y", "3") + + with pytest.raises(RuntimeError, match="FREETOKEN_GGUF_MMV_Y must be 1, 2, 4, or 8"): + gguf._hip_gguf_cflags() diff --git a/tests/kernels/test_kda.py b/tests/kernels/test_kda.py new file mode 100644 index 0000000000..b217ee32ea --- /dev/null +++ b/tests/kernels/test_kda.py @@ -0,0 +1,106 @@ +"""Pins for OUR divergences from the upstream-vendored KDA kernels. + +The vendored kernels (freetoken/kernel/fla) are tested upstream and not re-tested +here. The eager reference replicates their exact math (safe gate ``gk = +lower_bound * sigmoid(exp(A_log) * (g_raw + dt_bias))``, ``beta = +sigmoid(beta_raw)``, in-loop q/k l2norm, per-channel-decayed delta rule on a +[V, K] state) so a divergence pin can assert numerics, not just reachability. +""" + +from __future__ import annotations + +import pytest +import torch + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + +H, D = 4, 128 # head count trimmed; head_dim matches GLM-5.3 (kernel specializes on D) +LOWER_BOUND = -5.0 +SCALE = D**-0.5 + + +def _l2norm(x: torch.Tensor) -> torch.Tensor: + return x / torch.sqrt((x * x).sum(-1, keepdim=True) + 1e-6) + + +def _reference( + q: torch.Tensor, # [T, H, D] bf16 + k: torch.Tensor, + v: torch.Tensor, + g_raw: torch.Tensor, # [T, H, D] bf16 + beta_raw: torch.Tensor, # [T, H] bf16 + a_log: torch.Tensor, # [H] fp32 + dt_bias: torch.Tensor, # [H*D] fp32 + h0: torch.Tensor | None = None, # [H, V, K] fp32 +) -> tuple[torch.Tensor, torch.Tensor]: + T = q.shape[0] + h = ( + h0.clone().float() + if h0 is not None + else torch.zeros(H, D, D, dtype=torch.float32, device=q.device) + ) + amp = a_log.float().exp().view(H, 1) + bias = dt_bias.float().view(H, D) + outs = [] + for t in range(T): + gk = LOWER_BOUND * torch.sigmoid(amp * (g_raw[t].float() + bias)) # [H, K] + h = h * gk.exp().unsqueeze(1) # decay per k-channel: [H, V, K] * [H, 1, K] + kt = _l2norm(k[t].float()) + v_err = v[t].float() - torch.einsum("hvk,hk->hv", h, kt) + v_err = v_err * torch.sigmoid(beta_raw[t].float()).unsqueeze(-1) + h = h + torch.einsum("hv,hk->hvk", v_err, kt) + qt = _l2norm(q[t].float()) * SCALE + outs.append(torch.einsum("hvk,hk->hv", h, qt)) + return torch.stack(outs), h + + +def _rand_inputs(T: int, seed: int = 0, device="cuda"): + torch.manual_seed(seed) + mk = lambda *s: torch.randn(*s, device=device, dtype=torch.bfloat16) + q, k, v, g_raw = mk(T, H, D), mk(T, H, D), mk(T, H, D), mk(T, H, D) + beta_raw = mk(T, H) + a_log = torch.randn(H, device=device, dtype=torch.float32) * 0.5 + dt_bias = torch.randn(H * D, device=device, dtype=torch.float32) * 0.5 + return q, k, v, g_raw, beta_raw, a_log, dt_bias + + +def _assert_close(ours, ref, tag, atol=2e-2, rtol=2e-2): + ours, ref = ours.float(), ref.float() + err = (ours - ref).abs().max().item() + rel = err / (ref.abs().max().item() + 1e-8) + assert torch.allclose(ours, ref, atol=atol, rtol=rtol), ( + f"{tag}: max abs err {err:.5f}, rel {rel:.5f}" + ) + + +def test_fused_recurrent_serves_slot_zero(): + """--cache-type naive keys state by raw table_idx, so a real request can sit + on slot 0. Upstream vLLM's kernel treats 0 as its NULL_BLOCK_ID sentinel and + silently skips it (state frozen, garbage output); our vendored copy diverges + to accept every non-negative slot (GDN-kernel parity). Same math as the + parametrized reference test, just on slot 0.""" + from freetoken.kernel.fla import fused_recurrent_kda + + T = 7 + q, k, v, g_raw, beta_raw, a_log, dt_bias = _rand_inputs(T) + ref_o, ref_h = _reference(q, k, v, g_raw, beta_raw, a_log, dt_bias) + + pool = torch.zeros(2, H, D, D, dtype=torch.float32, device="cuda") + indices = torch.zeros((1, T), dtype=torch.int64, device="cuda") # slot 0 + cu = torch.tensor([0, T], dtype=torch.int32, device="cuda") + o, _ = fused_recurrent_kda( + q=q.unsqueeze(0), k=k.unsqueeze(0), v=v.unsqueeze(0), + g=g_raw.unsqueeze(0), beta=beta_raw.unsqueeze(0), + initial_state=pool, + use_qk_l2norm_in_kernel=True, + cu_seqlens=cu, + ssm_state_indices=indices, + sigmoid_beta=True, + a_log=a_log, + g_bias=dt_bias, + compute_gate=True, + lower_bound=LOWER_BOUND, + ) + _assert_close(o[0], ref_o, "slot-0 output") + _assert_close(pool[0], ref_h, "slot-0 final state") + assert pool[1].abs().max().item() == 0.0 # only slot 0 was touched diff --git a/tests/kernels/test_pinned_tensor.py b/tests/kernels/test_pinned_tensor.py index e61108fd53..396bd9d60e 100644 --- a/tests/kernels/test_pinned_tensor.py +++ b/tests/kernels/test_pinned_tensor.py @@ -86,6 +86,40 @@ def fail_jit_load(*args, **kwargs): torch.testing.assert_close(output, torch.full_like(output, -1.0)) +def test_fused_copy_grid_selection_is_bounded_to_cached_variants(monkeypatch): + """Only explicit AOT grid widths may be selected by the service environment.""" + + import freetoken.kernel.fast_index_copy as fast_index_copy + + monkeypatch.delenv(fast_index_copy.FUSED_COPY_BLOCKS_PER_BANK_ENV, raising=False) + assert fast_index_copy.fused_copy_blocks_per_bank() == 8 + + monkeypatch.setenv(fast_index_copy.FUSED_COPY_BLOCKS_PER_BANK_ENV, "64") + assert fast_index_copy.fused_copy_blocks_per_bank() == 64 + + monkeypatch.setenv(fast_index_copy.FUSED_COPY_BLOCKS_PER_BANK_ENV, "16") + with pytest.raises(ValueError, match="must be 8 or 64"): + fast_index_copy.fused_copy_blocks_per_bank() + + +def test_aot_catalog_excludes_legacy_rows_without_full_vector_transactions(): + """AOT must not ask HIP to compile templates rejected by their static assertion.""" + + from freetoken.kernel.aot import DEFAULT_FAST_INDEX_COPY_FEATURE_SIZES, default_kernel_specs + from freetoken.kernel.fast_index_copy import legacy_fast_index_copy_is_supported + + # These rows are valid only for the fused multi-bank path, not the legacy + # 128-byte vector kernel. The AOT catalog must omit them entirely so a + # strict no-JIT runtime never asks HIP to compile an invalid template. + assert {240, 400}.isdisjoint(DEFAULT_FAST_INDEX_COPY_FEATURE_SIZES) + assert not legacy_fast_index_copy_is_supported(240) + assert not legacy_fast_index_copy_is_supported(400) + + names = {spec.name for spec in default_kernel_specs()} + assert not any("fast_index_copy_240_" in name for name in names) + assert not any("fast_index_copy_400_" in name for name in names) + + def test_device_ptr_pinned_bank_resolves(): if not torch.cuda.is_available(): pytest.skip("needs CUDA") @@ -122,7 +156,11 @@ def test_host_device_ptr_is_identity_under_uva(): pytest.skip("non-UVA platform: host_device_ptr rejects unregistered memory instead") # Under UVA cudaHostGetDevicePointer degenerates to identity for any host pointer # (no registration validation); rejection of pageable memory only exists on - # non-identity platforms (Windows/WDDM), where the translation is real. + # non-identity CUDA platforms (Windows/WDDM), where the translation is real. + # HIP validates registration even when registered memory has an identity + # address on Linux. The pinned identity check above is the relevant test. + if torch.version.hip is not None: + return pageable = torch.empty(64, dtype=torch.uint8) ext = _load_pinned_extension() assert ext.host_device_ptr(pageable.data_ptr()) == pageable.data_ptr() diff --git a/tests/kernels/test_swiglu_clamp.py b/tests/kernels/test_swiglu_clamp.py new file mode 100644 index 0000000000..2515bcdd82 --- /dev/null +++ b/tests/kernels/test_swiglu_clamp.py @@ -0,0 +1,43 @@ +"""swiglu_clamp (GLM-5.3 ``swiglu_limit``) activation parity. + +Reference: vLLM's SiluAndMulWithClamp with alpha=1, beta=0 -- +``clamp(gate, max=L) * sigmoid(gate_clamped) * clamp(up, +-L)``. Checks the +Triton kernel, its distinction from swigluoai (the +1 up bias), and that the +compiled CPU MoE extension advertises the new generic act id. +""" + +from __future__ import annotations + +import pytest +import torch + +LIMIT = 10.0 + + +def _ref(x: torch.Tensor, limit: float = LIMIT) -> torch.Tensor: + d = x.shape[-1] // 2 + gate = x[..., :d].float().clamp(max=limit) + up = x[..., d:].float().clamp(min=-limit, max=limit) + return (gate * torch.sigmoid(gate) * up).to(x.dtype) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_triton_matches_reference(): + from freetoken.layers import swiglu_clamp_and_mul + + torch.manual_seed(0) + # Scale up so the clamp actually engages on a good fraction of elements. + x = torch.randn(129, 2 * 512, device="cuda", dtype=torch.bfloat16) * 8.0 + out = swiglu_clamp_and_mul(x, alpha=1.0, limit=LIMIT) + ref = _ref(x) + assert (out.float() - ref.float()).abs().max().item() < 2e-2 + assert (x[..., :512].float() > LIMIT).any(), "test data never hit the clamp" + + +def test_cpu_extension_supports_swiglu_clamp(): + from freetoken.moe.cpu_executor import compiled_extension_supports + + assert compiled_extension_supports("swiglu_clamp"), ( + "compiled _cpu_moe extension is stale -- rebuild with ACT_SWIGLU_CLAMP " + "(python setup.py build_ext --inplace)" + ) diff --git a/tests/kernels/test_triton_attention.py b/tests/kernels/test_triton_attention.py index 6f4afca9e9..d12644c51b 100644 --- a/tests/kernels/test_triton_attention.py +++ b/tests/kernels/test_triton_attention.py @@ -6,6 +6,146 @@ import torch +def test_image_group_ids_only_unmask_contiguous_soft_image_tokens(): + """Two images get distinct groups while text and delimiters remain causal.""" + from freetoken.attention.triton import _image_group_ids_for_prefill + + req = SimpleNamespace( + input_ids=torch.tensor([11, 99, 99, 12, 99, 99, 99, 13], dtype=torch.int32), + cached_len=0, + device_len=8, + ) + + actual = _image_group_ids_for_prefill([req], image_token_id=99) + + assert actual is not None + assert actual.tolist() == [-1, 0, 0, -1, 1, 1, 1, -1] + + +def test_image_group_ids_only_cover_the_uncached_prefill_suffix(): + """A cached image span cannot make a later decode or continuation non-causal.""" + from freetoken.attention.triton import _image_group_ids_for_prefill + + req = SimpleNamespace( + input_ids=torch.tensor([99, 99, 10, 99, 99, 20], dtype=torch.int32), + cached_len=3, + device_len=6, + ) + + actual = _image_group_ids_for_prefill([req], image_token_id=99) + + assert actual is not None + assert actual.tolist() == [0, 0, -1] + + +def test_image_group_ids_are_absent_without_soft_image_tokens(): + """Text-only and delimiter-only extensions stay on the existing causal fast path.""" + from freetoken.attention.triton import _image_group_ids_for_prefill + + req = SimpleNamespace( + input_ids=torch.tensor([11, 12, 13], dtype=torch.int32), + cached_len=0, + device_len=3, + ) + + assert _image_group_ids_for_prefill([req], image_token_id=99) is None + + +def test_triton_backend_enables_image_groups_only_when_layer_requests_them(monkeypatch): + """Gemma full layers stay causal while its sliding layers opt in explicitly.""" + from freetoken.attention import AttentionSpec + from freetoken.attention.triton import TritonAttentionBackend, TritonMetadata + + class FakeKVCache: + device = torch.device("cpu") + + def store_kv(self, *_args): + pass + + def k_cache(self, _layer_id): + return torch.zeros(4, 1, 4) + + def v_cache(self, _layer_id): + return torch.zeros(4, 1, 4) + + monkeypatch.setattr( + "freetoken.attention.triton.get_global_ctx", + lambda: SimpleNamespace(kv_cache=FakeKVCache()), + ) + captured = [] + + def fake_extend(**kwargs): + captured.append(kwargs["image_group_ids"]) + return torch.zeros_like(kwargs["q"]) + + monkeypatch.setattr("freetoken.kernel.triton.attention.extend_paged_attention", fake_extend) + backend = TritonAttentionBackend(SimpleNamespace()) + metadata = TritonMetadata( + cu_seqlens_q_gpu=torch.tensor([0, 2], dtype=torch.int32), + indptr=torch.tensor([0, 2], dtype=torch.int32), + indices=torch.tensor([0, 1], dtype=torch.int32), + q_to_req=torch.tensor([0, 0], dtype=torch.int32), + q_positions=torch.tensor([0, 1], dtype=torch.int64), + is_decode=False, + prefix_lens=torch.tensor([0], dtype=torch.int32), + max_q_len=2, + image_group_ids=torch.tensor([0, 0], dtype=torch.int32), + ) + batch = SimpleNamespace(attn_metadata=metadata, out_loc=torch.tensor([0, 1], dtype=torch.int32)) + q = torch.randn(2, 2, 4, dtype=torch.bfloat16) + k = torch.randn(2, 4, dtype=torch.bfloat16) + v = torch.randn(2, 4, dtype=torch.bfloat16) + + backend.forward(q, k, v, 0, batch, AttentionSpec(multimodal_bidirectional=False)) + backend.forward(q, k, v, 0, batch, AttentionSpec(multimodal_bidirectional=True)) + + assert captured == [None, metadata.image_group_ids] + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Triton attention needs CUDA or ROCm") +def test_extend_triton_attention_unmasks_only_same_image_group(): + """The ROCm kernel must match Gemma's image-only bidirectional mask.""" + from freetoken.kernel.triton.attention import extend_paged_attention + + torch.manual_seed(7) + device = torch.device("cuda") + token_count, q_heads, kv_heads, head_dim = 6, 2, 1, 256 + q = torch.randn(token_count, q_heads, head_dim, device=device, dtype=torch.bfloat16) + k_extend = torch.randn(token_count, kv_heads, head_dim, device=device, dtype=torch.bfloat16) + v_extend = torch.randn(token_count, kv_heads, head_dim, device=device, dtype=torch.bfloat16) + k_cache = torch.zeros_like(k_extend) + v_cache = torch.zeros_like(v_extend) + groups = torch.tensor([-1, 0, 0, -1, 1, 1], dtype=torch.int32, device=device) + actual = extend_paged_attention( + q=q, + k_cache=k_cache, + v_cache=v_cache, + qo_indptr=torch.tensor([0, token_count], dtype=torch.int32, device=device), + kv_indptr=torch.tensor([0, token_count], dtype=torch.int32, device=device), + kv_indices=torch.arange(token_count, dtype=torch.int32, device=device), + prefix_lens=torch.tensor([0], dtype=torch.int32, device=device), + max_q_len=token_count, + sm_scale=head_dim**-0.5, + k_extend=k_extend, + v_extend=v_extend, + image_group_ids=groups, + ) + + expected_rows = [] + k = k_extend.repeat_interleave(q_heads // kv_heads, dim=1).transpose(0, 1).float() + v = v_extend.repeat_interleave(q_heads // kv_heads, dim=1).transpose(0, 1).float() + for query_index in range(token_count): + causal = torch.arange(token_count, device=device) <= query_index + same_group = (groups == groups[query_index]) & (groups[query_index] >= 0) + allowed = causal | same_group + scores = torch.einsum("hd,hkd->hk", q[query_index].float(), k) * (head_dim**-0.5) + probabilities = torch.softmax(scores.masked_fill(~allowed.unsqueeze(0), float("-inf")), dim=-1) + expected_rows.append(torch.einsum("hk,hkd->hd", probabilities, v)) + expected = torch.stack(expected_rows).to(actual.dtype) + + torch.testing.assert_close(actual.float(), expected.float(), atol=2e-2, rtol=2e-2) + + def _reference_paged_attention( q: torch.Tensor, k_cache: torch.Tensor, diff --git a/tests/kvcache/test_hybrid_linear_paged_pools.py b/tests/kvcache/test_hybrid_linear_paged_pools.py new file mode 100644 index 0000000000..8b60801cb5 --- /dev/null +++ b/tests/kvcache/test_hybrid_linear_paged_pools.py @@ -0,0 +1,117 @@ +"""Pool-factory generalization: hybrid linear x ANY paged family. + +The old factory hard-required "linear + one GQA group" (Qwen3.5 GDN shape); +glm5_next is linear (KDA) x DSA. Checks the factory dispatch, the MLA/DSA +layer-id remap (34 KDA layers cost no latent slabs), the kpool pool selection +(gate slab), and the KV cost model's kpool double-count of the index slabs. +""" + +from __future__ import annotations + +import pytest +import torch + +from freetoken.kvcache import create_kvcache_pool, resolve_pool_class +from freetoken.kvcache.dsa_pool import DSAKVCache, KpoolDSAKVCache +from freetoken.models.config import ( + FullAttentionGroupConfig, + LinearGatedDeltaGroupConfig, + ModelConfig, + RotaryConfig, +) + + +@pytest.fixture(autouse=True) +def _single_rank_tp(): + from freetoken.distributed import set_tp_info, try_get_tp_info + + if try_get_tp_info() is None: + set_tp_info(rank=0, size=1) + + +def _glm5_like_config(index_kpool=4, index_head_dim=128): + n_layers = 12 + dsa_ids = tuple(range(3, n_layers, 4)) # 3, 7, 11 + kda_ids = tuple(i for i in range(n_layers) if i not in dsa_ids) + rotary = RotaryConfig(head_dim=256, rotary_dim=0, max_position=4096, base=1e4, scaling=None) + groups = ( + LinearGatedDeltaGroupConfig( + name="linear", layer_ids=kda_ids, + num_key_heads=4, num_value_heads=4, key_head_dim=128, value_head_dim=128, + conv_kernel_dim=4, output_gate="sigmoid", variant="kda", + ), + FullAttentionGroupConfig( + name="full", layer_ids=dsa_ids, num_kv_heads=1, head_dim=512, + rotary_config=rotary, mla=True, + index_head_dim=index_head_dim, num_index_layers=len(dsa_ids), + index_ratio=index_kpool, + ), + ) + return ModelConfig( + num_layers=n_layers, num_qo_heads=4, num_kv_heads=1, head_dim=512, + hidden_size=256, vocab_size=1000, intermediate_size=512, + rms_norm_eps=1e-5, rotary_config=rotary, hidden_act="silu", + tie_word_embeddings=False, num_experts=8, num_experts_per_tok=2, + moe_intermediate_size=64, norm_topk_prob=True, model_type="glm5_next", + architectures=["Glm5NextForCausalLM"], moe_enabled=True, + attention_groups=groups, + ) + + +def test_factory_builds_kpool_pool_with_layer_remap(): + cfg = _glm5_like_config() + assert resolve_pool_class(cfg) is KpoolDSAKVCache + + pool = create_kvcache_pool( + model_config=cfg, num_pages=4, page_size=64, + dtype=torch.bfloat16, device=torch.device("cpu"), num_req_slots=5, + ) + assert isinstance(pool, KpoolDSAKVCache) + # Latent slabs back ONLY the 3 DSA layers (34-of-45 economy at real scale). + assert pool._kv_buffer.shape[1] == 3 + # Global layer-id addressing: DSA layers resolve, KDA layers have no slab. + for lid in (3, 7, 11): + assert pool.latent_rows(lid).shape == (256, 512) + with pytest.raises(KeyError): + pool.latent_rows(0) # a KDA layer + # Shadow index slab: tokens/ratio rows + one scratch row per request slot. + assert pool.index_k_cache(0).shape == (256 // 4 + 5, 128) + assert pool.cmp_scratch_base == 256 // 4 + # kpool tail rings exist at [num_req_slots, ratio, head_dim] per indexer layer. + assert pool.tail_k(0).shape == pool.tail_gate(0).shape + assert pool.tail_k(0).shape == (5, 4, 128) + + +def test_factory_kpool1_builds_plain_dsa_pool(): + cfg = _glm5_like_config(index_kpool=1) + assert resolve_pool_class(cfg) is DSAKVCache + pool = create_kvcache_pool( + model_config=cfg, num_pages=16, page_size=1, + dtype=torch.bfloat16, device=torch.device("cpu"), + ) + assert type(pool) is DSAKVCache + + +def test_cost_model_kpool_shadow_slab_quarter_cost(): + """The shadow slab stores one row per index_ratio tokens: the kpool spec's + index bytes are 1/ratio of the plain DSA slab (rings/scratch are per-request + and not part of the per-token price).""" + from types import SimpleNamespace + + from freetoken.kvcache.base import spec_kv_bytes_per_token + + tp = SimpleNamespace(size=1) + econf = SimpleNamespace(tp_info=tp, dtype=torch.bfloat16) + (spec,) = [ + s for s in _glm5_like_config().kv_cache_group_specs() if s.num_layers > 0 + ] + (spec1,) = [ + s + for s in _glm5_like_config(index_kpool=1).kv_cache_group_specs() + if s.num_layers > 0 + ] + index_full = spec.index_head_dim * spec.num_index_layers * 2 + assert ( + spec_kv_bytes_per_token(spec1, econf) - spec_kv_bytes_per_token(spec, econf) + == index_full - index_full // 4 + ) \ No newline at end of file diff --git a/tests/kvcache/test_kv_cache_rebuild.py b/tests/kvcache/test_kv_cache_rebuild.py index 6dff34eab2..a5ec94a084 100644 --- a/tests/kvcache/test_kv_cache_rebuild.py +++ b/tests/kvcache/test_kv_cache_rebuild.py @@ -150,7 +150,7 @@ def test_linear_state_pool_rebuild_resizes_preserves_identity_and_dtypes(): _init_tp() group = LinearGatedDeltaGroupConfig( name="linear", layer_ids=(0, 1, 2), num_key_heads=4, num_value_heads=8, - key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True, + key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu", ) pool = LinearStatePool(group=group, num_slots=10, dtype=torch.bfloat16, device=torch.device("cpu")) pid = id(pool) diff --git a/tests/kvcache/test_linear_state_pool_alloc.py b/tests/kvcache/test_linear_state_pool_alloc.py index 16058efcb5..e9fe405426 100644 --- a/tests/kvcache/test_linear_state_pool_alloc.py +++ b/tests/kvcache/test_linear_state_pool_alloc.py @@ -1,22 +1,31 @@ -"""P1 unit: LinearStatePool free-list allocator (alloc/free/clear_slots/copy_from). +"""LinearStatePool unit: the free-list allocator and the declared slot-state siblings. CPU-only, fast — pure slot bookkeeping + state copy/zero, no kernels.""" from __future__ import annotations +from types import SimpleNamespace + import pytest import torch -from freetoken.kvcache.linear_state_pool import LinearStatePool -from freetoken.models.config import LinearGatedDeltaGroupConfig +from freetoken.kvcache.linear_state_pool import ( + LinearStatePool, + linear_state_bytes_per_req, + state_pool_bytes, +) +from freetoken.models.config import LinearGatedDeltaGroupConfig, SlotStateSpec -def _pool(num_slots=8, device="cpu"): - group = LinearGatedDeltaGroupConfig( +def _group(): + return LinearGatedDeltaGroupConfig( name="linear", layer_ids=(0, 1), num_key_heads=2, num_value_heads=4, - key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True, + key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu", ) - return LinearStatePool(group=group, num_slots=num_slots, dtype=torch.bfloat16, - device=torch.device(device), tp_size=1) + + +def _pool(num_slots=8, device="cpu", slot_states=()): + return LinearStatePool(group=_group(), num_slots=num_slots, dtype=torch.bfloat16, + device=torch.device(device), tp_size=1, slot_states=slot_states) def test_alloc_free_roundtrip(): @@ -70,3 +79,113 @@ def test_copy_from_snapshot(): test_clear_slots_zeros_all_layers() test_copy_from_snapshot() print("LinearStatePool allocator unit: PASS") + + +# qwen4_exp PLE conv-history shape at toy size: one PLE layer, 32 channels, 9 taps +_SPECS = (SlotStateSpec(name="ple_conv", shape=(32, 9), layer_ids=(1,)),) + + +def test_no_slot_states_by_default(): + pool = _pool() + assert pool.slot_states == {} and not pool.has_slot_state("ple_conv") + base = pool.bytes_per_slot() + pool.clear_slots([1, 2]) + pool.copy_from(1, 2) + pool.reset(3) + pool.rebuild(4) + assert pool.slot_states == {} and pool.bytes_per_slot() == base + + +def test_slot_state_geometry_and_accessor(): + pool = _pool(num_slots=6, slot_states=_SPECS) + slab = pool.slot_states["ple_conv"] + assert slab.shape == (1, 6, 32, 9) and slab.dtype is torch.bfloat16 + assert pool.slot_state("ple_conv", 1).shape == (6, 32, 9) + with pytest.raises(KeyError): + pool.slot_state("ple_conv", 0) # not a declared layer + with pytest.raises(KeyError): + pool.slot_state("other") + with pytest.raises(AssertionError): + pool.slot_state("ple_conv") # per-layer state needs layer_id + + +def test_layerless_spec_dtype_and_fill_value(): + spec = SlotStateSpec(name="ngram", shape=(2,), dtype=torch.int32, fill_value=7.0) + pool = _pool(num_slots=6, slot_states=(spec,)) + t = pool.slot_state("ngram") + assert t.shape == (6, 2) and t.dtype is torch.int32 and bool((t == 7).all()) + t[3] = 1 + pool.clear_slots([3]) + assert bool((pool.slot_state("ngram")[3] == 7).all()) + t[2] = 1 + pool.reset(2) + assert bool((pool.slot_state("ngram")[2] == 7).all()) + pool.rebuild(5) + assert bool((pool.slot_state("ngram") == 7).all()) + + +def test_duplicate_names_rejected(): + with pytest.raises(ValueError, match="duplicate"): + _pool(slot_states=_SPECS + _SPECS) + + +def test_slot_state_follows_every_slot_operation(): + pool = _pool(slot_states=_SPECS) + pool.slot_states["ple_conv"].fill_(1.0) + pool.conv_states.fill_(1.0) + + pool.clear_slots([2]) + slab = pool.slot_state("ple_conv", 1) + assert slab[2].abs().sum().item() == 0.0 + assert slab[3].abs().sum().item() > 0.0 + + pool.copy_from(3, 2) + assert torch.equal(slab[2], slab[3]) + + pool.reset(3) + assert slab[3].abs().sum().item() == 0.0 + + pool.rebuild(9) + slab = pool.slot_state("ple_conv", 1) # rebuild replaces the tensor + assert pool.slot_states["ple_conv"].shape == (1, 9, 32, 9) + assert slab.abs().sum().item() == 0.0 + + # the ops write the rebuilt tensor, not a stale alias + pool.slot_states["ple_conv"].fill_(1.0) + pool.clear_slots([5]) + assert pool.slot_state("ple_conv", 1)[5].abs().sum().item() == 0.0 + pool.copy_from(1, 5) + assert torch.equal(pool.slot_state("ple_conv", 1)[5], pool.slot_state("ple_conv", 1)[1]) + + +def test_slot_state_in_the_byte_account(): + group = _group() + gdn_only = linear_state_bytes_per_req(group, 1, torch.bfloat16) + with_state = linear_state_bytes_per_req(group, 1, torch.bfloat16, _SPECS) + assert with_state - gdn_only == 32 * 9 * 2 + assert _pool(slot_states=_SPECS).bytes_per_slot() == with_state + + mc = SimpleNamespace(slot_states=_SPECS) + mc.linear_attention_group = lambda: group + config = SimpleNamespace( + model_config=mc, dtype=torch.bfloat16, tp_info=SimpleNamespace(size=1), + cache_type="naive", max_running_req=3, linear_state_cache_ratio=0.5, + ) + assert state_pool_bytes(config, num_slots=4) == with_state * 4 + + mc.slot_states = () + assert state_pool_bytes(config, num_slots=4) == gdn_only * 4 + + mc.slot_states = _SPECS + mc.linear_attention_group = lambda: None + with pytest.raises(ValueError, match="slot_states"): + state_pool_bytes(config, num_slots=4) + + +def test_slot_state_bytes_for_the_real_geometry(): + # qwen4_exp PLE conv history: 4 streams x 2560 channels x 9 taps bf16 = 180 KiB per slot + spec = SlotStateSpec(name="ple_conv", shape=(4 * 2560, 9), layer_ids=(1,)) + group = _group() + delta = linear_state_bytes_per_req(group, 1, torch.bfloat16, (spec,)) - \ + linear_state_bytes_per_req(group, 1, torch.bfloat16) + assert delta == 4 * 2560 * 9 * 2 == 180 * 1024 diff --git a/tests/kvcache/test_pool_sizing_surface.py b/tests/kvcache/test_pool_sizing_surface.py index d50a5be49a..516090b099 100644 --- a/tests/kvcache/test_pool_sizing_surface.py +++ b/tests/kvcache/test_pool_sizing_surface.py @@ -249,7 +249,7 @@ def test_linear_state_pool_prices_itself(): group = LinearGatedDeltaGroupConfig( name="linear", layer_ids=(1, 3), num_key_heads=2, num_value_heads=4, - key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True, + key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu", ) config = _generic_config() config.linear_state_cache_ratio = 0.5 diff --git a/tests/kvcache/test_qsa_pool.py b/tests/kvcache/test_qsa_pool.py new file mode 100644 index 0000000000..9aec2bb606 --- /dev/null +++ b/tests/kvcache/test_qsa_pool.py @@ -0,0 +1,218 @@ +"""QSAKVCache tiers (paged K/V, compressed index slab, pending ring, scratch). + +Pins the three things the QSA kernels and the startup budget both depend on: the compressed +slab is a 1/index_ratio shadow of the K/V pages with the scratch rows behind it, the ring and +scratch are fixed (concurrency-sized) and priced apart from the per-token slider, and the K/V +slabs cover the sparse layers only. The PLE conv history rides the GDN slots, so it must +follow every slot operation and show up in the state-pool byte account. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +from freetoken.attention import AttnType +from freetoken.kvcache.base import spec_kv_bytes_per_token +from freetoken.kvcache.qsa_pool import QSAKVCache +from freetoken.models.config import KVCacheGroupSpec + +DEV = torch.device("cpu") + +# Qwen3.8-Flash-Next: 48 layers, every 4th is QSA; 2 kv heads x 256, indexer 128 wide, ratio 4. +FULL_LAYER_IDS = tuple(range(3, 48, 4)) +REAL_KV_BYTES = 2 * 256 * 2 * 2 * 12 +REAL_INDEX_BYTES = 128 * 12 * 2 // 4 + + +@pytest.fixture(autouse=True) +def _tp(monkeypatch): + from freetoken.distributed.info import DistributedInfo + + monkeypatch.setattr( + "freetoken.kvcache.mha_pool.get_tp_info", + lambda: DistributedInfo(rank=0, size=1), + ) + + +def _pool(num_pages=4, page_size=64, index_ratio=4, num_req_slots=4, ring_capacity=None): + return QSAKVCache( + num_kv_heads=2, + num_layers=8, + head_dim=64, + num_pages=num_pages, + page_size=page_size, + dtype=torch.bfloat16, + device=DEV, + index_head_dim=32, + num_index_layers=4, + index_ratio=index_ratio, + num_req_slots=num_req_slots, + ring_capacity=ring_capacity, + layer_ids=(1, 3, 5, 7), + ) + + +def _spec(*, index_ratio=4, attn_type=AttnType.QSA, num_kv_heads=2, head_dim=64, + index_head_dim=32, num_index_layers=4, layer_ids=(1, 3, 5, 7)): + return KVCacheGroupSpec( + name="full", + layer_ids=layer_ids, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + sliding_window=None, + index_head_dim=index_head_dim, + num_index_layers=num_index_layers, + index_ratio=index_ratio, + attn_type=attn_type, + ) + + +def _config(spec, *, page_size=64, max_running_req=3): + mc = SimpleNamespace(num_layers=8, has_swa_attention=False, has_linear_attention=True) + mc.kv_cache_group_specs = lambda: (spec,) + return SimpleNamespace( + model_config=mc, + page_size=page_size, + dtype=torch.bfloat16, + tp_info=SimpleNamespace(size=1), + max_running_req=max_running_req, + ) + + +# --------------------------------------------------------------------- slab / ring geometry + + +def test_slab_ring_and_scratch_shapes(): + pool = _pool(num_pages=4) + # 4 pages x 64 tokens / ratio 4 = 64 shadow rows, then one scratch row per request slot + assert pool.cmp_scratch_base == 64 + assert pool.cmp_k_cache(0).shape == (64 + 4, 32) + assert pool.cmp_k_cache(3).shape == (64 + 4, 32) + assert pool.pending_ring(0).shape == (4, QSAKVCache.ring_capacity_for(4), 32) + assert pool.cmp_k_cache(0).abs().sum().item() == 0.0 + assert pool.k_cache(1).shape == (4, 64, 2, 64) + + +def test_kv_slabs_cover_sparse_layers_only(): + # Copying the BSA branch (no layer_ids) would back all 8 model layers instead of 4. + pool = _pool() + assert pool._kv_buffer.shape[1] == 4 + pool.k_cache(7) + with pytest.raises(KeyError): + pool.k_cache(0) + + +def test_ring_capacity_and_ratio_are_parameters(): + pool = _pool(num_pages=8, index_ratio=2, num_req_slots=3, ring_capacity=6) + assert pool.index_ratio == 2 and pool.ring_capacity == 6 + assert pool.cmp_scratch_base == 8 * 64 // 2 + assert pool.pending_ring(0).shape == (3, 6, 32) + + +def test_ring_capacity_formula_and_floor(): + assert QSAKVCache.ring_capacity_for(4) == 4 + assert QSAKVCache.ring_capacity_for(8) == 8 + assert QSAKVCache.ring_capacity_for(4, num_speculative_tokens=3) == 8 + with pytest.raises(ValueError, match="ring_capacity"): + _pool(ring_capacity=2, index_ratio=4) + + +def test_group_must_not_straddle_a_page(): + with pytest.raises(ValueError, match="divisible"): + _pool(page_size=6, index_ratio=4) + + +def test_index_slab_needs_a_two_byte_dtype(): + with pytest.raises(AssertionError, match="2 bytes"): + QSAKVCache( + num_kv_heads=2, num_layers=8, head_dim=64, num_pages=4, page_size=64, + dtype=torch.float32, device=DEV, index_head_dim=32, num_index_layers=4, + index_ratio=4, num_req_slots=4, layer_ids=(1, 3, 5, 7), + ) + + +def test_shadow_row_is_shared_by_a_whole_group(): + pool = _pool() + cmp = pool.cmp_k_cache(2) + row = torch.randn(32, dtype=torch.bfloat16) + for slot in (64, 65, 66, 67): + assert slot // pool.index_ratio == 16 + cmp[16] = row + assert torch.equal(pool.cmp_k_cache(2)[16], row) + # the other sparse layers keep their own rows + assert pool.cmp_k_cache(1)[16].abs().sum().item() == 0.0 + + +def test_rebuild_resizes_every_tier_and_keeps_identity(): + pool = _pool(num_pages=4) + ident = id(pool) + pool.rebuild(16) + assert id(pool) == ident + assert pool.k_cache(1).shape == (16, 64, 2, 64) + assert pool.cmp_scratch_base == 16 * 64 // 4 + assert pool.cmp_k_cache(0).shape == (16 * 64 // 4 + 4, 32) + assert pool.pending_ring(3).shape == (4, pool.ring_capacity, 32) + assert pool._kv_buffer.shape[1] == 4 # sparse-layer slabs survive the resize + pool.k_cache(7) + + +# ------------------------------------------------------------------------------ budgeting + + +def test_spec_bytes_per_token_divides_the_index_slab(): + spec = _spec(num_kv_heads=2, head_dim=256, index_head_dim=128, num_index_layers=12, + layer_ids=FULL_LAYER_IDS) + config = _config(spec) + assert spec_kv_bytes_per_token(spec, config) == REAL_KV_BYTES + REAL_INDEX_BYTES + assert spec_kv_bytes_per_token(spec, config) == 24576 + 768 + + # BSA/DSA keep one index row per token (ratio 1) + bsa = _spec(num_kv_heads=2, head_dim=256, index_head_dim=128, num_index_layers=12, + layer_ids=FULL_LAYER_IDS, index_ratio=1, attn_type=AttnType.BSA) + assert spec_kv_bytes_per_token(bsa, config) == REAL_KV_BYTES + 128 * 12 * 2 + + +def test_kv_cost_prices_ring_and_scratch_as_fixed(): + spec = _spec() + config = _config(spec, max_running_req=3) + per_page, fixed, page_tokens, min_reserve = QSAKVCache.kv_cost(config) + assert per_page == spec_kv_bytes_per_token(spec, config) * 64 + assert page_tokens == 64 and min_reserve == 0 + row = 32 * 4 * 2 + assert fixed == 4 * row * (QSAKVCache.ring_capacity_for(4) + 1) + + +def test_unit_bytes_matches_the_cost_model(): + spec = _spec() + config = _config(spec) + pool = _pool() + kv_bytes, swa_bytes = pool.unit_bytes() + assert swa_bytes == 0 + # the scratch rows and the ring must NOT inflate the per-token slider + assert kv_bytes == spec_kv_bytes_per_token(spec, config) + assert kv_bytes * 64 == QSAKVCache.kv_cost(config)[0] + + +def test_resolve_pool_class_and_factory(): + from freetoken.kvcache import create_kvcache_pool, resolve_pool_class + + spec = _spec() + mc = SimpleNamespace( + num_layers=8, has_swa_attention=False, has_linear_attention=True, + num_kv_heads=2, head_dim=64, dsv4_args=None, + ) + mc.kv_cache_group_specs = lambda: (spec,) + assert resolve_pool_class(mc) is QSAKVCache + + pool = create_kvcache_pool( + mc, num_pages=4, page_size=64, dtype=torch.bfloat16, device=DEV, num_req_slots=4 + ) + assert isinstance(pool, QSAKVCache) + assert pool._kv_buffer.shape[1] == 4 # not the model's 8 layers + assert pool.cmp_k_cache(0).shape == (4 * 64 // 4 + 4, 32) + + with pytest.raises(ValueError, match="num_req_slots"): + create_kvcache_pool(mc, num_pages=4, page_size=64, dtype=torch.bfloat16, device=DEV) diff --git a/tests/layers/test_mhc.py b/tests/layers/test_mhc.py new file mode 100644 index 0000000000..4cf144e720 --- /dev/null +++ b/tests/layers/test_mhc.py @@ -0,0 +1,178 @@ +"""mHC (Manifold-Constrained Hyper-Connections) unit tests. + +Checks the algebraic contracts of layers/mhc.py at GLM-5.3 geometry (n=4): +Sinkhorn projection yields an (approximately) doubly-stochastic comb matrix, +pre/post mixing matches naive per-token einsums, and identity-ish weights give +the classic single-stream residual behaviour. +""" + +from __future__ import annotations + +import pytest +import torch + +from freetoken.layers.mhc import ( + hc_contract, + hc_expand, + mhc_fused_post_pre, + mhc_post, + mhc_pre, +) + +N, HIDDEN, T = 4, 64, 9 +MIX = 2 * N + N * N +EPS = 1e-6 +RMS_EPS = 1e-5 +POST_MULT = 2.0 +SINKHORN = 20 + + +def _weights(seed=0, device="cpu"): + torch.manual_seed(seed) + fn = torch.randn(MIX, N * HIDDEN, dtype=torch.float32, device=device) * 0.05 + scale = torch.randn(3, dtype=torch.float32, device=device).abs() + 0.5 + base = torch.randn(MIX, dtype=torch.float32, device=device) * 0.3 + return fn, scale, base + + +def _residual(seed=1, device="cpu"): + torch.manual_seed(seed) + return torch.randn(T, N, HIDDEN, dtype=torch.bfloat16, device=device) + + +def test_comb_is_doubly_stochastic(): + fn, scale, base = _weights(seed=2) + res = _residual(seed=3) + _, comb, _ = mhc_pre(res, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN) + rows = comb.sum(dim=-1) + cols = comb.sum(dim=-2) + assert torch.allclose(rows, torch.ones_like(rows), atol=1e-3) + assert torch.allclose(cols, torch.ones_like(cols), atol=1e-3) + assert (comb > 0).all() + + +def test_post_matches_naive(): + res = _residual(seed=4) + x = torch.randn(T, HIDDEN, dtype=torch.bfloat16) + post = torch.rand(T, N, 1, dtype=torch.float32) * POST_MULT + comb = torch.softmax(torch.randn(T, N, N), dim=-1) + out = mhc_post(x, res, post, comb) + assert out.shape == (T, N, HIDDEN) + + ref = torch.zeros(T, N, HIDDEN, dtype=torch.float32) + for t in range(T): + for j in range(N): + acc = post[t, j, 0] * x[t].float() + for i in range(N): + acc = acc + comb[t, i, j] * res[t, i].float() + ref[t, j] = acc + assert torch.allclose(out.float(), ref.to(torch.bfloat16).float()) + + +def test_fused_equals_decomposed(): + fn, scale, base = _weights(seed=7) + res = _residual(seed=8) + x = torch.randn(T, HIDDEN, dtype=torch.bfloat16) + post0, comb0, _ = mhc_pre(res, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN) + + r1, p1, c1, li1 = mhc_fused_post_pre( + x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN + ) + r_ref = mhc_post(x, res, post0, comb0) + p_ref, c_ref, li_ref = mhc_pre( + r_ref, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN + ) + assert torch.equal(r1, r_ref) + assert torch.equal(p1, p_ref) + assert torch.equal(c1, c_ref) + assert torch.equal(li1, li_ref) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +@pytest.mark.parametrize("t,hidden", [(1, 64), (9, 64), (3, 4096)]) +def test_triton_fused_matches_torch(t, hidden): + """The fused triton kernel must reproduce the decomposed torch reference + (hc_post -> hc_pre) on every output, including GLM-5.3's real hidden size.""" + from freetoken.layers.mhc import mhc_fused_post_pre_torch + from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton + + torch.manual_seed(11) + mix = 2 * N + N * N + fn = torch.randn(mix, N * hidden, dtype=torch.float32, device="cuda") * 0.05 + scale = torch.rand(3, dtype=torch.float32, device="cuda") + 0.5 + base = torch.randn(mix, dtype=torch.float32, device="cuda") * 0.3 + res = torch.randn(t, N, hidden, dtype=torch.bfloat16, device="cuda") + x = torch.randn(t, hidden, dtype=torch.bfloat16, device="cuda") + post0 = torch.rand(t, N, 1, dtype=torch.float32, device="cuda") * POST_MULT + comb0 = torch.softmax(torch.randn(t, N, N, device="cuda"), dim=-1) + + ref = mhc_fused_post_pre_torch( + x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN + ) + got = mhc_fused_post_pre_triton( + x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN + ) + names = ("residual", "post", "comb", "layer_input") + tols = (2e-2, 2e-3, 2e-3, 2e-2) + for name, r, g, tol in zip(names, ref, got, tols): + assert g.shape == r.shape, (name, g.shape, r.shape) + err = (g.float() - r.float()).abs().max().item() + assert err < tol, f"{name}: max abs err {err}" + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +@pytest.mark.parametrize("dtype,tol", [(torch.float16, 2e-2), (torch.float32, 1e-4)]) +def test_triton_fused_respects_input_dtype(dtype, tol): + """--dtype float16/float32 must not be silently bf16-rounded: the kernel + stores in the OUTPUT tensor's dtype (regression for the hard-coded + tl.bfloat16 stores; fp32's tolerance is far below bf16's 2^-8 grid).""" + from freetoken.layers.mhc import mhc_fused_post_pre_torch + from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton + + torch.manual_seed(13) + t, hidden = 4, 4096 + mix = 2 * N + N * N + fn = torch.randn(mix, N * hidden, dtype=torch.float32, device="cuda") * 0.05 + scale = torch.rand(3, dtype=torch.float32, device="cuda") + 0.5 + base = torch.randn(mix, dtype=torch.float32, device="cuda") * 0.3 + res = torch.randn(t, N, hidden, dtype=dtype, device="cuda") + x = torch.randn(t, hidden, dtype=dtype, device="cuda") + post0 = torch.rand(t, N, 1, dtype=torch.float32, device="cuda") * POST_MULT + comb0 = torch.softmax(torch.randn(t, N, N, device="cuda"), dim=-1) + + ref = mhc_fused_post_pre_torch( + x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN + ) + got = mhc_fused_post_pre_triton( + x, res, post0, comb0, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN + ) + assert got[0].dtype == dtype and got[3].dtype == dtype + for name, r, g in zip(("residual", "layer_input"), (ref[0], ref[3]), (got[0], got[3])): + err = (g.float() - r.float()).abs().max().item() + assert err < tol, f"{name} [{dtype}]: max abs err {err}" + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_triton_pre_only_matches_torch(): + """HAS_POST=False path (layer 0's standalone hc_pre through the fused kernel).""" + from freetoken.kernel.triton.mhc import mhc_fused_post_pre_triton + + torch.manual_seed(12) + t, hidden = 5, 128 + mix = 2 * N + N * N + fn = torch.randn(mix, N * hidden, dtype=torch.float32, device="cuda") * 0.05 + scale = torch.rand(3, dtype=torch.float32, device="cuda") + 0.5 + base = torch.randn(mix, dtype=torch.float32, device="cuda") * 0.3 + res = torch.randn(t, N, hidden, dtype=torch.bfloat16, device="cuda") + + ref_post, ref_comb, ref_li = mhc_pre( + res, fn, scale, base, RMS_EPS, EPS, POST_MULT, SINKHORN + ) + got_res, got_post, got_comb, got_li = mhc_fused_post_pre_triton( + res.new_empty(t, hidden), res, None, None, fn, scale, base, + RMS_EPS, EPS, POST_MULT, SINKHORN, + ) + assert torch.equal(got_res, res) # pass-through when no post + assert (got_post.float() - ref_post.float()).abs().max().item() < 2e-3 + assert (got_comb.float() - ref_comb.float()).abs().max().item() < 2e-3 + assert (got_li.float() - ref_li.float()).abs().max().item() < 2e-2 diff --git a/tests/models/qwen4_exp/__init__.py b/tests/models/qwen4_exp/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/models/qwen4_exp/common.py b/tests/models/qwen4_exp/common.py new file mode 100644 index 0000000000..1f9c117bd8 --- /dev/null +++ b/tests/models/qwen4_exp/common.py @@ -0,0 +1,257 @@ +"""Shared fixtures for the qwen4_exp tests: toy configs, hash constants, the QSA pool and backend. + +The geometry is the shipping one everywhere it matters for QSA (head_dim 256, index head_dim +128 with a 64-wide partial rope, index_ratio 4, budget 2048 -> 512 blocks -> 2051 selected +tokens, page_size 64); only the head counts, the hidden size and the layer count are scaled +down so a test fits on a shared GPU. Holds no tests itself. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +from freetoken.distributed import set_tp_info, try_get_tp_info + +EOS = 7 +VOCAB = 512 + +requires_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU") + + +def hf_config( + num_layers: int = 4, + head_dim: int = 256, + num_q: int = 4, + num_kv: int = 2, + index_head_dim: int = 128, + index_heads: int = 4, + budget: int = 2048, + ratio: int = 4, + hidden: int = 256, + max_position: int = 1 << 16, + rope_theta: float = 10000000.0, + **text_overrides, +) -> SimpleNamespace: + text = SimpleNamespace( + num_hidden_layers=num_layers, + hidden_size=hidden, + vocab_size=VOCAB, + head_dim=head_dim, + num_attention_heads=num_q, + num_key_value_heads=num_kv, + layer_types=[ + "full_attention" if (i + 1) % 4 == 0 else "linear_attention" + for i in range(num_layers) + ], + rope_parameters={ + "rope_type": "default", + "rope_theta": rope_theta, + "partial_rotary_factor": 0.25, + }, + max_position_embeddings=max_position, + rms_norm_eps=1e-6, + hidden_act="silu", + tie_word_embeddings=False, + num_experts=8, + num_experts_per_tok=2, + moe_intermediate_size=64, + shared_expert_intermediate_size=64, + linear_num_key_heads=2, + linear_num_value_heads=6, + linear_key_head_dim=32, + linear_value_head_dim=32, + linear_conv_kernel_dim=4, + output_gate_type="sigmoid", + indexer_n_heads=index_heads, + indexer_kv_heads=1, + indexer_head_dim=index_head_dim, + indexer_budget=budget, + indexer_compress_ratio=ratio, + hc_count=4, + hc_lowrank=16, + ple_layer_ids=[2], + ple_embed_dim=64, + ple_conv_kernel_size=4, + ngram_size=3, + heads_per_ngram=2, + ngram_vocab_size_base=1000, + make_ngram_vocab_size_divisible_by=8, + split_ngram_parts=4, + eos_token_id=EOS, + ) + for name, value in text_overrides.items(): + setattr(text, name, value) + return SimpleNamespace( + model_type="qwen4_exp", + architectures=["Qwen4ExpForConditionalGeneration"], + text_config=text, + quantization_config=None, + ) + + +def toy_hf_config(num_layers: int = 4, **text_overrides) -> SimpleNamespace: + """The small-geometry config the PLE/skeleton tests share (hidden 128, head_dim 64).""" + return hf_config( + num_layers=num_layers, head_dim=64, num_kv=1, index_head_dim=64, index_heads=2, + budget=16, hidden=128, max_position=4096, rope_theta=10000.0, **text_overrides, + ) + + +def hash_constants(args): + """Checkpoint-shape int64 hash tensors, derived like the dummy-weight path.""" + from freetoken.models.qwen4_exp.ple import derive_ngram_hash_constants + + multipliers, sizes, offsets = derive_ngram_hash_constants( + vocab_size=VOCAB, + ngram_size=args.ngram_size, + num_ngram_heads=args.num_ngram_heads, + ngram_vocab_size_base=args.ngram_vocab_size_base, + ple_layer_index=0, + ) + return [torch.tensor(v, dtype=torch.int64) for v in (multipliers, sizes, offsets)] + + +def parsed_config(**kwargs): + from freetoken.models.qwen4_exp.config import parse_config + + if try_get_tp_info() is None: + set_tp_info(rank=0, size=1) + return parse_config(hf_config(**kwargs)) + + +def fresh_ctx(page_size: int = 64, **fields): + import freetoken.core as core + from freetoken.core import Context, set_global_ctx + + core._GLOBAL_CTX = None # test-only: each scenario builds its own ctx + ctx = Context(page_size=page_size) + for name, value in fields.items(): + setattr(ctx, name, value) + set_global_ctx(ctx) + return ctx + + +def fill_weights(op, seed: int, device: torch.device, scale: float = 0.05) -> None: + gen = torch.Generator(device=device).manual_seed(seed) + for tensor in op.state_dict().values(): + if tensor.is_floating_point(): + tensor.normal_(0.0, scale, generator=gen) + else: + tensor.zero_() + + +class Fixture: + """QSA pool + page table + the sparse backend, with a first-fit page allocator.""" + + def __init__( + self, + config, + num_pages: int, + max_running_req: int = 8, + device: str = "cuda", + dtype: torch.dtype = torch.bfloat16, + page_size: int = 64, + ) -> None: + from freetoken.attention.qsa_sparse import QSASparseAttnBackend + from freetoken.kvcache import create_kvcache_pool + + self.config = config + self.device = torch.device(device) + self.dtype = dtype + self.page_size = page_size + self.num_req_slots = max_running_req + 1 + self.pool = create_kvcache_pool( + model_config=config, + num_pages=num_pages + 1, # + 1 for the dummy page, as create_kv_pool does + page_size=page_size, + dtype=dtype, + device=self.device, + num_req_slots=self.num_req_slots, + ) + self.page_table = torch.zeros( + (self.num_req_slots, num_pages * page_size), dtype=torch.int32, device=self.device + ) + self.page_table[max_running_req].fill_(num_pages * page_size) # dummy page + self.ctx = fresh_ctx( + page_size=page_size, page_table=self.page_table, kv_cache=self.pool + ) + self.backend = QSASparseAttnBackend(config) + self.ctx.attn_backend = self.backend + self._free = list(range(num_pages)) + + def layer(self, layer_id: int, seed: int = 1): + from freetoken.models.qwen4_exp.attention import Qwen4ExpAttention + from freetoken.utils.torch_utils import torch_dtype + + with torch.device(self.device), torch_dtype(self.dtype): + attn = Qwen4ExpAttention(self.config, layer_id=layer_id) + fill_weights(attn, seed, self.device) + return attn + + def allocate(self, table_idx: int, cached_len: int, device_len: int) -> None: + for page in range(-(-cached_len // self.page_size), -(-device_len // self.page_size)): + base = self._free.pop(0) * self.page_size + columns = slice(page * self.page_size, (page + 1) * self.page_size) + self.page_table[table_idx, columns] = torch.arange( + base, base + self.page_size, dtype=torch.int32, device=self.device + ) + + def req(self, table_idx: int, cached_len: int, device_len: int) -> SimpleNamespace: + self.allocate(table_idx, cached_len, device_len) + return SimpleNamespace( + table_idx=table_idx, + cached_len=cached_len, + device_len=device_len, + extend_len=device_len - cached_len, + ) + + def step(self, req: SimpleNamespace) -> None: + self.allocate(req.table_idx, req.device_len, req.device_len + 1) + req.cached_len, req.device_len, req.extend_len = req.device_len, req.device_len + 1, 1 + + def batch(self, reqs, phase: str) -> SimpleNamespace: + positions = torch.cat( + [ + torch.arange(r.cached_len, r.device_len, dtype=torch.int32, device=self.device) + for r in reqs + ] + ) + out_loc = torch.cat( + [self.page_table[r.table_idx, r.cached_len : r.device_len] for r in reqs] + ).contiguous() + batch = SimpleNamespace( + reqs=reqs, + padded_reqs=reqs, + phase=phase, + size=len(reqs), + padded_size=len(reqs), + is_prefill=phase == "prefill", + is_decode=phase == "decode", + positions=positions, + out_loc=out_loc, + attn_metadata=None, + active_table_idx=torch.tensor( + [r.table_idx for r in reqs], dtype=torch.int32, device=self.device + ), + ) + self.backend.prepare_metadata(batch) + return batch + + +def selection_spy(monkeypatch, backend) -> dict: + """Record the expanded token selection of every ``_select`` call.""" + from freetoken.attention.qsa_sparse import QSASparseAttnBackend + + seen: dict[str, torch.Tensor] = {} + original = QSASparseAttnBackend._select + + def spy(self, index, md, slot): + indices = original(self, index, md, slot) + seen["indices"] = indices.clone() + return indices + + monkeypatch.setattr(QSASparseAttnBackend, "_select", spy) + return seen diff --git a/tests/models/qwen4_exp/conftest.py b/tests/models/qwen4_exp/conftest.py new file mode 100644 index 0000000000..2b9e80214f --- /dev/null +++ b/tests/models/qwen4_exp/conftest.py @@ -0,0 +1,15 @@ +"""Package-wide runtime hygiene: TP info set once, the global ctx never leaks across tests.""" + +import pytest + + +@pytest.fixture(autouse=True) +def _runtime(): + import freetoken.core as core + from freetoken.distributed import set_tp_info, try_get_tp_info + + if try_get_tp_info() is None: + set_tp_info(rank=0, size=1) + core._GLOBAL_CTX = None + yield + core._GLOBAL_CTX = None diff --git a/tests/models/qwen4_exp/ple_hf_ref.py b/tests/models/qwen4_exp/ple_hf_ref.py new file mode 100644 index 0000000000..35c5295597 --- /dev/null +++ b/tests/models/qwen4_exp/ple_hf_ref.py @@ -0,0 +1,70 @@ +"""HF ground truth for the PLE parity tests, run as a script under the transformers-main venv. + +``transformers`` in the serving venv predates ``models/qwen4_exp``, so the reference classes cannot +be imported next to FreeToken. ``test_ple.py`` spawns this file with the reference interpreter: +``python test_ple_hf_ref.py spec.json inputs.npz out.npz``. It holds no tests; every import is +inside ``main`` so pytest can still collect the module. +""" + +from __future__ import annotations + + +def main() -> None: + import json + import sys + + import numpy as np + import torch + from torch import nn + from transformers.models.qwen4_exp.configuration_qwen4_exp import Qwen4ExpTextConfig + from transformers.models.qwen4_exp.modeling_qwen4_exp import ( + Qwen4ExpTextNGramEmbedding, + Qwen4ExpTextPLELayer, + ) + + class CaptureEmbedding(nn.Module): + """Stands in for the table so the hashed ids can be read out of the HF module.""" + + def __init__(self, dim: int) -> None: + super().__init__() + self.weight = nn.Parameter(torch.zeros(1, dim), requires_grad=False) + self.ids = None + + def forward(self, ids: torch.Tensor) -> torch.Tensor: + self.ids = ids.clone() + return torch.zeros(*ids.shape, self.weight.shape[1]) + + with open(sys.argv[1], encoding="utf-8") as fh: + spec = json.load(fh) + data = np.load(sys.argv[2]) + config = Qwen4ExpTextConfig(**spec["config"]) + layer_idx, ple_index = spec["layer_idx"], spec["ple_layer_index"] + out = {} + + embed = Qwen4ExpTextNGramEmbedding(config, config.ple_embed_dim, layer_idx, ple_index) + out["layer_multipliers"] = embed.layer_multipliers.numpy() + out["ngram_heads_vocab_sizes"] = embed.ngram_heads_vocab_sizes.numpy() + out["ngram_heads_offsets"] = embed.ngram_heads_offsets.numpy() + out["padded_vocab_size"] = np.array(embed.ngram_embedding.weight.shape[0]) + + capture = CaptureEmbedding(embed.ngram_embedding.embedding_dim) + embed.ngram_embedding = capture + embed(torch.as_tensor(data["hash_tokens"]).long(), None) + out["hash_ids"] = capture.ids.numpy() + + layer = Qwen4ExpTextPLELayer(config, layer_idx, ple_index) + with torch.no_grad(): + for name in ("key_proj", "value_proj", "norm_key", "norm_query", "norm_conv"): + getattr(layer, name).weight.copy_(torch.as_tensor(data[name])) + layer.conv1d.weight.copy_(torch.as_tensor(data["conv1d"])) + layer.ple_embedding.ngram_embedding.weight.copy_(torch.as_tensor(data["table"])) + out["layer_out"] = layer( + torch.as_tensor(data["hidden"]).float(), + torch.as_tensor(data["layer_tokens"]).long(), + None, + ).numpy() + np.savez(sys.argv[3], **out) + + +if __name__ == "__main__": + main() diff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.py new file mode 100644 index 0000000000..8fc2e5500e --- /dev/null +++ b/tests/models/qwen4_exp/test_config.py @@ -0,0 +1,170 @@ +"""qwen4_exp.parse_config against a synthetic config shaped like the RadixArk NVFP4 checkpoint.""" + +from types import SimpleNamespace + +import pytest + +from freetoken.attention import AttnType +from freetoken.models.config import FullAttentionGroupConfig, LinearGatedDeltaGroupConfig +from freetoken.models.qwen4_exp.config import parse_config + + +def _text_config(): + return SimpleNamespace( + num_hidden_layers=48, + hidden_size=2560, + vocab_size=248320, + head_dim=256, + num_attention_heads=24, + num_key_value_heads=2, + layer_types=[ + "full_attention" if (i + 1) % 4 == 0 else "linear_attention" for i in range(48) + ], + rope_parameters={ + "rope_type": "default", + "rope_theta": 10000000, + "partial_rotary_factor": 0.25, + "mrope_interleaved": True, + "mrope_section": [11, 11, 10], + }, + max_position_embeddings=262144, + rms_norm_eps=1e-6, + hidden_act="silu", + tie_word_embeddings=False, + num_experts=512, + num_experts_per_tok=10, + moe_intermediate_size=640, + shared_expert_intermediate_size=640, + linear_num_key_heads=16, + linear_num_value_heads=48, + linear_key_head_dim=128, + linear_value_head_dim=128, + linear_conv_kernel_dim=4, + output_gate_type="sigmoid", + indexer_n_heads=4, + indexer_kv_heads=1, + indexer_head_dim=128, + indexer_budget=2048, + indexer_compress_ratio=4, + hc_count=4, + hc_lowrank=320, + ple_layer_ids=[2], + ple_embed_dim=2560, + ple_conv_kernel_size=4, + ngram_size=3, + heads_per_ngram=8, + ngram_vocab_size_base=20000000, + make_ngram_vocab_size_divisible_by=128, + split_ngram_parts=128, + bos_token_id=248044, + eos_token_id=248044, + ) + + +def _hf_config(): + return SimpleNamespace( + model_type="qwen4_exp", + architectures=["Qwen4ExpForConditionalGeneration"], + image_token_id=248056, + text_config=_text_config(), + quantization_config={ + "quant_algo": "NVFP4", + "quant_method": "modelopt", + "ignore": [ + "model.embed_tokens", + "mtp.*", + "model.mtp.*", + "*.self_attn.*", + "*.linear_attn.*", + "*.mlp.gate*", + "*.mlp.shared_expert.*", + "*.mlp.shared_expert_gate*", + "*hyper_connection*", + "*.ple.*", + "model.visual.*", + "model.language_model.embed_tokens", + "lm_head", + ], + }, + ) + + +def test_groups_and_layer_split(): + cfg = parse_config(_hf_config()) + full = [g for g in cfg.attention_groups if isinstance(g, FullAttentionGroupConfig)] + linear = [g for g in cfg.attention_groups if isinstance(g, LinearGatedDeltaGroupConfig)] + assert len(full) == 1 and len(linear) == 1 + assert full[0].layer_ids == tuple(range(3, 48, 4)) + assert len(linear[0].layer_ids) == 36 + assert full[0].index_head_dim == 128 + assert full[0].num_index_layers == 12 + assert full[0].index_ratio == 4 + assert full[0].rotary_config.rotary_dim == 64 + assert full[0].rotary_config.scaling is None + assert linear[0].output_gate == "sigmoid" + assert linear[0].num_key_heads == 16 and linear[0].num_value_heads == 48 + + +def test_kv_specs_resolve_qsa(): + cfg = parse_config(_hf_config()) + specs = {s.name: s for s in cfg.kv_cache_group_specs()} + assert specs["full"].attn_type is AttnType.QSA + assert specs["full"].index_ratio == 4 + assert cfg.attn_type_for_layer(3) is AttnType.QSA + assert cfg.attn_type_for_layer(0) is AttnType.LINEAR + assert cfg.has_linear_attention + + +def test_moe_and_quant_flags(): + cfg = parse_config(_hf_config()) + assert cfg.num_experts == 512 + assert cfg.num_experts_per_tok == 10 + assert cfg.norm_topk_prob is True + assert cfg.moe_enabled + assert cfg.expert_quant == "nvfp4" + assert cfg.dense_quant == "none" + assert cfg.attn_quant == "none" + assert cfg.lm_head_quant == "none" + + +def test_unquantized_config_parses(): + hf = _hf_config() + hf.quantization_config = None + assert parse_config(hf).expert_quant == "none" + + +def test_qwen4_args_payload(): + args = parse_config(_hf_config()).qwen4_args + assert args.ple_layer_ids == (1,) + assert args.hc_count == 4 and args.hc_lowrank == 320 + assert args.index_topk_blocks == 512 + assert args.num_ngram_heads == 16 + assert args.ngram_head_dim == 160 + assert args.ple_conv_state_len == 9 + assert args.ple_state_width == 10240 + assert args.ngram_boundary_token_id == 248044 + + +def test_ple_on_full_attention_layer_rejected(): + hf = _hf_config() + hf.text_config.ple_layer_ids = [4] # one-indexed 4 == zero-based 3, a full_attention layer + with pytest.raises(ValueError, match="linear_attention"): + parse_config(hf) + + +def test_output_gate_null_falls_back_to_hidden_act(): + hf = _hf_config() + hf.text_config.output_gate_type = None + linear = [ + g + for g in parse_config(hf).attention_groups + if isinstance(g, LinearGatedDeltaGroupConfig) + ] + assert linear[0].output_gate == "silu" + + +def test_eos_token_id_list_uses_the_first_entry(): + base = parse_config(_hf_config()).qwen4_args.ngram_boundary_token_id + hf = _hf_config() + hf.text_config.eos_token_id = [base, base + 1] + assert parse_config(hf).qwen4_args.ngram_boundary_token_id == base diff --git a/tests/models/qwen4_exp/test_gdn.py b/tests/models/qwen4_exp/test_gdn.py new file mode 100644 index 0000000000..81dd0e7673 --- /dev/null +++ b/tests/models/qwen4_exp/test_gdn.py @@ -0,0 +1,175 @@ +"""qwen4_exp GatedDeltaNet op vs the pure-torch HF reference math. + +The oracle is ``models/qwen4_exp/gdn_reference.py``, whose two delta rules and forward are +transcribed from the ``modeling_qwen4_exp.py`` snapshot, so no transformers build carrying +qwen4_exp is needed here. Covered: prefill at 128 and 1000 tokens, a decode step continuing +from the prefill state, ragged bs=3, both GQA head ratios, and the sigmoid output gate. +""" + +from __future__ import annotations + +import pytest +import torch + +from freetoken.core import Batch, Context, Req, SamplingParams +from freetoken.models.config import LinearGatedDeltaGroupConfig +from freetoken.models.qwen4_exp.gdn import Qwen4ExpGatedDeltaNet +from freetoken.models.qwen4_exp.gdn_reference import Qwen4ExpGatedDeltaNetReference +from freetoken.utils import torch_dtype + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + +DEV = torch.device("cuda") +HIDDEN, HEAD_DIM, CONV_K, EPS = 256, 128, 4, 1e-6 +RTOL = ATOL = 2e-2 +# (num_k_heads, num_v_heads) per value:key head ratio; 3:1 is the Qwen3.8-Flash-Next shape. +HEADS = {2: (8, 16), 3: (16, 48)} + + +def _bf(t: torch.Tensor) -> torch.Tensor: + return t.detach().to(DEV, torch.bfloat16) + + +def _state_dict(ref) -> dict[str, torch.Tensor]: + """HF's four in_proj matrices fused into the op's single qkv|z|b|a GEMM. A_log / dt_bias + stay fp32, as the weight loader keeps them.""" + return { + "in_proj.weight": _bf(torch.cat([ref.in_proj_qkv.weight, ref.in_proj_z.weight, + ref.in_proj_b.weight, ref.in_proj_a.weight], dim=0)), + "conv1d.weight": _bf(ref.conv1d.weight), + "dt_bias": ref.dt_bias.detach().to(DEV, torch.float32), + "A_log": ref.A_log.detach().to(DEV, torch.float32), + "norm.weight": _bf(ref.norm.weight), + "out_proj.weight": _bf(ref.out_proj.weight), + } + + +def _make_layer(ratio: int, output_gate: str = "sigmoid", seed: int = 0): + """fp32 reference + bf16 kernel op over one set of weights. The op is built on meta under + the serving dtype, the way the engine builds a model, so load_state_dict's dtype check bites.""" + num_k, num_v = HEADS[ratio] + torch.manual_seed(seed) + ref = Qwen4ExpGatedDeltaNetReference( + hidden_size=HIDDEN, num_k_heads=num_k, num_v_heads=num_v, head_k_dim=HEAD_DIM, + head_v_dim=HEAD_DIM, conv_kernel_size=CONV_K, rms_norm_eps=EPS, output_gate=output_gate, + ).to(DEV).float().eval() + with torch.no_grad(): + # HF inits A_log = log(U(0.01, 16)); a zero dt_bias or a unit gate norm would hide sign errors. + ref.A_log.uniform_(0.01, 16.0).log_() + ref.dt_bias.uniform_(-1.0, 1.0) + ref.norm.weight.normal_(1.0, 0.1) + with torch.device("meta"), torch_dtype(torch.bfloat16): + op = Qwen4ExpGatedDeltaNet( + hidden_size=HIDDEN, num_k_heads=num_k, num_v_heads=num_v, head_k_dim=HEAD_DIM, + head_v_dim=HEAD_DIM, conv_kernel_size=CONV_K, rms_norm_eps=EPS, layer_id=0, + output_gate=output_gate, + ) + op.load_state_dict(_state_dict(ref)) + return op, ref + + +def _ctx(ratio: int, num_slots: int = 8) -> Context: + import freetoken.core as core + from freetoken.kvcache.linear_state_pool import LinearStatePool + + num_k, num_v = HEADS[ratio] + group = LinearGatedDeltaGroupConfig( + name="linear", layer_ids=(0,), num_key_heads=num_k, num_value_heads=num_v, + key_head_dim=HEAD_DIM, value_head_dim=HEAD_DIM, conv_kernel_dim=CONV_K, + output_gate="sigmoid", + ) + core._GLOBAL_CTX = None + ctx = Context(page_size=64) + ctx.linear_state_pool = LinearStatePool(group, num_slots, torch.bfloat16, DEV, tp_size=1) + core.set_global_ctx(ctx) + return ctx + + +def _prefill(op, ctx: Context, lengths: list[int], seed: int): + """One ragged prefill batch, one state slot per request. Returns the per-request hidden + states, the reqs (for a follow-up decode) and the packed output.""" + torch.manual_seed(seed) + hidden = [torch.randn(n, HIDDEN, device=DEV, dtype=torch.bfloat16) for n in lengths] + reqs = [ + Req(input_ids=torch.zeros(n, dtype=torch.int32), table_idx=i + 1, cached_len=0, + output_len=1, uid=i, sampling_params=SamplingParams(), cache_handle=None) + for i, n in enumerate(lengths) + ] + batch = Batch(reqs=reqs, phase="prefill") + batch.padded_reqs = reqs + with ctx.forward_batch(batch): + out = op.forward(torch.cat(hidden, dim=0)) + return hidden, reqs, out + + +def _decode(op, ctx: Context, reqs, hidden: torch.Tensor) -> torch.Tensor: + batch = Batch(reqs=reqs, phase="decode") + batch.padded_reqs = reqs + batch.linear_table_idx = torch.tensor( + [r.table_idx for r in reqs], dtype=torch.int32, device=DEV + ) + with ctx.forward_batch(batch): + return op.forward(hidden) + + +@torch.no_grad() +def _ref_out(ref, hidden: torch.Tensor, use_chunk_rule: bool = False) -> torch.Tensor: + return ref(hidden.float().unsqueeze(0), use_chunk_rule=use_chunk_rule)[0] + + +@pytest.mark.parametrize("length", (1000,)) +@pytest.mark.parametrize("ratio", (2, 3)) +def test_prefill_matches_reference(ratio, length): + op, ref = _make_layer(ratio, seed=ratio) + hidden, _, out = _prefill(op, _ctx(ratio), [length], seed=11) + torch.testing.assert_close(out.float(), _ref_out(ref, hidden[0]), rtol=RTOL, atol=ATOL) + + +@pytest.mark.parametrize("ratio", (2, 3)) +def test_ragged_prefill_then_decode(ratio): + """bs=3 ragged prefill, then one decode step per request off the carried conv + recurrent + state. The decode oracle is the whole (prefill + 1) sequence in one reference pass, so a + state that did not survive the prefill shows up immediately.""" + op, ref = _make_layer(ratio, seed=ratio) + ctx = _ctx(ratio) + lengths = [128, 1000, 37] + hidden, reqs, out = _prefill(op, ctx, lengths, seed=13) + + off = 0 + for h, n in zip(hidden, lengths): + torch.testing.assert_close( + out[off:off + n].float(), _ref_out(ref, h), rtol=RTOL, atol=ATOL + ) + off += n + + nxt = torch.randn(len(lengths), HIDDEN, device=DEV, dtype=torch.bfloat16) + dec = _decode(op, ctx, reqs, nxt) + for i, h in enumerate(hidden): + full = _ref_out(ref, torch.cat([h, nxt[i:i + 1]], dim=0)) + torch.testing.assert_close(dec[i].float(), full[-1], rtol=RTOL, atol=ATOL) + + +def test_chunk_and_recurrent_rules_agree(): + """The chunked form (what the fla prefill kernel implements) against the sequential + definition, both fp32: the chunk oracle is only worth anything if it reproduces the + recurrence to fp32 precision.""" + _, ref = _make_layer(3, seed=1) + torch.manual_seed(17) + hidden = torch.randn(1000, HIDDEN, device=DEV, dtype=torch.bfloat16) + torch.testing.assert_close( + _ref_out(ref, hidden, use_chunk_rule=True), _ref_out(ref, hidden), rtol=1e-4, atol=1e-4 + ) + + +def test_output_gate_comes_from_the_config(): + """The gate activation is the group config's string, not a hardcoded silu. Both gates track + their own reference, and the two are far apart -- so a stuck activation cannot pass.""" + op_silu, ref_silu = _make_layer(3, output_gate="silu", seed=2) + hidden, _, out_silu = _prefill(op_silu, _ctx(3), [128], seed=19) + torch.testing.assert_close(out_silu.float(), _ref_out(ref_silu, hidden[0]), rtol=RTOL, atol=ATOL) + + op_sig, ref_sig = _make_layer(3, output_gate="sigmoid", seed=2) + _, _, out_sig = _prefill(op_sig, _ctx(3), [128], seed=19) + torch.testing.assert_close(out_sig.float(), _ref_out(ref_sig, hidden[0]), rtol=RTOL, atol=ATOL) + + assert (out_sig.float() - out_silu.float()).abs().max().item() > 10 * ATOL diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py new file mode 100644 index 0000000000..0d2461427e --- /dev/null +++ b/tests/models/qwen4_exp/test_ple.py @@ -0,0 +1,697 @@ +"""PLE layer acceptance: hash vs HF, pinned-host table vs the GPU oracle, conv state +advancement (prefill / chunked / stepwise decode), CUDA-graph decode, and prefetch overlap. + +The HF ground truth comes from ``ple_hf_ref.py`` run under a transformers build that ships +qwen4_exp (``FREETOKEN_QWEN4_HF_PYTHON``); those tests skip when it is unset. +""" + +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest +import torch + +from freetoken.models.config import ModelConfig +from freetoken.models.qwen4_exp.config import parse_config +from freetoken.models.qwen4_exp.ple import ( + GpuResidentTable, + PinnedUVATable, + PLELayer, + PLEMetadata, + build_ple_metadata, + commit_ngram_context, + short_conv_reference, +) + +from .common import EOS, VOCAB, hash_constants, requires_cuda, toy_hf_config + +_HF_REF_PYTHON = os.environ.get("FREETOKEN_QWEN4_HF_PYTHON", "") +_HF_REF_SCRIPT = Path(__file__).with_name("ple_hf_ref.py") + +requires_hf_ref = pytest.mark.skipif( + not (_HF_REF_PYTHON and Path(_HF_REF_PYTHON).exists()), + reason="set FREETOKEN_QWEN4_HF_PYTHON to a transformers build that ships qwen4_exp", +) + + +# -------------------------------------------------------------------------------------- +# fixtures +# -------------------------------------------------------------------------------------- + +def _config() -> ModelConfig: + return parse_config(toy_hf_config()) + + +def _padded_vocab(args) -> int: + """HF pads the concatenated per-head vocabs up to make_ngram_vocab_size_divisible_by.""" + _, sizes, _ = hash_constants(args) + div = args.make_ngram_vocab_size_divisible_by + return -(-int(sizes.sum()) // div) * div + + +def _make_layer(config, *, device="cpu", dtype=torch.float32, rows=None, seed=3, table=None): + from freetoken.utils.torch_utils import torch_dtype + + args = config.qwen4_args + rows = _padded_vocab(args) if rows is None else rows + device = torch.device(device) + gen = torch.Generator(device=device).manual_seed(seed) + with torch.device(device), torch_dtype(dtype): + layer = PLELayer(config, args.ple_layer_ids[0]) + for tensor in layer.state_dict().values(): + if tensor.is_floating_point(): + tensor.normal_(0.0, 0.05, generator=gen) + multipliers, sizes, offsets = hash_constants(args) + layer.ple_embedding.layer_multipliers.copy_(multipliers) + layer.ple_embedding.ngram_heads_vocab_sizes.copy_(sizes) + layer.ple_embedding.ngram_heads_offsets.copy_(offsets) + if table is None: + weight = torch.randn(rows, args.ngram_head_dim, generator=gen, device=device, dtype=dtype) + table = GpuResidentTable(weight * 0.05, dtype=dtype) + layer.ple_embedding.attach_table(table) + return layer + + +def _meta(sequences, contexts, *, device="cpu", slots=None, fresh=None, decode=False): + lens = [len(s) for s in sequences] + to = lambda xs, dtype: torch.tensor(xs, dtype=dtype, device=device) + cu = torch.tensor([0, *lens], dtype=torch.int64).cumsum(0).to(device) + return PLEMetadata( + input_ids=to([t for s in sequences for t in s], torch.int64), + cu_seqlens=cu, + seq_lens=tuple(lens), + ngram_context=to(contexts, torch.int64), + state_slots=( + torch.arange(len(sequences), dtype=torch.int64, device=device) + if slots is None + else to(slots, torch.int64) + ), + fresh_slots=None if fresh is None else to(fresh, torch.bool), + is_decode=decode, + ) + + +def _run_hf_reference(tmp_path, data: dict, layer_idx=2, ple_layer_index=0) -> dict: + spec = {"config": vars(toy_hf_config().text_config), "layer_idx": layer_idx, "ple_layer_index": ple_layer_index} + (tmp_path / "spec.json").write_text(json.dumps(spec), encoding="utf-8") + np.savez(tmp_path / "in.npz", **data) + subprocess.run( + [_HF_REF_PYTHON, str(_HF_REF_SCRIPT), str(tmp_path / "spec.json"), + str(tmp_path / "in.npz"), str(tmp_path / "out.npz")], + check=True, + capture_output=True, + ) + return dict(np.load(tmp_path / "out.npz")) + + +# -------------------------------------------------------------------------------------- +# hash +# -------------------------------------------------------------------------------------- + +# sequence start (all-eos context), eos inside the chunk, eos as the newest context token +_HASH_CASES = [ + ([EOS, EOS], [3, 4, EOS, 5, 6, 8]), + ([21, 22], [2, EOS, 11, 12, 13, 14]), + ([EOS, 31], [9, 10, 11, 12, 13, 14]), + ([31, EOS], [9, 10, 11, 12, 13, 14]), +] + + +@requires_hf_ref +def test_hash_ids_match_hf(tmp_path): + """row_ids equals HF Qwen4ExpTextNGramEmbedding per id, over eos boundaries and at sequence start.""" + config = _config() + layer = _make_layer(config) + # HF pads its own all-eos context, so feeding [context | tokens] reproduces a resumed request + tokens = np.array([c + s for c, s in _HASH_CASES], dtype=np.int64) + ref = _run_hf_reference(tmp_path, {"hash_tokens": tokens, **_layer_ref_inputs(config, layer)}) + hf_ids = torch.as_tensor(ref["hash_ids"])[:, len(_HASH_CASES[0][0]) :] + + contexts = [c for c, _ in _HASH_CASES] + sequences = [s for _, s in _HASH_CASES] + got = layer.ple_embedding.row_ids(_meta(sequences, contexts)) + offset = 0 + for i, seq in enumerate(sequences): + assert torch.equal(got[offset : offset + len(seq)], hf_ids[i]), f"request {i}" + offset += len(seq) + + +@requires_hf_ref +def test_hash_constants_match_hf(tmp_path): + """derive_ngram_hash_constants reproduces the multipliers/vocab sizes/offsets HF builds at init.""" + config = _config() + layer = _make_layer(config) + ref = _run_hf_reference( + tmp_path, + {"hash_tokens": np.array([[EOS, EOS, 3, 4]], dtype=np.int64), **_layer_ref_inputs(config, layer)}, + ) + multipliers, sizes, offsets = hash_constants(config.qwen4_args) + assert torch.equal(multipliers, torch.as_tensor(ref["layer_multipliers"])) + assert torch.equal(sizes, torch.as_tensor(ref["ngram_heads_vocab_sizes"])) + assert torch.equal(offsets, torch.as_tensor(ref["ngram_heads_offsets"])) + + +def test_decode_hash_matches_prefill_hash(): + """The decode window (context + one token) hashes to the same ids as the same token in a prefill.""" + config = _config() + layer = _make_layer(config) + sequences = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12, 13, 14]] + contexts = [[EOS, EOS], [21, 22]] + prefill = layer.ple_embedding.row_ids(_meta(sequences, contexts)) + for step in range(len(sequences[0])): + window = [(contexts[i] + s)[step : step + 2] for i, s in enumerate(sequences)] + got = layer.ple_embedding.row_ids( + _meta([[s[step]] for s in sequences], window, decode=True) + ) + for i in range(len(sequences)): + assert torch.equal(got[i], prefill[i * len(sequences[0]) + step]) + + +# -------------------------------------------------------------------------------------- +# table backends +# -------------------------------------------------------------------------------------- + + +def _pinned_bank(rows: int, dim: int, dtype: torch.dtype, seed: int = 11): + from freetoken.moe.host_banks import HostBank + + gen = torch.Generator().manual_seed(seed) + bank = HostBank((rows, dim), dtype) + bank.tensor.copy_((torch.randn(rows, dim, generator=gen) * 0.4).to(dtype)) + bank.pin() + return bank + + +@requires_cuda +@pytest.mark.parametrize("dtype", [torch.float8_e4m3fn, torch.bfloat16]) +def test_pinned_uva_matches_gpu_resident(dtype): + """PinnedUVATable is bitwise equal to the GPU-resident oracle, through lookup and prefetch.""" + rows, dim, scale = 8192, 160, 0.0234375 + bank = _pinned_bank(rows, dim, dtype) + oracle = GpuResidentTable(bank.tensor.cuda(), scale, dtype=torch.bfloat16) + pinned = PinnedUVATable(bank.tensor, scale) + + ids = torch.randint(0, rows, (37, 16), device="cuda") + want = oracle.lookup(ids) + assert torch.equal(pinned.lookup(ids), want) + + pinned.prefetch(ids) + assert torch.equal(pinned.lookup(ids), want) + + # a stale prefetch must still be joined before its staging buffer is reused + pinned.prefetch(ids) + other = torch.randint(0, rows, (37, 16), device="cuda") + assert torch.equal(pinned.lookup(other), oracle.lookup(other)) + + out = torch.empty(37, 16 * dim, dtype=torch.bfloat16, device="cuda") + assert pinned.lookup(ids, out) is out + assert torch.equal(out, want) + + +@requires_cuda +def test_pinned_uva_zeroes_out_of_range_ids(): + bank = _pinned_bank(64, 160, torch.float8_e4m3fn) + pinned = PinnedUVATable(bank.tensor, 1.0) + ids = torch.tensor([[0, 64, 1, -1]], device="cuda") + rows = pinned.lookup(ids).view(4, 160) + assert rows[1].abs().sum() == 0 and rows[3].abs().sum() == 0 + assert torch.equal(rows[0], bank.tensor[0].cuda().to(torch.bfloat16)) + + +@pytest.mark.skipif( + not os.environ.get("FREETOKEN_QWEN4EXP_MODEL"), reason="needs FREETOKEN_QWEN4EXP_MODEL" +) +@requires_cuda +def test_pinned_uva_real_table(): + """The real 47.7 GiB FP8 table: sampled rows equal the checkpoint bytes dequantized on CPU.""" + import safetensors + from freetoken.models.qwen4_exp.weight import _PLE_SHARD_RE, _ple_table_files, load_ple_table + + path = os.environ["FREETOKEN_QWEN4EXP_MODEL"] + with open(os.path.join(path, "config.json"), encoding="utf-8") as fh: + text = json.load(fh)["text_config"] + heads = (text["ngram_size"] - 1) * text["heads_per_ngram"] + args = SimpleNamespace( + split_ngram_parts=text["split_ngram_parts"], + ngram_head_dim=text["ple_embed_dim"] // heads, + ) + table = load_ple_table(path, args) + scale = float(table.weight_scale) + rows_per_shard = table.tensor.shape[0] // args.split_ngram_parts + backend = PinnedUVATable(table.tensor, scale) + + gen = torch.Generator().manual_seed(5) + sample = torch.randint(0, table.tensor.shape[0], (1000,), generator=gen) + got = backend.lookup(sample.view(-1, 1).cuda()).cpu() + + shard_key = {} + for file in _ple_table_files(path): + with safetensors.safe_open(file, framework="pt", device="cpu") as fh: + for key in fh.keys(): + match = _PLE_SHARD_RE.search(key) + if match is not None: + shard_key[int(match.group("shard"))] = (file, key) + + by_file = {} + for i, row in enumerate(sample.tolist()): + file, key = shard_key[row // rows_per_shard] + by_file.setdefault(file, []).append((i, key, row % rows_per_shard)) + for file, items in by_file.items(): + with safetensors.safe_open(file, framework="pt", device="cpu") as fh: + for i, key, offset in items: + raw = fh.get_slice(key)[offset : offset + 1] + want = (raw.float() * scale).to(torch.bfloat16).reshape(-1) + assert torch.equal(got[i], want), f"sample {i}" + + +# -------------------------------------------------------------------------------------- +# conv state +# -------------------------------------------------------------------------------------- + + +def _forward(layer, R, meta, states): + return layer.forward(R, batch=None, meta=meta, conv_states=states) + + +def test_prefill_conv_matches_reference(): + """The packed single-conv prefill equals the per-request reference conv, chunks shorter than the state included.""" + torch.manual_seed(12) + config = _config() + args = config.qwen4_args + layer = _make_layer(config) + sequences = [[3, 4, EOS, 5, 6, 8, 9, 2, 4, 5, 6], [2, EOS, 11], [9]] + contexts = [[EOS, EOS], [21, 22], [EOS, 31]] + meta = _meta(sequences, contexts) + total = sum(len(s) for s in sequences) + x = torch.randn(total, args.ple_state_width) + states = torch.randn(len(sequences), args.ple_state_width, args.ple_conv_state_len) * 0.1 + + got_states = states.clone() + got = layer._short_conv(x, meta, got_states) + ref_states = states.clone() + ref = short_conv_reference(x, meta, ref_states, layer.conv1d.weight, args.ple_conv_dilation) + assert torch.allclose(got, ref, rtol=1e-5, atol=1e-6) + assert torch.allclose(got_states, ref_states, rtol=1e-5, atol=1e-6) + + +def test_fresh_slots_read_a_zero_state(): + """A request marked fresh ignores whatever the pool slot still holds.""" + torch.manual_seed(13) + config = _config() + args = config.qwen4_args + layer = _make_layer(config) + meta = _meta([[3, 4, 5], [6, 7, 8]], [[EOS, EOS]] * 2, fresh=[True, False]) + x = torch.randn(6, args.ple_state_width) + dirty = torch.randn(2, args.ple_state_width, args.ple_conv_state_len) + clean = dirty.clone() + clean[0] = 0 + got = layer._short_conv(x, meta, dirty.clone()) + want = layer._short_conv(x, _meta([[3, 4, 5], [6, 7, 8]], [[EOS, EOS]] * 2), clean.clone()) + assert torch.equal(got, want) + + +@pytest.mark.parametrize("cuts", [[1], [2, 3, 4], [9]], ids=["first-token", "uneven-mix", "penultimate"]) +def test_chunked_prefill_matches_one_shot(cuts): + """Chunked prefill at arbitrary cut points (including chunks shorter than the conv state) matches one shot.""" + torch.manual_seed(14) + config = _config() + args = config.qwen4_args + layer = _make_layer(config) + sequences = [[3, 4, EOS, 5, 6, 8, 9, 2, 4, 5, 6, 7], [2, EOS, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]] + length = len(sequences[0]) + contexts = [[EOS, EOS], [21, 22]] + x = torch.randn(len(sequences) * length, args.ple_state_width) + per_req = [x[i * length : (i + 1) * length] for i in range(len(sequences))] + zeros = torch.zeros(len(sequences), args.ple_state_width, args.ple_conv_state_len) + + full_states = zeros.clone() + full = _forward(layer, x, _meta(sequences, contexts), full_states) + + chunk_states = zeros.clone() + pieces, start = [], 0 + for size in [*cuts, length]: + end = min(start + size, length) + if end == start: + continue + window = [(c + s)[start : start + 2] for c, s in zip(contexts, sequences)] + pieces.append( + _forward( + layer, + torch.cat([r[start:end] for r in per_req]), + _meta([s[start:end] for s in sequences], window), + chunk_states, + ) + ) + start = end + + for i, seq in enumerate(sequences): + rebuilt = torch.cat( + [p.chunk(len(sequences))[i] for p in pieces] + ) + assert torch.allclose(rebuilt, full[i * length : (i + 1) * length], rtol=1e-4, atol=1e-5) + assert torch.allclose(chunk_states, full_states, rtol=1e-4, atol=1e-5) + + +def _state_pool(config, num_slots=8): + from freetoken.kvcache.linear_state_pool import LinearStatePool + + return LinearStatePool( + config.linear_attention_group(), num_slots, torch.float32, + torch.device("cpu"), tp_size=1, slot_states=config.slot_states, + ) + + +def _track_batch(req, tokens, pool): + """Prefill batch whose FLAMetadata carries the hybrid-radix track indices for ``req``.""" + import freetoken.core as core + from freetoken.attention.linear import build_fla_metadata + from freetoken.core import Context, set_global_ctx + + core._GLOBAL_CTX = None # test-only: build_fla_metadata reads the state pool off the ctx + set_global_ctx(Context(page_size=64, linear_state_pool=pool)) + batch = _fake_batch([req], decode=False, input_ids=tokens) + batch.fla_metadata = build_fla_metadata(batch, torch.device("cpu")) + return batch + + +def _tracked_req(table_idx, cached_len, tokens, *, live, ping_pong): + req = _req(table_idx, cached_len, tokens, extend_len=len(tokens)) + req.linear_slot_idx = live + req.mamba_ping_pong = ping_pong + req.mamba_next_track_idx = 0 + return req + + +def _no_eos_tokens(n, start=0): + return [(t + start) * 13 % (VOCAB - 8) + 8 for t in range(n)] + + +def test_track_snapshot_equals_a_prefill_stopped_at_the_boundary(): + """The snapshot in the donated slot equals the state a prefill truncated at the boundary leaves.""" + from freetoken.kernel.fla.chunk import CHUNK_SIZE + + torch.manual_seed(17) + config = _config() + args = config.qwen4_args + layer = _make_layer(config) + pool = _state_pool(config) + live, dst = 1, 5 + tokens = _no_eos_tokens(CHUNK_SIZE + 6) + req = _tracked_req(0, 0, tokens, live=live, ping_pong=(dst, 6)) + batch = _track_batch(req, tokens, pool) + + fla = batch.fla_metadata + assert fla.track_dst.tolist() == [dst] + assert req.mamba_last_track_seqlen == CHUNK_SIZE + assert fla.track_boundary_row.tolist() == [CHUNK_SIZE] + + R = torch.randn(len(tokens), args.ple_state_width) + slab = pool.slot_state("ple_conv", args.ple_layer_ids[0]) + layer.forward(R, batch, meta=_meta([tokens], [[EOS, EOS]], slots=[live]), conv_states=slab) + got = pool.slot_state("ple_conv", args.ple_layer_ids[0])[dst].clone() + + stopped = torch.zeros_like(slab) + _forward(layer, R[:CHUNK_SIZE], _meta([tokens[:CHUNK_SIZE]], [[EOS, EOS]], slots=[live]), stopped) + assert torch.equal(got, stopped[live]) + + +def test_prefix_hit_matches_the_uncached_run(): + """A prefix hit that COW-restores the donated snapshot reproduces the tail of an uncached prefill.""" + from freetoken.kernel.fla.chunk import CHUNK_SIZE + + torch.manual_seed(18) + config = _config() + args = config.qwen4_args + layer = _make_layer(config) + pool = _state_pool(config) + tokens = _no_eos_tokens(CHUNK_SIZE + 6) + context = [[EOS, EOS]] + R = torch.randn(len(tokens), args.ple_state_width) + + uncached = _forward( + layer, R, _meta([tokens], context, slots=[1]), torch.zeros_like(pool.slot_state("ple_conv", args.ple_layer_ids[0])) + ) + + live, dst = 1, 5 + req = _tracked_req(0, 0, tokens, live=live, ping_pong=(dst, 6)) + batch = _track_batch(req, tokens, pool) + layer.forward(R, batch, meta=_meta([tokens], context, slots=[live]), conv_states=pool.slot_state("ple_conv", args.ple_layer_ids[0])) + + resumed_slot = 3 + pool.copy_from(dst, resumed_slot) + tail = tokens[CHUNK_SIZE:] + got = _forward( + layer, + R[CHUNK_SIZE:], + _meta([tail], [tokens[CHUNK_SIZE - 2 : CHUNK_SIZE]], slots=[resumed_slot]), + pool.slot_state("ple_conv", args.ple_layer_ids[0]), + ) + assert torch.allclose(got, uncached[CHUNK_SIZE:], rtol=1e-5, atol=1e-6) + + +def test_prefill_matches_stepwise_decode(): + """bs=3 ragged prefill equals feeding the same tokens one decode step at a time.""" + torch.manual_seed(15) + config = _config() + args = config.qwen4_args + layer = _make_layer(config) + sequences = [[3, 4, EOS, 5, 6, 8, 9, 2, 4, 5, 6, 12], [2, EOS, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20], + [9, 10, 11, 12, 13, 14, EOS, 16, 17, 18, 19, 20]] + length = len(sequences[0]) + contexts = [[EOS, EOS], [21, 22], [EOS, 31]] + x = torch.randn(len(sequences) * length, args.ple_state_width) + per_req = [x[i * length : (i + 1) * length] for i in range(len(sequences))] + zeros = torch.zeros(len(sequences), args.ple_state_width, args.ple_conv_state_len) + + full_states = zeros.clone() + full = _forward(layer, x, _meta(sequences, contexts), full_states) + + step_states = zeros.clone() + steps = [] + for t in range(length): + window = [(c + s)[t : t + 2] for c, s in zip(contexts, sequences)] + steps.append( + _forward( + layer, + torch.stack([r[t] for r in per_req]), + _meta([[s[t]] for s in sequences], window, decode=True), + step_states, + ) + ) + for i in range(len(sequences)): + got = torch.stack([step[i] for step in steps]) + assert torch.allclose(got, full[i * length : (i + 1) * length], rtol=1e-4, atol=1e-5) + assert torch.allclose(step_states, full_states, rtol=1e-4, atol=1e-5) + + +# -------------------------------------------------------------------------------------- +# full layer vs HF +# -------------------------------------------------------------------------------------- + + +def _layer_ref_inputs(config, layer, tokens=None, hidden=None): + args = config.qwen4_args + data = { + "key_proj": layer.key_proj.weight.float().cpu().numpy(), + "value_proj": layer.value_proj.weight.float().cpu().numpy(), + "norm_key": layer.norm_key.weight.float().cpu().numpy(), + "norm_query": layer.norm_query.weight.float().cpu().numpy(), + "norm_conv": layer.norm_conv.weight.float().cpu().numpy(), + "conv1d": layer.conv1d.weight.float().cpu().numpy(), + "table": layer.ple_embedding.table.weight.float().cpu().numpy(), + } + if tokens is None: + tokens = np.array([[3, 4]], dtype=np.int64) + if hidden is None: + hidden = np.zeros((1, tokens.shape[1], args.ple_state_width), dtype=np.float32) + data["layer_tokens"] = tokens + data["hidden"] = hidden + return data + + +@requires_cuda +@requires_hf_ref +def test_layer_matches_hf(tmp_path): + """bf16 PLELayer output matches the fp32 HF Qwen4ExpTextPLELayer within 2e-2.""" + torch.manual_seed(16) + config = _config() + args = config.qwen4_args + layer = _make_layer(config) + tokens = [3, 4, EOS, 5, 6, 8, 9, 2, 4, 5, 6, 12, 13, 14] + hidden = (torch.randn(1, len(tokens), args.ple_state_width) * 0.5).numpy() + ref = _run_hf_reference( + tmp_path, + { + "hash_tokens": np.array([[EOS, EOS, 3]], dtype=np.int64), + **_layer_ref_inputs(config, layer, np.array([tokens], dtype=np.int64), hidden), + }, + ) + want = torch.as_tensor(ref["layer_out"])[0] + assert int(ref["padded_vocab_size"]) == layer.ple_embedding.table.num_rows + + gpu = _make_layer(config, device="cuda", dtype=torch.bfloat16) + for name in ("key_proj", "value_proj", "norm_key", "norm_query", "norm_conv"): + getattr(gpu, name).weight.copy_(getattr(layer, name).weight) + gpu.conv1d.weight.copy_(layer.conv1d.weight) + gpu.ple_embedding.attach_table( + GpuResidentTable(layer.ple_embedding.table.weight.to("cuda", torch.bfloat16), dtype=torch.bfloat16) + ) + R = torch.as_tensor(hidden)[0].to("cuda", torch.bfloat16) + states = torch.zeros(1, args.ple_state_width, args.ple_conv_state_len, device="cuda", dtype=torch.bfloat16) + got = _forward(gpu, R, _meta([tokens], [[EOS, EOS]], device="cuda"), states) + assert torch.allclose(got.float().cpu(), want, rtol=2e-2, atol=2e-2) + + +# -------------------------------------------------------------------------------------- +# metadata +# -------------------------------------------------------------------------------------- + + +def _fake_batch(reqs, *, decode, input_ids, positions=None, table_idx=None, device="cpu"): + return SimpleNamespace( + padded_reqs=reqs, + reqs=reqs, + is_decode=decode, + is_prefill=not decode, + input_ids=torch.tensor(input_ids, dtype=torch.int64, device=device), + positions=None if positions is None else torch.tensor(positions, dtype=torch.int32, device=device), + linear_table_idx=( + None if table_idx is None else torch.tensor(table_idx, dtype=torch.int32, device=device) + ), + ) + + +def _req(table_idx, cached_len, host_ids, extend_len=1): + return SimpleNamespace( + table_idx=table_idx, + cached_len=cached_len, + extend_len=extend_len, + linear_slot_idx=None, + input_ids=torch.tensor(host_ids, dtype=torch.int64), + ) + + +def test_commit_writes_the_track_slot_at_the_boundary(): + """The donated snapshot must carry the context AT the xCHUNK boundary, not the chunk end.""" + from freetoken.kernel.fla.chunk import CHUNK_SIZE + + args = _config().qwen4_args + eos = args.ngram_boundary_token_id + ctxp = torch.full((8, 2), eos, dtype=torch.int32) + tokens = _no_eos_tokens(CHUNK_SIZE + 6) + batch = _fake_batch([_req(1, 0, tokens, extend_len=len(tokens))], decode=False, input_ids=tokens) + meta = build_ple_metadata(batch, args, torch.device("cpu"), context_pool=ctxp) + fla = SimpleNamespace( + track_boundary_row=torch.tensor([CHUNK_SIZE]), track_dst=torch.tensor([5]) + ) + commit_ngram_context(meta, fla, ctxp) + assert ctxp[1].tolist() == tokens[-2:] + assert ctxp[5].tolist() == tokens[CHUNK_SIZE - 2 : CHUNK_SIZE] + + +def test_context_matches_the_token_history_across_chunks_and_decode(): + """Rolling the slot state chunk by chunk reproduces the last-2-tokens oracle exactly.""" + args = _config().qwen4_args + eos = args.ngram_boundary_token_id + ctxp = torch.full((3, 2), 99, dtype=torch.int32) # stale tenant garbage; fresh rows must mask to eos + history = _no_eos_tokens(11, start=3) + cached = 0 + for chunk in (3, 1, 2, 5): + ids = history[cached : cached + chunk] + batch = _fake_batch( + [_req(1, cached, history[: cached + chunk], extend_len=chunk)], + decode=False, input_ids=ids, + ) + meta = build_ple_metadata(batch, args, torch.device("cpu"), context_pool=ctxp) + assert meta.ngram_context.tolist() == [([eos, eos] + history[:cached])[-2:]] + commit_ngram_context(meta, None, ctxp) + cached += chunk + for step in range(3): + tok = 200 + step + batch = _fake_batch( + [_req(1, cached, history + [tok], extend_len=1)], + decode=True, input_ids=[tok], positions=[cached], table_idx=[1], + ) + meta = build_ple_metadata(batch, args, torch.device("cpu"), context_pool=ctxp) + assert meta.ngram_context.tolist() == [history[-2:]] + commit_ngram_context(meta, None, ctxp) + history.append(tok) + cached += 1 + + +# -------------------------------------------------------------------------------------- +# CUDA graph + prefetch overlap +# -------------------------------------------------------------------------------------- + + +@requires_cuda +def test_decode_graph_replay_matches_eager(): + """A captured decode PLE forward replays to the eager result, table gather included.""" + torch.manual_seed(17) + config = _config() + args = config.qwen4_args + rows, bs = 4096, 4 + layer = _make_layer(config, device="cuda", dtype=torch.bfloat16, rows=rows) + bank = _pinned_bank(rows, args.ngram_head_dim, torch.float8_e4m3fn) + layer.ple_embedding.attach_table(PinnedUVATable(bank.tensor, 0.05)) + + ctxp = torch.full((bs + 1, 2), EOS, dtype=torch.int32, device="cuda") + ctxp[1:] = torch.randint(0, VOCAB, (bs, 2), device="cuda", dtype=torch.int32) + positions = torch.full((bs,), 8, dtype=torch.int32, device="cuda") + slots = torch.arange(1, bs + 1, dtype=torch.int32, device="cuda") + batch = SimpleNamespace( + padded_reqs=[None] * bs, is_decode=True, is_prefill=False, + input_ids=torch.randint(0, VOCAB, (bs,), device="cuda", dtype=torch.int32), + positions=positions, linear_table_idx=slots, + ) + R = torch.randn(bs, args.ple_state_width, device="cuda", dtype=torch.bfloat16) + states0 = torch.randn(bs + 1, args.ple_state_width, args.ple_conv_state_len, + device="cuda", dtype=torch.bfloat16) * 0.1 + states = states0.clone() + + def step(): + layer.start_prefetch(batch, build_ple_metadata(batch, args, R.device, context_pool=ctxp)) + return layer.forward(R, batch, conv_states=states) + + eager = step().clone() + eager_states = states.clone() + + warmup = torch.cuda.Stream() + warmup.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(warmup): + for _ in range(3): + states.copy_(states0) + step() + torch.cuda.current_stream().wait_stream(warmup) + + graph = torch.cuda.CUDAGraph() + states.copy_(states0) + with torch.cuda.graph(graph): + static_out = step() + states.copy_(states0) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(static_out, eager) + assert torch.equal(states, eager_states) + + # new inputs in the same buffers must flow through the replay + ctxp[1:] = torch.randint(0, VOCAB, (bs, 2), device="cuda", dtype=torch.int32) + batch.input_ids.copy_(torch.randint(0, VOCAB, (bs,), device="cuda", dtype=torch.int32)) + states.copy_(states0) + graph.replay() + replayed = static_out.clone() + states.copy_(states0) + assert torch.equal(step(), replayed) + + # a bigger eager gather (prefill) must not move the buffer the graph writes into + layer.ple_embedding.table.lookup(torch.randint(0, rows, (4096, 16), device="cuda")) + states.copy_(states0) + graph.replay() + torch.cuda.synchronize() + assert torch.equal(static_out, replayed) diff --git a/tests/models/qwen4_exp/test_ple_disk.py b/tests/models/qwen4_exp/test_ple_disk.py new file mode 100644 index 0000000000..fa11807390 --- /dev/null +++ b/tests/models/qwen4_exp/test_ple_disk.py @@ -0,0 +1,334 @@ +"""Disk PLE backend, module level: store byte fidelity, on-disk layouts and errors, table vs GPU oracle, and the CUDA-graph sync protocol.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +from freetoken.models.qwen4_exp.config import parse_config +from freetoken.models.qwen4_exp.ple import GpuResidentTable, NGramEmbedding + +from .common import EOS, hash_constants, requires_cuda, toy_hf_config +from .test_ple import _meta + +_ple_store = pytest.importorskip("freetoken.kernel._ple_store") + +_KEY_PREFIX = "model.layers.1.ple.ple_embedding.ngram_embedding" + + +def _embedding() -> NGramEmbedding: + args = parse_config(toy_hf_config()).qwen4_args + emb = NGramEmbedding(args) + multipliers, sizes, offsets = hash_constants(args) + emb.layer_multipliers.copy_(multipliers) + emb.ngram_heads_vocab_sizes.copy_(sizes) + emb.ngram_heads_offsets.copy_(offsets) + return emb + + +def _bitwise_equal(got: torch.Tensor, want: torch.Tensor) -> bool: + # random table bytes include fp8 NaN encodings, and NaN != NaN under torch.equal + return torch.equal(got.view(torch.int16), want.view(torch.int16)) + + +def _make_store(tmp_path, *, write=True, use_io_uring=True): + args = parse_config(toy_hf_config()).qwen4_args + multipliers, sizes, offsets = hash_constants(args) + total_rows = int(offsets[-1] + sizes[-1]) + cols = args.ngram_head_dim + gen = torch.Generator().manual_seed(5) + table = torch.randint(0, 256, (total_rows, cols), dtype=torch.uint8, generator=gen) + path = tmp_path / "ple-table.bin" + if write: + path.write_bytes(table.numpy().tobytes()) + store = _ple_store.PleStore( + paths=[str(path)], + extent_file=[0], + extent_base=[0], + rows_per_extent=total_rows, + row_bytes=cols, + row_stride=cols, + multipliers=multipliers.tolist(), + head_vocab_sizes=sizes.tolist(), + head_offsets=offsets.tolist(), + eos_token_id=EOS, + use_io_uring=use_io_uring, + ) + return store, table, args + + +def _fill(store, args, window, tokens): + ctx = torch.tensor([window[0], window[1], *tokens], dtype=torch.int64) + staging = torch.empty(len(tokens) * args.num_ngram_heads * args.ngram_head_dim, dtype=torch.uint8) + store.stage(ctx.data_ptr(), len(tokens), staging.data_ptr()) + store.flush(0) + return staging + + +def _write_checkpoint(tmp_path, table, n_shards): + from safetensors.torch import save_file + + per = table.shape[0] // n_shards + tensors = { + f"{_KEY_PREFIX}.shard_{i}.weight": table[i * per : (i + 1) * per].view(torch.float8_e4m3fn) + for i in range(n_shards) + } + tensors[f"{_KEY_PREFIX}.weight_scale"] = torch.tensor(0.03125, dtype=torch.bfloat16) + save_file(tensors, str(tmp_path / "model.safetensors")) + + +def _make_table(tmp_path): + from freetoken.models.qwen4_exp.ple_disk import DiskRowTable, source_from_safetensors + + args = parse_config(toy_hf_config()).qwen4_args + multipliers, sizes, offsets = hash_constants(args) + total_rows = int(offsets[-1] + sizes[-1]) + gen = torch.Generator().manual_seed(9) + table = torch.randint(0, 256, (total_rows, args.ngram_head_dim), dtype=torch.uint8, generator=gen) + n_shards = next(k for k in (4, 2, 1) if total_rows % k == 0) + _write_checkpoint(tmp_path, table, n_shards) + constants = { + "num_ngram_heads": args.num_ngram_heads, + "layer_multipliers": multipliers.tolist(), + "per_head_vocab_sizes": sizes.tolist(), + "per_head_offsets": offsets.tolist(), + "eos_token_id": EOS, + } + disk = DiskRowTable(source_from_safetensors(str(tmp_path)), constants) + oracle = GpuResidentTable(table.cuda().view(torch.float8_e4m3fn), scale=0.03125) + return disk, oracle, args + + +def _decode_batch(history, token): + req = SimpleNamespace( + input_ids=torch.tensor(history, dtype=torch.int32), + device_len=len(history) + 1, + cached_len=len(history), + ) + return SimpleNamespace( + is_decode=True, + input_ids=torch.tensor([token], dtype=torch.int32, device="cuda"), + reqs=[req], + size=1, + padded_size=1, + ) + + +def test_store_stages_bitwise_rows(tmp_path): + store, table, args = _make_store(tmp_path) + emb = _embedding() + row = args.num_ngram_heads * args.ngram_head_dim + + # full run with mid-sequence eos vs production row ids + seq = [3, 4, EOS, 5, EOS, EOS, 8, 9, 3, 4] + whole = _fill(store, args, (EOS, EOS), seq) + ids = emb.row_ids(_meta([seq], [[EOS, EOS]])) + assert torch.equal(whole, table[ids.reshape(-1)].reshape(-1)), "prefill vs oracle" + + # decode = many 1-token stages; must reproduce the same bytes + parts, window = [], (EOS, EOS) + for t in seq: + parts.append(_fill(store, args, window, [t])) + window = (window[1], t) + assert torch.equal(torch.cat(parts), whole), "split stages vs one stage" + + # several lanes merged into one flush stay independent + contexts = [(3, 4), (EOS, EOS), (7, EOS)] + ctx = torch.tensor([[o, nw, 9] for o, nw in contexts], dtype=torch.int64) + staging = torch.empty(3 * row, dtype=torch.uint8) + for i in range(len(contexts)): + store.stage(ctx.data_ptr() + 24 * i, 1, staging.data_ptr() + i * row) + store.flush(0) + for i, context in enumerate(contexts): + assert torch.equal(staging[i * row : (i + 1) * row], _fill(store, args, context, [9])), f"lane {i}" + + # hundreds of deduped reads through the 64-deep pipeline + gen = torch.Generator().manual_seed(23) + big = torch.randint(0, EOS, (150,), generator=gen, dtype=torch.int64) + got = _fill(store, args, (EOS, EOS), big.tolist()) + ids = emb.row_ids(_meta([big.tolist()], [[EOS, EOS]])) + assert torch.equal(got, table[ids.reshape(-1)].reshape(-1)), "pipeline vs oracle" + + # flush signals the flag, even when nothing was staged + flag = torch.zeros(1, dtype=torch.int64) + store.flush(flag.data_ptr()) + assert int(flag[0]) == 1, "empty flush must still signal" + + +def test_layouts_readers_and_errors(tmp_path): + # 4 extents in 2 files, out of order, unaligned junk between; the last extent ends at EOF + sizes, offsets = [500, 400, 300, 800], [0, 500, 900, 1200] + total, per, cols, eos = 2000, 500, 24, 90 + gen = torch.Generator().manual_seed(11) + table = torch.randint(0, 256, (total, cols), dtype=torch.uint8, generator=gen) + shard = lambda i: table[i * per : (i + 1) * per].numpy().tobytes() # noqa: E731 + nb = per * cols + flat = tmp_path / "flat.bin" + flat.write_bytes(table.numpy().tobytes()) + fa, fb = tmp_path / "a.bin", tmp_path / "b.bin" + fa.write_bytes(b"j" * 1231 + shard(0) + b"k" * 77 + shard(2)) + fb.write_bytes(shard(1) + b"m" * 4095 + shard(3)) + kwargs = dict( + rows_per_extent=per, row_bytes=cols, row_stride=cols, + multipliers=[3, 5, 7], head_vocab_sizes=sizes, head_offsets=offsets, + eos_token_id=eos, + ) + ref = _ple_store.PleStore( + paths=[str(flat)], extent_file=[0, 0, 0, 0], extent_base=[0, nb, 2 * nb, 3 * nb], **kwargs + ) + multi = _ple_store.PleStore( + paths=[str(fa), str(fb)], extent_file=[0, 1, 0, 1], + extent_base=[1231, 0, 1231 + nb + 77, nb + 4095], **kwargs, + ) + tokens = torch.randint(0, eos, (40,), generator=gen, dtype=torch.int64) + ctx = torch.cat((torch.tensor([eos, eos], dtype=torch.int64), tokens)) + + def run(store): + staging = torch.empty(40 * 4 * cols, dtype=torch.uint8) + store.stage(ctx.data_ptr(), 40, staging.data_ptr()) + store.flush(0) + return staging + + assert torch.equal(run(multi), run(ref)), "multi-extent vs flat" + + # thread-pool fallback must produce the same bytes as io_uring + ring, _, args = _make_store(tmp_path) + pool, _, _ = _make_store(tmp_path, write=False, use_io_uring=False) + gen2 = torch.Generator().manual_seed(31) + seq = torch.randint(0, EOS, (150,), generator=gen2, dtype=torch.int64).tolist() + assert torch.equal(_fill(pool, args, (EOS, EOS), seq), _fill(ring, args, (EOS, EOS), seq)), "pool vs ring" + + # geometry that exceeds the file is rejected at construction + del ring, pool + with open(tmp_path / "ple-table.bin", "r+b") as fh: + fh.truncate(1000) + with pytest.raises(Exception, match="extent needs"): + _make_store(tmp_path, write=False) + + # checkpoint scan guards + from safetensors.torch import save_file + + from freetoken.models.qwen4_exp.ple_disk import source_from_safetensors + + save_file( + {f"{_KEY_PREFIX}.shard_0.weight": torch.zeros(8, 4, dtype=torch.uint8), + f"{_KEY_PREFIX}.weight_scale": torch.tensor(1.0, dtype=torch.bfloat16)}, + str(tmp_path / "model.safetensors"), + ) + with pytest.raises(ValueError, match="dtype"): + source_from_safetensors(str(tmp_path)) + save_file( + {f"{_KEY_PREFIX}.shard_1.weight": torch.zeros(8, 4, dtype=torch.float8_e4m3fn), + f"{_KEY_PREFIX}.weight_scale": torch.tensor(1.0, dtype=torch.bfloat16)}, + str(tmp_path / "model.safetensors"), + ) + with pytest.raises(ValueError, match="contiguous"): + source_from_safetensors(str(tmp_path)) + save_file( + {f"{_KEY_PREFIX}.shard_0.weight": torch.zeros(8, 4, dtype=torch.float8_e4m3fn), + f"{_KEY_PREFIX}.weight_scale": torch.tensor(1.0, dtype=torch.bfloat16)}, + str(tmp_path / "model.safetensors"), + ) + save_file( + {f"{_KEY_PREFIX}.shard_0.weight": torch.zeros(8, 4, dtype=torch.float8_e4m3fn)}, + str(tmp_path / "model-2.safetensors"), + ) + with pytest.raises(ValueError, match="duplicate"): + source_from_safetensors(str(tmp_path)) + (tmp_path / "model-2.safetensors").unlink() + + # truncated checkpoint: a contiguous shard prefix passes the scan, init rejects the row count + from freetoken.models.qwen4_exp.ple_disk import DiskRowTable + + args = parse_config(toy_hf_config()).qwen4_args + multipliers, vocab, offs = hash_constants(args) + rows = int(offs[-1] + vocab[-1]) + _write_checkpoint(tmp_path, torch.zeros(rows // 2, args.ngram_head_dim, dtype=torch.uint8), 1) + constants = { + "num_ngram_heads": args.num_ngram_heads, "layer_multipliers": multipliers.tolist(), + "per_head_vocab_sizes": vocab.tolist(), "per_head_offsets": offs.tolist(), "eos_token_id": EOS, + } + with pytest.raises(ValueError, match="hash addresses"): + DiskRowTable(source_from_safetensors(str(tmp_path)), constants) + + +@requires_cuda +def test_disk_table_matches_oracle(tmp_path): + disk, oracle, args = _make_table(tmp_path) + emb = _embedding() + + # prefill: two segments, one fresh and one mid-sequence window + seqs = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12, 13, 14]] + disk.fill([torch.tensor([EOS, EOS, *seqs[0]]), torch.tensor([21, 22, *seqs[1]])], graph=False) + row_ids = emb.row_ids(_meta(seqs, [[EOS, EOS], [21, 22]])).cuda() + assert _bitwise_equal(disk.lookup(row_ids), oracle.lookup(row_ids)), "prefill" + + # decode steps with a rolling window, plus the out= contract + older, newer = 41, EOS + for token in (7, 9, 13): + disk.fill([torch.tensor([older, newer, token])], graph=False) + ids = emb.row_ids(_meta([[token]], [[older, newer]], decode=True)).cuda() + out = torch.empty((1, ids.shape[-1] * disk.head_dim), dtype=disk.dtype, device="cuda") + assert disk.lookup(ids, out) is out and _bitwise_equal(out, oracle.lookup(ids)), f"token {token}" + older, newer = newer, token + + # the engine hook end to end: eager decode, then fresh + continuation prefill + disk.host_fill_batch(_decode_batch([3, 4, EOS, 5], 9), use_graph=False) + ids = emb.row_ids(_meta([[9]], [[EOS, 5]], decode=True)).cuda() + assert _bitwise_equal(disk.lookup(ids), oracle.lookup(ids)), "hook decode" + + prompt = [3, 4, EOS, 5, 6, 8] + fresh = SimpleNamespace(input_ids=torch.tensor(prompt[:4], dtype=torch.int32), device_len=4, cached_len=0) + cont = SimpleNamespace(input_ids=torch.tensor(prompt, dtype=torch.int32), device_len=6, cached_len=4) + disk.host_fill_batch(SimpleNamespace(is_decode=False, padded_reqs=[fresh, cont]), use_graph=False) + ids = emb.row_ids(_meta([prompt[:4], prompt[4:]], [[EOS, EOS], [prompt[2], prompt[3]]])).cuda() + assert _bitwise_equal(disk.lookup(ids), oracle.lookup(ids)), "hook prefill" + + +@requires_cuda +def test_graph_sync_protocol(tmp_path, monkeypatch): + disk, oracle, args = _make_table(tmp_path) + emb = _embedding() + rows = 1 + row_ids = torch.zeros((rows, args.num_ngram_heads), dtype=torch.int64, device="cuda") + out = torch.empty((rows, args.num_ngram_heads * disk.head_dim), dtype=disk.dtype, device="cuda") + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=stream): + disk.lookup(row_ids, out) + torch.cuda.current_stream().wait_stream(stream) + + # plain fill-then-replay + disk.fill([torch.tensor([3, 4, 7])], graph=True) + graph.replay() + torch.cuda.synchronize() + ids = emb.row_ids(_meta([[7]], [[3, 4]], decode=True)).cuda() + assert _bitwise_equal(out, oracle.lookup(ids)), "capture+replay" + + if disk._wait_sync: + # launch first, fill after: an early WAIT pass would surface the previous step's bytes + older, newer = 3, 4 + for token in (7, 9, 13): + graph.replay() + disk.fill([torch.tensor([older, newer, token])], graph=True) + torch.cuda.synchronize() + ids = emb.row_ids(_meta([[token]], [[older, newer]], decode=True)).cuda() + assert _bitwise_equal(out, oracle.lookup(ids)), f"wait-sync token {token}" + older, newer = newer, token + + # the engine-shaped seam: replay inside the context, deferred fill on exit + with disk.forward_host_ctx(_decode_batch([3, 4], 7), use_graph=True): + graph.replay() + torch.cuda.synchronize() + ids = emb.row_ids(_meta([[7]], [[3, 4]], decode=True)).cuda() + assert _bitwise_equal(out, oracle.lookup(ids)), "forward_host_ctx deferred" + + # gate mode: the hook fills inline and returns no deferred + monkeypatch.setenv("FREETOKEN_PLE_SYNC", "gate") + gated, _, _ = _make_table(tmp_path) + assert not gated._wait_sync + assert gated.host_fill_batch(_decode_batch([3, 4], 5), use_graph=True) is None diff --git a/tests/models/qwen4_exp/test_qsa_backend.py b/tests/models/qwen4_exp/test_qsa_backend.py new file mode 100644 index 0000000000..1d3b944ce1 --- /dev/null +++ b/tests/models/qwen4_exp/test_qsa_backend.py @@ -0,0 +1,250 @@ +"""The QSA backend behind the real Qwen4ExpAttention layer. + +(a) dense-oracle equivalence -- while a request sees at most ``index_budget + index_ratio - 1`` + tokens every complete block is selected, so QSA IS dense attention: the selection must be + exactly the causal prefix and the layer output must match ``TorchDenseQSAReference`` (fp32) + and a flashinfer dense run over the same pool; +(b) chunked prefill at unaligned cut points equals one-shot prefill (the dual-source compress); +(c) a captured decode replay equals the eager decode step. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +from .common import Fixture, requires_cuda, parsed_config, selection_spy + +QSA_LAYER = 3 + + +def _inputs(fixture: Fixture, lengths, extra: int = 0, seed: int = 11): + generator = torch.Generator(device=fixture.device).manual_seed(seed) + return [ + torch.randn( + n + extra, fixture.config.hidden_size, device=fixture.device, + dtype=fixture.dtype, generator=generator, + ) + * 0.5 + for n in lengths + ] + + +def _assert_selection_is_causal_prefix(indices: torch.Tensor, positions: torch.Tensor) -> None: + for row, position in enumerate(positions.tolist()): + selected = indices[row][indices[row] >= 0] + assert torch.equal( + selected.sort().values, + torch.arange(position + 1, dtype=selected.dtype, device=selected.device), + ), f"row {row} (position {position}) did not select its whole causal prefix" + + +@requires_cuda +def test_prefill_is_dense_below_the_budget(monkeypatch): + """bs=3 ragged prefill, longest request exactly at budget + ratio - 1.""" + config = parsed_config() + fixture = Fixture(config, num_pages=128) + attn = fixture.layer(QSA_LAYER) + lengths = [2051, 1000, 137] + inputs = _inputs(fixture, lengths) + x = torch.cat([row[:n] for row, n in zip(inputs, lengths)]) + reqs = [fixture.req(i, 0, n) for i, n in enumerate(lengths)] + + seen = selection_spy(monkeypatch, fixture.backend) + batch = fixture.batch(reqs, "prefill") + got = attn.forward(x, batch) + _assert_selection_is_causal_prefix(seen["indices"], batch.positions) + + fixture.ctx.attn_backend = _dense_oracle(fixture) + reference = attn.forward(x, batch) + torch.testing.assert_close(got.float(), reference.float(), rtol=2e-2, atol=2e-2) + + +def _dense_oracle(fixture: Fixture): + from freetoken.models.qwen4_exp.attention import TorchDenseQSAReference + + return TorchDenseQSAReference( + fixture.config, + num_slots=fixture.num_req_slots, + max_len=4096, + device=fixture.device, + dtype=fixture.dtype, + ) + + +@requires_cuda +def test_decode_is_dense_below_the_budget(monkeypatch): + """Prefill then five decode steps, sparse path vs the fp32 dense oracle.""" + config = parsed_config() + fixture = Fixture(config, num_pages=128) + attn = fixture.layer(QSA_LAYER) + lengths, steps = [300, 411, 64], 5 + inputs = _inputs(fixture, lengths, extra=steps) + oracle = _dense_oracle(fixture) + + reqs = [fixture.req(i, 0, n) for i, n in enumerate(lengths)] + seen = selection_spy(monkeypatch, fixture.backend) + + steps_x = [torch.cat([row[:n] for row, n in zip(inputs, lengths)])] + steps_x += [ + torch.stack([row[n + step] for row, n in zip(inputs, lengths)]) for step in range(steps) + ] + for step, x in enumerate(steps_x): + if step: + for req in reqs: + fixture.step(req) + batch = fixture.batch(reqs, "prefill" if step == 0 else "decode") + fixture.ctx.attn_backend = fixture.backend + got = attn.forward(x, batch) + _assert_selection_is_causal_prefix(seen["indices"], batch.positions) + fixture.ctx.attn_backend = oracle + reference = attn.forward(x, batch) + torch.testing.assert_close(got.float(), reference.float(), rtol=2e-2, atol=2e-2) + + +@requires_cuda +def test_flashinfer_dense_matches_the_sparse_path(): + """The engine's dense FULL backend over the same pool, as an independent oracle.""" + pytest.importorskip("flashinfer") + from freetoken.attention.fi import FlashInferBackend + + config = parsed_config() + fixture = Fixture(config, num_pages=64) + attn = fixture.layer(QSA_LAYER) + length = 500 + x = _inputs(fixture, [length])[0] + req = fixture.req(0, 0, length) + got = attn.forward(x, fixture.batch([req], "prefill")) + + dense = FlashInferBackend(config) + fixture.ctx.attn_backend = SimpleNamespace( + qsa_forward=lambda q, k, v, index, layer_id, batch: dense.forward( + q, k, v, layer_id, batch + ) + ) + batch = fixture.batch([req], "prefill") + dense.prepare_metadata(batch) + reference = attn.forward(x, batch) + torch.testing.assert_close(got.float(), reference.float(), rtol=2e-2, atol=2e-2) + + +@requires_cuda +@pytest.mark.parametrize("cut", [1001, 4096, 4097], ids=["unaligned", "page-boundary", "boundary+1"]) +def test_chunked_prefill_matches_one_shot(cut: int): + """Cut points that are not multiples of index_ratio exercise the dual-source compress.""" + config = parsed_config() + fixture = Fixture(config, num_pages=512) + attn = fixture.layer(QSA_LAYER) + length = 5000 + x = _inputs(fixture, [length])[0] + + one_shot = attn.forward(x, fixture.batch([fixture.req(0, 0, length)], "prefill")) + head = fixture.req(1, 0, cut) + attn.forward(x[:cut], fixture.batch([head], "prefill")) + tail = fixture.req(1, cut, length) + got = attn.forward(x[cut:], fixture.batch([tail], "prefill")) + assert torch.equal(got, one_shot[cut:]) + + +@requires_cuda +def test_decode_graph_replay_matches_eager(): + config = parsed_config() + fixture = Fixture(config, num_pages=256) + attn = fixture.layer(QSA_LAYER) + lengths, steps = [300, 411], 4 + bs = len(lengths) + inputs = _inputs(fixture, lengths, extra=steps) + reqs = [fixture.req(i, 0, n) for i, n in enumerate(lengths)] + attn.forward( + torch.cat([row[:n] for row, n in zip(inputs, lengths)]), + fixture.batch(reqs, "prefill"), + ) + + fixture.backend.init_capture_graph(max_seq_len=fixture.page_table.shape[1], bs_list=[bs]) + dummy = SimpleNamespace( + table_idx=fixture.num_req_slots - 1, cached_len=1, device_len=2, extend_len=1 + ) + static = { + "x": torch.zeros(bs, config.hidden_size, device=fixture.device, dtype=fixture.dtype), + "positions": torch.zeros(bs, dtype=torch.int32, device=fixture.device), + "out_loc": torch.zeros(bs, dtype=torch.int32, device=fixture.device), + } + capture_batch = SimpleNamespace( + padded_reqs=[dummy] * bs, reqs=[dummy] * bs, phase="decode", size=bs, padded_size=bs, + is_prefill=False, is_decode=True, positions=static["positions"], + out_loc=static["out_loc"], attn_metadata=None, active_table_idx=None, + ) + fixture.backend.prepare_for_capture(capture_batch) + attn.forward(static["x"], capture_batch) # warmup, same metadata object as the capture + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured_out = attn.forward(static["x"], capture_batch) + torch.cuda.synchronize() + + for step in range(steps): + for req in reqs: + fixture.step(req) + x = torch.stack([row[n + step] for row, n in zip(inputs, lengths)]) + batch = fixture.batch(reqs, "decode") + static["x"].copy_(x) + static["positions"].copy_(batch.positions) + static["out_loc"].copy_(batch.out_loc) + fixture.backend.prepare_for_replay(batch) + # replay must stage into the captured buffers, never reallocate them + md = batch.attn_metadata + assert md.block_table.data_ptr() == fixture.backend._graph["block_table"].data_ptr() + graph.replay() + replayed = captured_out.clone() + eager = attn.forward(x, fixture.batch(reqs, "decode")) + assert torch.equal(replayed, eager), f"graph replay diverged at decode step {step}" + + +@requires_cuda +def test_row_chunked_scoring_matches_one_chunk(monkeypatch): + """The scoring workspace bound splits long prefills into row chunks.""" + import freetoken.attention.qsa_sparse as qsa_sparse + + config = parsed_config() + fixture = Fixture(config, num_pages=64) + attn = fixture.layer(QSA_LAYER) + length = 600 + x = _inputs(fixture, [length])[0] + whole = attn.forward(x, fixture.batch([fixture.req(0, 0, length)], "prefill")) + + columns = fixture.page_table.shape[1] // config.qwen4_args.index_ratio + monkeypatch.setattr(qsa_sparse, "_LOGITS_WORKSPACE_BYTES", 64 * columns * 4) + chunked = attn.forward(x, fixture.batch([fixture.req(1, 0, length)], "prefill")) + assert torch.equal(chunked, whole) + + +@requires_cuda +def test_two_qsa_layers_keep_separate_slab_slots(monkeypatch): + """Both QSA layers of one forward must hit their own slab slot and ring slice.""" + config = parsed_config(num_layers=8) + assert config.attention_groups[1].layer_ids == (3, 7) + fixture = Fixture(config, num_pages=64) + layers = [fixture.layer(layer_id, seed=layer_id) for layer_id in (3, 7)] + oracle = _dense_oracle(fixture) + lengths, steps = [200, 71], 3 + inputs = _inputs(fixture, lengths, extra=steps) + reqs = [fixture.req(i, 0, n) for i, n in enumerate(lengths)] + + xs = [torch.cat([row[:n] for row, n in zip(inputs, lengths)])] + xs += [torch.stack([row[n + step] for row, n in zip(inputs, lengths)]) for step in range(steps)] + for step, x in enumerate(xs): + if step: + for req in reqs: + fixture.step(req) + batch = fixture.batch(reqs, "prefill" if step == 0 else "decode") + for attn in layers: + fixture.ctx.attn_backend = fixture.backend + got = attn.forward(x, batch) + fixture.ctx.attn_backend = oracle + reference = attn.forward(x, batch) + torch.testing.assert_close(got.float(), reference.float(), rtol=2e-2, atol=2e-2) + + slab = fixture.pool.cmp_k_cache + assert not torch.equal(slab(0), slab(1)) diff --git a/tests/models/qwen4_exp/test_qsa_hf.py b/tests/models/qwen4_exp/test_qsa_hf.py new file mode 100644 index 0000000000..3d3a8382ee --- /dev/null +++ b/tests/models/qwen4_exp/test_qsa_hf.py @@ -0,0 +1,253 @@ +"""One QSA layer against the HF reference math at L = 3000. + +The fp32 reference here is transcribed from ``modeling_qwen4_exp.py`` +(``Qwen4ExpTextQSAIndexer``:611, ``Qwen4ExpTextAttention``:757): pool a group of raw index +keys in fp32, ``(1 + w)`` rmsnorm it, rope it at the group's FIRST position, score +``sum_h relu() / sqrt(index_head_dim)`` over complete blocks, keep the top +``budget // ratio``, expand, then attend to that set only. + +Two claims: the selected sets agree (ties near the top-k boundary may differ, so the bar is a +Jaccard floor) and, GIVEN the backend's own selection, the attention output matches. Set +``FREETOKEN_QWEN4_HF_PYTHON`` to an interpreter whose transformers ships ``qwen4_exp`` to run +the same comparison against the real HF module in a subprocess. +""" + +from __future__ import annotations + +import math +import os +import subprocess +import sys + +import pytest +import torch +import torch.nn.functional as F + +from .common import Fixture, requires_cuda, parsed_config, selection_spy + +QSA_LAYER = 3 +LENGTH = 3000 + + +def _plus_one_rmsnorm(x, weight, eps): + xf = x.float() + return xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) * (1.0 + weight.float()) + + +def _hf_rope(x, positions, rotary_dim, base): + """HF apply_rotary_pos_emb on [T, H, D], rotating only the first rotary_dim dims.""" + inv = 1.0 / ( + base + ** (torch.arange(0, rotary_dim, 2, device=x.device, dtype=torch.float32) / rotary_dim) + ) + freqs = positions.float().unsqueeze(-1) * inv + cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1).unsqueeze(1) + sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1).unsqueeze(1) + rotated = x[..., :rotary_dim].float() + half = rotary_dim // 2 + swapped = torch.cat([-rotated[..., half:], rotated[..., :half]], dim=-1) + return torch.cat([rotated * cos + swapped * sin, x[..., rotary_dim:].float()], dim=-1) + + +def _hf_block_scores(x, indexer, config, positions): + """[T, blocks] indexer scores; the pooled block keys do not depend on the query row.""" + args = config.qwen4_args + rotary = config.rotary_config + heads, dim, ratio = args.index_n_heads, args.index_head_dim, args.index_ratio + qk = F.linear(x.float(), indexer.index_qk_proj.weight.float()) + q = _plus_one_rmsnorm( + qk[:, : heads * dim].view(-1, heads, dim), indexer.q_layernorm.weight, config.rms_norm_eps + ) + q = _hf_rope(q, positions, rotary.rotary_dim, rotary.base) + raw = qk[:, heads * dim :] + blocks = raw.shape[0] // ratio + pooled = raw[: blocks * ratio].view(blocks, ratio, dim).mean(1) + pooled = _plus_one_rmsnorm(pooled, indexer.k_layernorm.weight, config.rms_norm_eps) + kbar = _hf_rope( + pooled.unsqueeze(1), positions[: blocks * ratio : ratio], rotary.rotary_dim, rotary.base + ).squeeze(1) + return torch.relu(torch.einsum("thd,bd->tbh", q, kbar)).sum(-1) / math.sqrt(dim) + + +def _hf_selection(scores, positions, ratio, budget): + """Per-query token ids: the top-(budget // ratio) complete blocks plus the open tail.""" + offsets = torch.arange(ratio, device=scores.device) + selected = [] + for row, position in enumerate(positions.tolist()): + visible = (position + 1) // ratio + chosen = torch.empty(0, dtype=torch.int64, device=scores.device) + if visible: + top = scores[row, :visible].topk(min(budget // ratio, visible)).indices + chosen = (top.unsqueeze(-1) * ratio + offsets).flatten() + tail = torch.arange(visible * ratio, position + 1, device=scores.device) + selected.append(torch.cat([chosen, tail]).sort().values) + return selected + + +def _hf_layer_output(x, attn, config, positions, selection): + """The HF gated-GQA layer restricted to a given per-query token selection.""" + rotary = config.rotary_config + length, dim = x.shape[0], attn.head_dim + qkv = F.linear(x.float(), attn.qkv_proj.weight.float()) + qg, k, v = qkv.split(attn._qkv_split, dim=-1) + qg = qg.view(length, attn.num_q, dim * 2) + q = _plus_one_rmsnorm(qg[..., :dim], attn.q_norm.weight, config.rms_norm_eps) + q = _hf_rope(q, positions, rotary.rotary_dim, rotary.base) + k = _plus_one_rmsnorm(k.view(length, attn.num_kv, dim), attn.k_norm.weight, config.rms_norm_eps) + k = _hf_rope(k, positions, rotary.rotary_dim, rotary.base) + repeat = attn.num_q // attn.num_kv + k = k.repeat_interleave(repeat, dim=1) + v = v.view(length, attn.num_kv, dim).repeat_interleave(repeat, dim=1).float() + out = torch.zeros(length, attn.num_q, dim, device=x.device, dtype=torch.float32) + for row, tokens in enumerate(selection): + scores = torch.einsum("hd,khd->hk", q[row], k[tokens]) * dim**-0.5 + out[row] = torch.einsum("hk,khd->hd", scores.softmax(-1), v[tokens]) + gate = torch.sigmoid(qg[..., dim:].reshape(length, -1).float()) + return F.linear(out.reshape(length, -1) * gate, attn.o_proj.weight.float()) + + +def _jaccard(indices, selection): + scores = [] + for row, tokens in enumerate(selection): + mine = set(indices[row][indices[row] >= 0].tolist()) + theirs = set(tokens.tolist()) + scores.append(len(mine & theirs) / max(len(mine | theirs), 1)) + return torch.tensor(scores) + + +@requires_cuda +def test_single_layer_matches_hf_reference(monkeypatch): + config = parsed_config() + fixture = Fixture(config, num_pages=128, max_running_req=4) + attn = fixture.layer(QSA_LAYER) + generator = torch.Generator(device=fixture.device).manual_seed(13) + x = ( + torch.randn( + LENGTH, config.hidden_size, device=fixture.device, dtype=fixture.dtype, + generator=generator, + ) + * 0.5 + ) + seen = selection_spy(monkeypatch, fixture.backend) + batch = fixture.batch([fixture.req(0, 0, LENGTH)], "prefill") + got = attn.forward(x, batch) + indices = seen["indices"] + + args = config.qwen4_args + scores = _hf_block_scores(x, attn.indexer, config, batch.positions) + reference_selection = _hf_selection( + scores, batch.positions, args.index_ratio, args.index_budget + ) + jaccard = _jaccard(indices, reference_selection) + assert jaccard.min() >= 0.97, f"worst-row Jaccard {jaccard.min():.4f}" + + own_selection = [row[row >= 0].long().sort().values for row in indices] + reference = _hf_layer_output(x, attn, config, batch.positions, own_selection) + torch.testing.assert_close(got.float(), reference, rtol=2e-2, atol=2e-2) + + +_HF_DRIVER = ''' +import sys, torch +from transformers.models.qwen4_exp.configuration_qwen4_exp import Qwen4ExpTextConfig +from transformers.models.qwen4_exp.modeling_qwen4_exp import Qwen4ExpTextAttention + +payload = torch.load(sys.argv[1], map_location="cuda", weights_only=False) +meta = payload["meta"] +config = Qwen4ExpTextConfig( + hidden_size=meta["hidden_size"], num_attention_heads=meta["num_q"], + num_key_value_heads=meta["num_kv"], head_dim=meta["head_dim"], rms_norm_eps=meta["eps"], + max_position_embeddings=meta["max_position"], + rope_parameters={"rope_type": "default", "rope_theta": meta["base"], + "partial_rotary_factor": meta["rotary_dim"] / meta["head_dim"], + "mrope_section": [11, 11, 10]}, + indexer_n_heads=meta["index_heads"], indexer_kv_heads=1, indexer_head_dim=meta["index_dim"], + indexer_budget=meta["budget"], indexer_compress_ratio=meta["ratio"], +) +config._attn_implementation = "eager" +torch.set_grad_enabled(False) +attn = Qwen4ExpTextAttention(config, layer_idx=0).to("cuda", torch.float32) +attn.load_state_dict({k: v.to("cuda", torch.float32) for k, v in payload["weights"].items()}) + +x = payload["x"].to(torch.float32).unsqueeze(0) +positions = payload["positions"].to("cuda").to(torch.long) +rotary_dim = meta["rotary_dim"] +pairs = torch.arange(0, rotary_dim, 2, device="cuda", dtype=torch.float32) +inv = 1.0 / (meta["base"] ** (pairs / rotary_dim)) +freqs = positions.float().unsqueeze(-1) * inv +cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1).unsqueeze(0) +sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1).unsqueeze(0) +length = x.shape[1] +causal = torch.arange(length, device="cuda") +mask = torch.zeros(1, 1, length, length, device="cuda", dtype=torch.float32) +mask.masked_fill_(causal[None, :] > causal[:, None], torch.finfo(torch.float32).min) + +out, _ = attn(x, (cos, sin), mask) +selected = attn.indexer(x, (cos, sin), mask, None)[0, 0] == 0 +torch.save({"out": out[0].cpu(), "selected": selected.cpu()}, sys.argv[2]) +''' + + +@requires_cuda +@pytest.mark.skipif( + not os.environ.get("FREETOKEN_QWEN4_HF_PYTHON"), + reason="set FREETOKEN_QWEN4_HF_PYTHON to a transformers build that ships qwen4_exp", +) +def test_single_layer_matches_upstream_hf(tmp_path, monkeypatch): + config = parsed_config() + fixture = Fixture(config, num_pages=128, max_running_req=4) + attn = fixture.layer(QSA_LAYER) + generator = torch.Generator(device=fixture.device).manual_seed(13) + x = ( + torch.randn( + LENGTH, config.hidden_size, device=fixture.device, dtype=fixture.dtype, + generator=generator, + ) + * 0.5 + ) + seen = selection_spy(monkeypatch, fixture.backend) + batch = fixture.batch([fixture.req(0, 0, LENGTH)], "prefill") + got = attn.forward(x, batch) + + args = config.qwen4_args + rotary = config.rotary_config + q_rows, kv_rows = attn.qo_attn_dim * 2, attn.kv_attn_dim + fused = attn.qkv_proj.weight + payload = tmp_path / "payload.pt" + result = tmp_path / "hf.pt" + driver = tmp_path / "driver.py" + driver.write_text(_HF_DRIVER) + torch.save( + { + "weights": { + "q_proj.weight": fused[:q_rows].cpu(), + "k_proj.weight": fused[q_rows : q_rows + kv_rows].cpu(), + "v_proj.weight": fused[q_rows + kv_rows :].cpu(), + "o_proj.weight": attn.o_proj.weight.cpu(), + "q_norm.weight": attn.q_norm.weight.cpu(), + "k_norm.weight": attn.k_norm.weight.cpu(), + "indexer.index_qk_proj.weight": attn.indexer.index_qk_proj.weight.cpu(), + "indexer.q_layernorm.weight": attn.indexer.q_layernorm.weight.cpu(), + "indexer.k_layernorm.weight": attn.indexer.k_layernorm.weight.cpu(), + }, + "x": x.cpu(), + "positions": batch.positions.cpu(), + "meta": { + "hidden_size": config.hidden_size, "num_q": attn.num_q, "num_kv": attn.num_kv, + "head_dim": attn.head_dim, "eps": config.rms_norm_eps, + "max_position": rotary.max_position, "base": rotary.base, + "rotary_dim": rotary.rotary_dim, "index_heads": args.index_n_heads, + "index_dim": args.index_head_dim, "budget": args.index_budget, + "ratio": args.index_ratio, + }, + }, + payload, + ) + subprocess.run( + [os.environ["FREETOKEN_QWEN4_HF_PYTHON"], str(driver), str(payload), str(result)], + check=True, stdout=sys.stderr, timeout=1800, + ) + upstream = torch.load(result, map_location=fixture.device, weights_only=False) + selection = [row.nonzero().flatten() for row in upstream["selected"].to(fixture.device)] + jaccard = _jaccard(seen["indices"], selection) + assert jaccard.min() >= 0.97, f"worst-row Jaccard {jaccard.min():.4f}" + torch.testing.assert_close(got.float(), upstream["out"].float(), rtol=2e-2, atol=2e-2) diff --git a/tests/models/qwen4_exp/test_qsa_kernels.py b/tests/models/qwen4_exp/test_qsa_kernels.py new file mode 100644 index 0000000000..4b678b3bdc --- /dev/null +++ b/tests/models/qwen4_exp/test_qsa_kernels.py @@ -0,0 +1,486 @@ +"""The modified and original QSA Triton kernels against pure-torch references. + +Only kernels FreeToken changed or wrote get unit tests: the compression kernel (re-addressed +pending ring, its own torch check) and the block top-k (original radix select, checked against +torch.topk and, through the expansion chain, against the vLLM reference semantics). score.py +and attend.py are vendored from vLLM and are covered by the backend and e2e tests. +``_qsa_mqa_paged_reference`` / ``_qsa_relative_topk_reference`` / ``_expand_qsa_indices_reference`` +are transcribed from ``vllm/tests/test_qsa_reference.py`` (Apache-2.0). +""" + +from __future__ import annotations + +import math + +import pytest +import torch + +from .common import Fixture, requires_cuda, parsed_config + +PAGE_SIZE = 64 +RATIO = 4 +BUDGET = 2048 +INDEX_DIM = 128 +CMP_PAGE = PAGE_SIZE // RATIO + + +# -------------------------------------------------------------------------------------- +# vLLM pure-torch references (tests/test_qsa_reference.py:87-227) +# -------------------------------------------------------------------------------------- + + +def _qsa_mqa_paged_reference(q, k_cache, page_table, token_to_req, visible_lengths): + pages = page_table.index_select(0, token_to_req.long()).long() + keys = k_cache[pages, :, 0, :].flatten(1, 2) + scores = torch.einsum("rhd,rnd->rnh", q.float(), keys.float()) + logits = torch.relu(scores).sum(dim=-1) / math.sqrt(q.shape[-1]) + positions = torch.arange(keys.shape[1], device=q.device).unsqueeze(0) + return logits.masked_fill(positions >= visible_lengths.unsqueeze(1), -torch.inf) + + +def _qsa_relative_topk_reference(logits, row_starts, row_ends, topk): + output = torch.full((logits.shape[0], topk), -1, dtype=torch.int32, device=logits.device) + for row in range(logits.shape[0]): + start = int(row_starts[row].item()) + length = int((row_ends[row] - row_starts[row]).item()) + width = min(length, topk) + if width: + output[row, :width] = torch.topk( + logits[row, start : start + length], width + ).indices.to(torch.int32) + return output + + +def _expand_qsa_indices_reference( + block_indices, query_positions, sequence_lengths, compress_ratio, token_topk +): + rows = block_indices.shape[0] + block_topk = token_topk // compress_ratio + output_width = token_topk + compress_ratio - 1 + offsets = torch.arange(compress_ratio, device=block_indices.device) + blocks = block_indices.long() + expanded = blocks.unsqueeze(-1) * compress_ratio + offsets + expanded = torch.where( + blocks.unsqueeze(-1) >= 0, expanded, torch.full_like(expanded, -1) + ).reshape(rows, block_topk * compress_ratio) + expanded = expanded[:, :token_topk] + expanded = torch.where( + (expanded >= 0) & (expanded < sequence_lengths.unsqueeze(1)), + expanded, + torch.full_like(expanded, -1), + ) + + tail_offsets = torch.arange(compress_ratio - 1, device=block_indices.device) + visible_tokens = query_positions + 1 + tail_start = visible_tokens // compress_ratio * compress_ratio + tail = tail_start.unsqueeze(1) + tail_offsets.unsqueeze(0) + tail_count = (visible_tokens - tail_start).unsqueeze(1) + tail_valid = (tail_offsets.unsqueeze(0) < tail_count) & ( + tail < sequence_lengths.unsqueeze(1) + ) + tail = torch.where(tail_valid, tail, torch.full_like(tail, -1)) + + result = torch.cat((expanded, tail), dim=1) + order = torch.arange(output_width, device=result.device).expand(rows, -1) + sort_key = torch.where(result >= 0, order, order + output_width) + return result.gather(1, torch.argsort(sort_key, dim=1, stable=True)).to(torch.int32) + + +class _Case: + def __init__(self, **fields): + self.__dict__.update(fields) + + +def _paged_case(length: int, bs: int, rows_per_req: int, seed: int, index_heads: int = 4): + """Synthetic paged geometry: shuffled pages, the last ``rows_per_req`` queries per request.""" + device = torch.device("cuda") + torch.manual_seed(seed) + generator = torch.Generator(device=device).manual_seed(seed) + pages_per_req = -(-length // PAGE_SIZE) + total_pages = bs * pages_per_req + block_table = ( + torch.randperm(total_pages, device=device).reshape(bs, pages_per_req).to(torch.int32) + ) + q = torch.randn( + bs * rows_per_req, index_heads, INDEX_DIM, device=device, dtype=torch.bfloat16, + generator=generator, + ) + token_to_req = torch.repeat_interleave( + torch.arange(bs, device=device, dtype=torch.int32), rows_per_req + ) + query_positions = torch.cat( + [ + torch.arange(length - rows_per_req, length, device=device, dtype=torch.int32) + for _ in range(bs) + ] + ) + seq_lens = torch.full((bs,), length, device=device, dtype=torch.int32) + return _Case( + device=device, + generator=generator, + length=length, + bs=bs, + pages_per_req=pages_per_req, + total_pages=total_pages, + block_table=block_table, + q=q, + token_to_req=token_to_req, + query_positions=query_positions, + seq_lens=seq_lens, + ) + + +@requires_cuda +@pytest.mark.parametrize("length", [20000]) +@pytest.mark.parametrize("bs", [1]) +@pytest.mark.parametrize("torch_topk", [False, True]) +def test_top_blocks_and_expansion_match_vllm_reference(length: int, bs: int, torch_topk: bool): + from freetoken.kernel.triton.qsa import expand_qsa_block_indices, qsa_mqa_paged + + config = parsed_config() + fixture = Fixture(config, num_pages=4, max_running_req=2) + case = _paged_case(length, bs, rows_per_req=4, seed=7 * length + bs) + cache = torch.randn( + case.total_pages, CMP_PAGE, 1, INDEX_DIM, device=case.device, + dtype=torch.bfloat16, generator=case.generator, + ) + rows, columns = case.q.shape[0], case.pages_per_req * CMP_PAGE + logits = torch.empty(rows, columns, dtype=torch.float32, device=case.device) + visible = torch.empty(rows, dtype=torch.int32, device=case.device) + qsa_mqa_paged( + case.q, cache, case.block_table, case.token_to_req, case.query_positions, + case.seq_lens, RATIO, logits, visible, + ) + blocks = torch.empty(rows, BUDGET // RATIO, dtype=torch.int32, device=case.device) + reference_logits = _qsa_mqa_paged_reference( + case.q, cache, case.block_table, case.token_to_req, visible + ) + if torch_topk: + fixture.backend._block_topk_kernel = None + fixture.backend._top_blocks(logits, visible, blocks) + + expected_blocks = _qsa_relative_topk_reference( + reference_logits, torch.zeros_like(visible), visible, BUDGET // RATIO + ) + # Ties between equal scores may land on either index; the SET is what selection means. + torch.testing.assert_close(blocks.sort(-1).values, expected_blocks.sort(-1).values) + + row_seq_lens = case.seq_lens.index_select(0, case.token_to_req.long()) + indices = torch.empty(rows, BUDGET + RATIO - 1, dtype=torch.int32, device=case.device) + expand_qsa_block_indices( + expected_blocks, case.query_positions, case.seq_lens, case.token_to_req, + RATIO, BUDGET, indices, + ) + expected = _expand_qsa_indices_reference( + expected_blocks, case.query_positions, row_seq_lens, RATIO, BUDGET + ) + torch.testing.assert_close(indices, expected) + + +@requires_cuda +@pytest.mark.parametrize("ring_capacity", [4, 8]) +def test_compression_reads_both_sources(ring_capacity: int): + """Members already consumed come from the ring, the rest from this forward's raw rows.""" + from freetoken.kernel.triton.qsa import qsa_compress_groups, qsa_store_rows + + device = torch.device("cuda") + dim, slots = 8, 3 + pairs = [(0, p) for p in range(2, 9)] + [(1, p) for p in range(5, 11)] + + def key(request: int, position: int) -> torch.Tensor: + return (torch.arange(dim, dtype=torch.float32) + request * 1000 + position * 10).to( + torch.bfloat16 + ) + + raw = torch.stack([key(*pair) for pair in pairs]).to(device) + token_to_req = torch.tensor([r for r, _ in pairs], dtype=torch.int32, device=device) + positions = torch.tensor([p for _, p in pairs], dtype=torch.int32, device=device) + cu_seqlens = torch.tensor([0, 7, 13], dtype=torch.int32, device=device) + ring_slots = torch.tensor([2, 0], dtype=torch.int32, device=device) + ring = torch.zeros(slots, ring_capacity, dim, device=device, dtype=torch.bfloat16) + for request, position, slot in ((0, 0, 2), (0, 1, 2), (1, 4, 0)): + ring[slot, position % ring_capacity] = key(request, position).to(device) + + pooled = torch.empty(len(pairs), dim, device=device, dtype=torch.bfloat16) + first = torch.empty(len(pairs), dtype=torch.int32, device=device) + qsa_compress_groups( + raw, ring, ring_slots, token_to_req, cu_seqlens, positions, RATIO, pooled, first + ) + + for row, (request, position) in enumerate(pairs): + if (position + 1) % RATIO: + continue + group = torch.stack([key(request, position - RATIO + 1 + k).float() for k in range(RATIO)]) + expected = group.mean(0).to(torch.bfloat16).to(device) + assert torch.equal(pooled[row], expected), (request, position) + assert int(first[row]) == position - RATIO + 1 + + # The ring keeps only each request's last ring_capacity rows. + rows = torch.arange(len(pairs), device=device) + ends = cu_seqlens.long().index_select(0, token_to_req.long() + 1) + slot = torch.where( + rows >= ends - ring_capacity, + ring_slots.long().index_select(0, token_to_req.long()) * ring_capacity + + positions.long() % ring_capacity, + torch.full_like(rows, -1), + ) + qsa_store_rows(ring, slot.to(torch.int32), raw) + for request, ring_slot in ((0, 2), (1, 0)): + for position in [p for r, p in pairs if r == request][-ring_capacity:]: + assert torch.equal( + ring[ring_slot, position % ring_capacity], key(request, position).to(device) + ) + + +# -------------------------------------------------------------------------------------- +# Block top-k (kernel/triton/qsa/topk.py) +# -------------------------------------------------------------------------------------- + +TOPK_SHAPES = [(512, 512), (4096, 512)] + + +def _torch_topk_blocks(logits, visible, width): + """The torch.topk fallback of ``qsa_sparse._top_blocks``, kept here as the reference.""" + columns = logits.shape[1] + out = torch.full((logits.shape[0], width), -1, dtype=torch.int32, device=logits.device) + column = torch.arange(columns, device=logits.device) + masked = logits.masked_fill(column.unsqueeze(0) >= visible.unsqueeze(1), -float("inf")) + take = min(width, columns) + values, chosen = torch.topk(masked, take, dim=-1) + out[:, :take] = torch.where(values > -float("inf"), chosen.to(torch.int32), -1) + return out + + +def _topk_case(n_blocks: int, bs: int, mode: str, seed: int): + device = torch.device("cuda") + generator = torch.Generator(device=device).manual_seed(seed) + logits = torch.randn(bs, n_blocks, device=device, generator=generator) + visible = torch.full((bs,), n_blocks, dtype=torch.int32, device=device) + if mode == "ties": + # Three distinct scores over every column: almost every selection sits on a tie. + logits = torch.randint(0, 3, (bs, n_blocks), device=device, generator=generator).float() + if mode == "ragged": + visible = torch.randint( + 0, n_blocks + 1, (bs,), dtype=torch.int32, device=device, generator=generator + ) + if mode == "dead": + logits[:, ::5] = -float("inf") + return logits, visible + + +@requires_cuda +@pytest.mark.parametrize("n_blocks,width", TOPK_SHAPES) +@pytest.mark.parametrize("bs", [4]) +@pytest.mark.parametrize("mode", ["random", "ties", "ragged", "dead"]) +def test_block_topk_matches_torch_topk(n_blocks: int, width: int, bs: int, mode: str): + from freetoken.kernel.triton.qsa import qsa_block_topk + + logits, visible = _topk_case(n_blocks, bs, mode, seed=31 * n_blocks + 7 * width + bs) + blocks = torch.empty(bs, width, dtype=torch.int32, device=logits.device) + qsa_block_topk(logits, visible, blocks) + expected = _torch_topk_blocks(logits, visible, width) + + # Selection is a set: torch.topk orders by descending score, the kernel by column id. + torch.testing.assert_close(blocks.sort(-1).values, expected.sort(-1).values) + live = (blocks >= 0).sum(-1) + torch.testing.assert_close(live, (expected >= 0).sum(-1)) + # expand.py reads ranks [0, complete_blocks), so a -1 may only sit in the tail. + ranks = torch.arange(width, device=blocks.device) + assert torch.equal(blocks >= 0, ranks.unsqueeze(0) < live.unsqueeze(1)) + assert bool(((blocks[:, 1:] > blocks[:, :-1]) | (blocks[:, 1:] < 0)).all()) + + +@requires_cuda +def test_block_topk_replays_in_a_cuda_graph(): + """Fixed grid, no host read: one capture serves every later sequence length.""" + from freetoken.kernel.triton.qsa import qsa_block_topk + + rows, columns, width = 4, 4096, 512 + device = torch.device("cuda") + logits = torch.randn(rows, columns, device=device) + visible = torch.full((rows,), columns, dtype=torch.int32, device=device) + blocks = torch.empty(rows, width, dtype=torch.int32, device=device) + + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + qsa_block_topk(logits, visible, blocks) + torch.cuda.current_stream().wait_stream(side) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + qsa_block_topk(logits, visible, blocks) + + for lengths in ([columns] * rows, [columns // 2, 900, 37, 0], [4095, 512, 511, 4096]): + logits.normal_() + visible.copy_(torch.tensor(lengths, dtype=torch.int32, device=device)) + blocks.fill_(0) + graph.replay() + expected = _torch_topk_blocks(logits, visible, width) + torch.testing.assert_close(blocks.sort(-1).values, expected.sort(-1).values) + + +def test_torch_topk_env_picks_the_fallback(monkeypatch): + from freetoken.attention.qsa_sparse import TORCH_TOPK_ENV, _resolve_block_topk + + assert _resolve_block_topk() is not None + monkeypatch.setenv(TORCH_TOPK_ENV, "1") + assert _resolve_block_topk() is None + + +# -------------------------------------------------------------------------------------- +# Block top-k, split + merge path (wide buffers) +# -------------------------------------------------------------------------------------- + +SPLIT_CHUNK = 4096 # _split_plan's chunk for every buffer these tests use + + +def _policy_topk_blocks(logits, visible, width): + """The kernel's documented order: highest score wins, lowest column breaks a tie.""" + columns = logits.shape[1] + column = torch.arange(columns, device=logits.device) + masked = logits.masked_fill(column.unsqueeze(0) >= visible.unsqueeze(1), -float("inf")) + take = min(width, columns) + order = masked.argsort(dim=-1, descending=True, stable=True)[:, :take] + out = torch.full((logits.shape[0], width), -1, dtype=torch.int32, device=logits.device) + out[:, :take] = torch.where( + masked.gather(1, order) > -float("inf"), order.to(torch.int32), -1 + ) + return out + + +def _split_topk_case(n_blocks: int, width: int, bs: int, mode: str, seed: int): + if mode != "boundary": + return _topk_case(n_blocks, bs, mode, seed) + # width - 212 columns beat the tie, so the 212 remaining winners start 100 columns below + # a chunk boundary and run past it into a chunk that is all tie. + logits = torch.zeros(bs, n_blocks, device="cuda") + for row in range(bs): + cut = SPLIT_CHUNK * (1 + row % (n_blocks // SPLIT_CHUNK - 1)) + logits[row, : width - 212] = 2.0 + logits[row, cut - 100 :] = 1.0 + return logits, torch.full((bs,), n_blocks, dtype=torch.int32, device="cuda") + + +@requires_cuda +@pytest.mark.parametrize("n_blocks", [65536]) +@pytest.mark.parametrize("bs", [4]) +@pytest.mark.parametrize("mode", ["random", "boundary", "ragged", "dead"]) +def test_block_topk_split_path_matches_torch_topk(n_blocks: int, bs: int, mode: str): + from freetoken.kernel.triton.qsa import qsa_block_topk, qsa_block_topk_scratch_width + + width = 512 + assert qsa_block_topk_scratch_width(n_blocks, width) > 0, "case must take the split path" + logits, visible = _split_topk_case(n_blocks, width, bs, mode, seed=n_blocks + bs + len(mode)) + blocks = torch.empty(bs, width, dtype=torch.int32, device=logits.device) + qsa_block_topk(logits, visible, blocks) + + torch.testing.assert_close( + blocks.sort(-1).values, _torch_topk_blocks(logits, visible, width).sort(-1).values + ) + # Tie determinism: the winners are the exact set the lowest-column-first policy names, + # including the ties that straddle a chunk boundary. + torch.testing.assert_close( + blocks.sort(-1).values, _policy_topk_blocks(logits, visible, width).sort(-1).values + ) + live = (blocks >= 0).sum(-1) + ranks = torch.arange(width, device=blocks.device) + assert torch.equal(blocks >= 0, ranks.unsqueeze(0) < live.unsqueeze(1)) + assert bool(((blocks[:, 1:] > blocks[:, :-1]) | (blocks[:, 1:] < 0)).all()) + + +@requires_cuda +@pytest.mark.parametrize("preallocated", [False, True]) +def test_block_topk_split_path_replays_in_a_cuda_graph(preallocated: bool): + """The split geometry comes from the buffer width, so one capture serves every length.""" + from freetoken.kernel.triton.qsa import qsa_block_topk, qsa_block_topk_scratch_width + + rows, columns, width = 4, 65536, 512 + device = torch.device("cuda") + scratch_width = qsa_block_topk_scratch_width(columns, width) + assert scratch_width > 0 + logits = torch.randn(rows, columns, device=device) + visible = torch.full((rows,), columns, dtype=torch.int32, device=device) + blocks = torch.empty(rows, width, dtype=torch.int32, device=device) + # The scratch never needs clearing: every split rewrites its own slots on every replay. + scratch = ( + torch.empty(rows, scratch_width, dtype=torch.int32, device=device) + if preallocated + else None + ) + + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + qsa_block_topk(logits, visible, blocks, scratch) + torch.cuda.current_stream().wait_stream(side) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + qsa_block_topk(logits, visible, blocks, scratch) + + for lengths in ([columns] * rows, [4096, 40000, 0, 65535], [1, 4097, 8192, columns]): + logits.normal_() + visible.copy_(torch.tensor(lengths, dtype=torch.int32, device=device)) + blocks.fill_(0) + graph.replay() + expected = _torch_topk_blocks(logits, visible, width) + torch.testing.assert_close(blocks.sort(-1).values, expected.sort(-1).values) + + +@requires_cuda +def test_block_topk_split_path_cost_tracks_live_blocks(): + """A wide buffer with a short row must not pay for the splits past its visible tail.""" + from freetoken.kernel.triton.qsa import qsa_block_topk, qsa_block_topk_scratch_width + + # wide enough that the split work dwarfs the 20-launch floor even at boosted clocks + rows, columns, width = 1, 262144, 512 + device = torch.device("cuda") + logits = torch.randn(rows, columns, device=device) + visible = torch.full((rows,), columns, dtype=torch.int32, device=device) + blocks = torch.empty(rows, width, dtype=torch.int32, device=device) + scratch = torch.empty( + rows, qsa_block_topk_scratch_width(columns, width), dtype=torch.int32, device=device + ) + + side = torch.cuda.Stream() + side.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(side): + qsa_block_topk(logits, visible, blocks, scratch) + torch.cuda.current_stream().wait_stream(side) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + for _ in range(20): + qsa_block_topk(logits, visible, blocks, scratch) + + def replay_us(live: int) -> float: + visible.fill_(live) + best = float("inf") + for _ in range(5): + start, stop = torch.cuda.Event(True), torch.cuda.Event(True) + start.record() + graph.replay() + stop.record() + torch.cuda.synchronize() + best = min(best, start.elapsed_time(stop) * 1000.0 / 20) + return best + + full, short = replay_us(columns), replay_us(SPLIT_CHUNK) + assert full > 2.0 * short, f"{columns} live {full:.1f}us vs {SPLIT_CHUNK} live {short:.1f}us" + + +@requires_cuda +def test_capture_graph_provisions_the_block_topk_scratch(): + from freetoken.kernel.triton.qsa import qsa_block_topk_scratch_width + + fixture = Fixture(parsed_config(), num_pages=320, max_running_req=2) + backend = fixture.backend + table_width = fixture.page_table.shape[1] + columns = table_width // PAGE_SIZE * CMP_PAGE + width = qsa_block_topk_scratch_width(columns, backend.block_topk) + assert width > 0, "the fixture must be wide enough to reach the split path" + + backend.init_capture_graph(table_width, [2]) + static = backend._graph["topk_scratch"] + assert static.shape[1] == width + assert backend._scratch("topk_scratch", 2, width, dtype=torch.int32).data_ptr() == ( + static.data_ptr() + ) diff --git a/tests/models/qwen4_exp/test_skeleton.py b/tests/models/qwen4_exp/test_skeleton.py new file mode 100644 index 0000000000..00f7c43f3b --- /dev/null +++ b/tests/models/qwen4_exp/test_skeleton.py @@ -0,0 +1,503 @@ +"""Skeleton tests: the frozen qwen4_exp interfaces and their torch references. + +Everything runs on a scaled-down copy of the real geometry (4 layers, full attention on layer 3, +PLE on layer 1, hc_count 4, 3-gram hash) so the shapes and the layer split are the shipping ones. +The hyper-connection and PLE references transcribed here are HF ``modeling_qwen4_exp.py`` +(``Qwen4ExpTextGatedResidual``:941, ``Qwen4ExpTextNGramEmbedding``:1018, ``Qwen4ExpTextPLELayer`` +:1117), so the torch implementations are checked against the math, not against themselves. +""" + +from __future__ import annotations + +import math +from types import SimpleNamespace + +import pytest +import torch +import torch.nn.functional as F + +from freetoken.layers import BaseOP, LinearReplicated +from freetoken.models.config import ModelConfig +from freetoken.models.qwen4_exp.config import parse_config +from freetoken.models.qwen4_exp.hc import GatedResidual +from freetoken.models.qwen4_exp.ple import GpuResidentTable, PLELayer, PLEMetadata + +from .common import EOS, hash_constants, requires_cuda, toy_hf_config + + +def _config(num_layers: int = 4) -> ModelConfig: + return parse_config(toy_hf_config(num_layers)) + + +def _fill(op, gen: torch.Generator, scale: float = 0.05) -> None: + """Random floats / zeroed ints for every state-dict tensor of an op tree.""" + for tensor in op.state_dict().values(): + if tensor.is_floating_point(): + tensor.normal_(0.0, scale, generator=gen) + else: + tensor.zero_() + + +def _group_norm(x, weight, eps, groups): + xf = x.float().reshape(*x.shape[:-1], groups, -1) + xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) + return (xf.flatten(-2) * (1.0 + weight.float())).type_as(x) + + +# -------------------------------------------------------------------------------------- +# hyper-connections +# -------------------------------------------------------------------------------------- + + +def _hf_gated_residual(hc, R, w_norm, w_down, w_up, w_inject, hidden, eps): + xn = _group_norm(R, w_norm, eps, hc) + mix = F.silu(F.linear(xn, w_down) / hc) + mix = torch.sigmoid(F.linear(mix, w_up)).unflatten(-1, (hc, hidden)) + mixed = (mix * xn.unflatten(-1, (hc, hidden))).mean(-2) + if w_inject is None: + return mixed, None + return mixed, 2 * torch.sigmoid(F.linear(xn, w_inject) / hc) + + +@pytest.mark.parametrize("tokens", [1, 7]) +def test_hc_mix_and_combine_match_hf(tokens: int): + torch.manual_seed(0) + config = _config() + args = config.qwen4_args + hc = GatedResidual(config) + _fill(hc, torch.Generator().manual_seed(1)) + + R = torch.randn(tokens, args.ple_state_width) + y = torch.randn(tokens, args.hidden_size) + x, s = hc.mix(R) + got = hc.combine(R, y, s) + + merged = hc.input_mix_weight_down_block_inject.weight + ref_x, ref_inject = _hf_gated_residual( + args.hc_count, + R, + hc.hc_norm.weight, + merged[: args.hc_lowrank], + hc.input_mix_weight_up.weight, + merged[args.hc_lowrank : args.hc_lowrank + args.hc_count], + args.hidden_size, + config.rms_norm_eps, + ) + ref = R.unflatten(-1, (args.hc_count, args.hidden_size)) + ref = (ref + y.unsqueeze(-2) * ref_inject.unsqueeze(-1)).flatten(-2) + + assert torch.allclose(x, ref_x, rtol=1e-5, atol=1e-6) + assert torch.allclose(got, ref, rtol=1e-5, atol=1e-6) + + +def test_hc_merged_gemm_layout_and_top_level_mixer(): + config = _config() + args = config.qwen4_args + hc = GatedResidual(config) + # 320 + 4 lowrank/inject rows padded to a multiple of 16 in the real config + assert hc.pad_size == (-(args.hc_lowrank + args.hc_count)) % 16 + merged = hc.input_mix_weight_down_block_inject.weight + assert merged.shape == ( + args.hc_lowrank + args.hc_count + hc.pad_size, + args.ple_state_width, + ) + assert set(hc.state_dict()) == { + "hc_norm.weight", + "input_mix_weight_down_block_inject.weight", + "input_mix_weight_up.weight", + } + + mixer = GatedResidual(config, use_combine=False) + _fill(mixer, torch.Generator().manual_seed(2)) + assert set(mixer.state_dict()) == { + "hc_norm.weight", + "input_mix_weight_down.weight", + "input_mix_weight_up.weight", + } + R = torch.randn(5, args.ple_state_width) + x, s = mixer.mix(R) + assert s is None + ref_x, ref_inject = _hf_gated_residual( + args.hc_count, + R, + mixer.hc_norm.weight, + mixer.input_mix_weight_down.weight, + mixer.input_mix_weight_up.weight, + None, + args.hidden_size, + config.rms_norm_eps, + ) + assert ref_inject is None + assert torch.allclose(x, ref_x, rtol=1e-5, atol=1e-6) + + +# -------------------------------------------------------------------------------------- +# PLE +# -------------------------------------------------------------------------------------- + + +def _hf_shift_right(tokens, shift, eos): + if shift == 0: + return tokens + batch, seq_len = tokens.shape + positions = torch.arange(seq_len) + eos_positions = torch.where(tokens == eos, positions, torch.full_like(positions, -1)) + previous = torch.cummax(eos_positions, dim=1).values + previous = torch.cat([eos_positions.new_full((batch, 1), -1), previous[:, :-1]], dim=1) + in_segment = positions.unsqueeze(0) - (previous + 1) + source = positions - shift + shifted = tokens.gather(1, source.clamp_min(0).unsqueeze(0).expand(batch, -1)) + valid = (in_segment >= shift) & (source.unsqueeze(0) >= 0) + return torch.where(valid, shifted, tokens.new_full((), eos)) + + +def _hf_ngram_ids(tokens, context, args, multipliers, sizes, offsets): + """HF Qwen4ExpTextNGramEmbedding id computation over a dense [B, L] batch.""" + history = torch.cat([context, tokens], dim=-1) + shifted = [_hf_shift_right(history, s, args.ngram_boundary_token_id) for s in range(args.ngram_size)] + blocks = [] + for ngram in range(2, args.ngram_size + 1): + start = (ngram - 2) * args.heads_per_ngram + end = start + args.heads_per_ngram + mixed = shifted[0] * multipliers[0] + for position in range(1, ngram): + mixed = torch.bitwise_xor(mixed, shifted[position] * multipliers[position]) + ids = torch.remainder(mixed.unsqueeze(-1), sizes[start:end].view(1, 1, -1)) + blocks.append(ids + offsets[start:end].view(1, 1, -1)) + return torch.cat(blocks, dim=-1)[:, -tokens.shape[1] :] + + +def _ragged(sequences, contexts, args, device="cpu"): + """Ragged PLEMetadata (prefill) for a list of per-request token lists.""" + lens = [len(s) for s in sequences] + cu = torch.tensor([0, *lens], dtype=torch.int64).cumsum(0) + return PLEMetadata( + input_ids=torch.tensor([t for s in sequences for t in s], dtype=torch.int64, device=device), + cu_seqlens=cu.to(device), + seq_lens=tuple(lens), + ngram_context=torch.tensor(contexts, dtype=torch.int64, device=device), + state_slots=torch.arange(len(sequences), dtype=torch.int64, device=device), + fresh_slots=None, + is_decode=False, + ) + + +def _make_ple(config, seed: int = 3, rows: int = 4096): + args = config.qwen4_args + gen = torch.Generator().manual_seed(seed) + layer = PLELayer(config, args.ple_layer_ids[0]) + _fill(layer, gen) + multipliers, sizes, offsets = hash_constants(args) + layer.ple_embedding.layer_multipliers.copy_(multipliers) + layer.ple_embedding.ngram_heads_vocab_sizes.copy_(sizes) + layer.ple_embedding.ngram_heads_offsets.copy_(offsets) + table = torch.randn(rows, args.ngram_head_dim, generator=gen) * 0.05 + layer.ple_embedding.attach_table(GpuResidentTable(table, dtype=torch.float32)) + return layer, (multipliers, sizes, offsets) + + +def test_ple_hash_matches_hf(): + """Ragged hash ids vs the HF dense reference, including eos inside and at the start of a request.""" + config = _config() + args = config.qwen4_args + layer, (multipliers, sizes, offsets) = _make_ple(config) + + sequences = [[3, 4, EOS, 5, 6], [EOS, 11, 12], [9]] + contexts = [[EOS, EOS], [21, 22], [EOS, 31]] + meta = _ragged(sequences, contexts, args) + got = layer.ple_embedding.row_ids(meta) + + offset = 0 + for tokens, context in zip(sequences, contexts): + ref = _hf_ngram_ids( + torch.tensor([tokens]), + torch.tensor([context]), + args, + multipliers, + sizes, + offsets, + )[0] + assert torch.equal(got[offset : offset + len(tokens)], ref) + offset += len(tokens) + # sequences[0][3] sits right after the boundary token, so its window is cut to eos padding + after_eos = layer.ple_embedding.row_ids(_ragged([[5]], [[EOS, EOS]], args))[0] + with_history = layer.ple_embedding.row_ids(_ragged([[5]], [[3, 4]], args))[0] + assert torch.equal(got[3], after_eos) + assert not torch.equal(got[3], with_history) + + +def test_ple_forward_matches_hf(): + """Full PLE forward (fp32) against the HF gate/norm chain and an explicit conv tap sum.""" + torch.manual_seed(4) + config = _config() + args = config.qwen4_args + layer, (multipliers, sizes, offsets) = _make_ple(config) + hidden, hc = args.hidden_size, args.hc_count + + sequences = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12, 13, 14]] + contexts = [[EOS, EOS], [21, 22]] + meta = _ragged(sequences, contexts, args) + total = sum(len(s) for s in sequences) + R = torch.randn(total, args.ple_state_width) + states = torch.randn(len(sequences), args.ple_state_width, args.ple_conv_state_len) * 0.1 + got = layer.forward(R, batch=None, meta=meta, conv_states=states.clone()) + + offset = 0 + for i, (tokens, context) in enumerate(zip(sequences, contexts)): + ids = _hf_ngram_ids( + torch.tensor([tokens]), torch.tensor([context]), args, multipliers, sizes, offsets + )[0] + embed = layer.ple_embedding.table.weight[ids.reshape(-1)].view(len(tokens), -1) + key = _group_norm( + F.linear(embed, layer.key_proj.weight), layer.norm_key.weight, config.rms_norm_eps, hc + ).unflatten(-1, (hc, hidden)) + value = F.linear(embed, layer.value_proj.weight) + rows = R[offset : offset + len(tokens)] + query = _group_norm( + rows, layer.norm_query.weight, config.rms_norm_eps, hc + ).unflatten(-1, (hc, hidden)) + gate = (key * query).sum(-1, keepdim=True) / math.sqrt(hidden) + gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt()) + gated = (gate * value.unsqueeze(-2)).flatten(-2) + normed = _group_norm(gated, layer.norm_conv.weight, config.rms_norm_eps, hc) + history = torch.cat([states[i], normed.transpose(0, 1)], dim=-1) + taps = sum( + layer.conv1d.weight[:, 0, k].unsqueeze(0) + * history.transpose(0, 1)[k * args.ple_conv_dilation :][: len(tokens)] + for k in range(args.ple_conv_kernel_size) + ) + ref = gated + F.silu(taps) + assert torch.allclose(got[offset : offset + len(tokens)], ref, rtol=1e-4, atol=1e-5) + offset += len(tokens) + + +def _fresh_ctx(**fields): + import freetoken.core as core + from freetoken.core import Context, set_global_ctx + + core._GLOBAL_CTX = None # test-only: each scenario builds its own ctx + ctx = Context(page_size=64) + for name, value in fields.items(): + setattr(ctx, name, value) + set_global_ctx(ctx) + return ctx + + +def _plus_one_rmsnorm(x, weight, eps): + xf = x.float() + xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) + return (xf * (1.0 + weight.float())).type_as(x) + + +def _hf_rope(x, positions, rotary_dim, base): + """HF apply_rotary_pos_emb on [T, H, D], rotating only the first rotary_dim dims.""" + inv = 1.0 / ( + base ** (torch.arange(0, rotary_dim, 2, device=x.device, dtype=torch.float32) / rotary_dim) + ) + freqs = positions.float().unsqueeze(-1) * inv + cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1).unsqueeze(1) + sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1).unsqueeze(1) + rot = x[..., :rotary_dim].float() + half = rotary_dim // 2 + rotated = torch.cat([-rot[..., half:], rot[..., :half]], dim=-1) + out = (rot * cos + rotated * sin).type_as(x) + return torch.cat([out, x[..., rotary_dim:]], dim=-1) + + +def _hf_attention(x, attn, config, positions): + """HF Qwen4ExpTextAttention with a dense causal mask (QSA selects every block at this length).""" + num_q, num_kv, dim = attn.num_q, attn.num_kv, attn.head_dim + qkv = F.linear(x, attn.qkv_proj.weight) + qg, k, v = qkv.split(attn._qkv_split, dim=-1) + qg = qg.view(-1, num_q, dim * 2) + q, gate = qg[..., :dim], qg[..., dim:].reshape(-1, num_q * dim) + q = _hf_rope(_plus_one_rmsnorm(q, attn.q_norm.weight, config.rms_norm_eps), positions, + config.rotary_config.rotary_dim, config.rotary_config.base) + k = _hf_rope( + _plus_one_rmsnorm(k.view(-1, num_kv, dim), attn.k_norm.weight, config.rms_norm_eps), + positions, config.rotary_config.rotary_dim, config.rotary_config.base, + ) + v = v.view(-1, num_kv, dim) + rep = num_q // num_kv + scores = torch.einsum("qhd,khd->hqk", q.float(), k.repeat_interleave(rep, 1).float()) + scores = scores * dim**-0.5 + mask = torch.arange(x.shape[0], device=x.device) > positions.unsqueeze(-1) + out = torch.einsum( + "hqk,khd->qhd", scores.masked_fill(mask, float("-inf")).softmax(-1), + v.repeat_interleave(rep, 1).float(), + ).to(x.dtype) + return F.linear(out.reshape(-1, num_q * dim) * torch.sigmoid(gate), attn.o_proj.weight) + + +@requires_cuda +def test_qsa_layer_matches_hf_dense(): + """The QSA layer under the dense oracle backend equals HF attention, and freezes what the indexer hands the backend.""" + from freetoken.models.qwen4_exp.attention import Qwen4ExpAttention, TorchDenseQSAReference + from freetoken.utils.torch_utils import torch_dtype + + torch.manual_seed(6) + config = _config() + device, dtype = torch.device("cuda"), torch.bfloat16 + with torch.device(device), torch_dtype(dtype): + attn = Qwen4ExpAttention(config, layer_id=3) + _fill(attn, torch.Generator(device=device).manual_seed(7)) + + seq_len = 24 + x = (torch.randn(seq_len, config.hidden_size, device=device, dtype=dtype) * 0.5) + positions = torch.arange(seq_len, device=device, dtype=torch.int64) + req = SimpleNamespace(extend_len=seq_len, cached_len=0, table_idx=1) + batch = SimpleNamespace(padded_reqs=[req], reqs=[req], positions=positions) + + backend = TorchDenseQSAReference(config, num_slots=4, max_len=64, device=device, dtype=dtype) + _fresh_ctx(attn_backend=backend) + ref = _hf_attention(x, attn, config, positions) + got = attn.forward(x, batch) + assert torch.allclose(got.float(), ref.float(), rtol=2e-2, atol=2e-2) + + index = attn.indexer.forward(x) + args = config.qwen4_args + raw = F.linear(x, attn.indexer.index_qk_proj.weight) + assert index.q.shape == (seq_len, args.index_n_heads, args.index_head_dim) + assert index.k.shape == (seq_len, args.index_head_dim) + assert torch.equal(index.q.reshape(seq_len, -1), raw[:, : args.index_n_heads * args.index_head_dim]) + assert torch.equal(index.k, raw[:, args.index_n_heads * args.index_head_dim :]) + assert index.q_norm_weight.data_ptr() == attn.indexer.q_layernorm.weight.data_ptr() + + +class _StubLinearMixer(BaseOP): + """Stands in for the GDN layer; same [T, hidden] -> [T, hidden] shape.""" + + def __init__(self, config, layer_id): + self.out_proj = LinearReplicated(config.hidden_size, config.hidden_size, has_bias=False) + + def forward(self, x): + return self.out_proj.forward(x) + + +@requires_cuda +def test_shared_expert_gate_fusion_matches_eager(): + """Qwen4ExpMoE only swaps qwen3_5's gemv+sigmoid+mul+add gate chain for two triton kernels.""" + from freetoken.models.qwen3_5_moe.moe import Qwen3_5MoE + from freetoken.models.qwen4_exp.moe import Qwen4ExpMoE + from freetoken.moe.fused import FusedMoe + from freetoken.utils.torch_utils import torch_dtype + + config = _config() + device, dtype = torch.device("cuda"), torch.bfloat16 + with torch.device(device), torch_dtype(dtype): + moe = Qwen4ExpMoE(config, 0) + _fill(moe, torch.Generator(device=device).manual_seed(21), scale=0.2) + _fresh_ctx(moe_backend=FusedMoe()) + + x = torch.randn(6, config.hidden_size, device=device, dtype=dtype) * 0.5 + fused = moe.forward(x.clone()) + eager = Qwen3_5MoE.forward(moe, x.clone()) + + routed = moe.experts.forward(hidden_states=x.clone(), router_logits=moe.gate.forward(x)) + gate = torch.sigmoid(x.float() @ moe.shared_expert_gate.weight.float().view(-1)) + ref = routed.float() + gate.unsqueeze(1) * moe.shared_expert.forward(x).float() + + assert fused.shape == x.shape and fused.dtype == dtype + torch.testing.assert_close(fused, eager, rtol=2e-2, atol=2e-2) + # The fused gate stays in fp32 where the eager chain rounds the scalar to bf16. + assert (fused.float() - ref).abs().max() <= (eager.float() - ref).abs().max() + + +@requires_cuda +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("num_tokens,hidden", [(1, 2560), (7, 640)]) +def test_shared_gate_kernels_match_torch(num_tokens, hidden, dtype): + """Shipping hidden size for the two shared-gate kernels, against the torch chain they replace.""" + from freetoken.kernel.triton.moe_shared_gate import shared_gate_mul_add, shared_gate_sigmoid + + gen = torch.Generator(device="cuda").manual_seed(hidden + num_tokens) + kw = {"generator": gen, "device": "cuda", "dtype": dtype} + x = torch.randn(num_tokens, hidden, **kw) + weight = torch.randn(1, hidden, **kw) * 0.05 + shared = torch.randn(num_tokens, hidden, **kw) + routed = torch.randn(num_tokens, hidden, **kw) + + fused = shared_gate_mul_add(routed, shared, shared_gate_sigmoid(x, weight.view(-1))) + eager = routed + shared * torch.sigmoid(F.linear(x, weight)) + ref = routed.float() + shared.float() * torch.sigmoid(x.float() @ weight.float().view(-1))[:, None] + + assert fused.dtype == dtype and fused.shape == routed.shape + torch.testing.assert_close(fused, eager, rtol=2e-2, atol=2e-2) + assert (fused.float() - ref).abs().max() <= (eager.float() - ref).abs().max() + 1e-6 + + +@requires_cuda +def test_decoder_stack_prefill_and_decode(monkeypatch): + """Ragged bs=3 prefill then a bs=3 decode step through the whole model with dummy weights.""" + from freetoken.kvcache.linear_state_pool import LinearStatePool + from freetoken.models.qwen4_exp import model as model_module + from freetoken.models.qwen4_exp.attention import TorchDenseQSAReference + from freetoken.models.qwen4_exp.ple import GpuResidentTable + from freetoken.moe.fused import FusedMoe + from freetoken.utils.torch_utils import torch_dtype + + torch.manual_seed(8) + config = _config() + args = config.qwen4_args + device, dtype = torch.device("cuda"), torch.bfloat16 + monkeypatch.setattr(model_module, "build_linear_mixer", _StubLinearMixer) + + with torch.device(device), torch_dtype(dtype): + model = model_module.Qwen4ExpForCausalLM(config) + gen = torch.Generator(device=device).manual_seed(9) + _fill(model, gen) + multipliers, sizes, offsets = hash_constants(args) + table = torch.randn(4096, args.ngram_head_dim, generator=gen, device=device, dtype=dtype) * 0.05 + for ple in model.model.ple_layers: + ple.ple_embedding.layer_multipliers.copy_(multipliers) + ple.ple_embedding.ngram_heads_vocab_sizes.copy_(sizes) + ple.ple_embedding.ngram_heads_offsets.copy_(offsets) + ple.ple_embedding.attach_table(GpuResidentTable(table, dtype=dtype)) + + num_slots, max_len = 4, 64 + pool = LinearStatePool( + config.linear_attention_group(), num_slots, dtype, device, + slot_states=config.slot_states, + ) + prompts = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12], [9, 10, 11, 12, 13]] + ctx = _fresh_ctx( + attn_backend=TorchDenseQSAReference(config, num_slots, max_len, device, dtype), + moe_backend=FusedMoe(), + linear_state_pool=pool, + ) + reqs = [ + SimpleNamespace( + extend_len=len(p), cached_len=0, table_idx=i + 1, linear_slot_idx=None, + input_ids=torch.tensor(p, dtype=torch.int64), + ) + for i, p in enumerate(prompts) + ] + flat = [t for p in prompts for t in p] + last = torch.tensor( + [sum(len(p) for p in prompts[: i + 1]) - 1 for i in range(len(prompts))], device=device + ) + batch = SimpleNamespace( + padded_reqs=reqs, reqs=reqs, size=len(reqs), is_prefill=True, is_decode=False, + input_ids=torch.tensor(flat, dtype=torch.int64, device=device), + positions=torch.cat([torch.arange(len(p)) for p in prompts]).to(device), + attn_metadata=SimpleNamespace(get_last_indices=lambda bs: last[:bs]), + ) + with ctx.forward_batch(batch): + logits = model.forward() + assert logits.shape == (len(prompts), config.vocab_size) + assert torch.isfinite(logits.float()).all() + + for r, p in zip(reqs, prompts): + r.cached_len = len(p) + r.extend_len = 1 + r.input_ids = torch.cat([r.input_ids, torch.tensor([14], dtype=torch.int64)]) + decode = SimpleNamespace( + padded_reqs=reqs, reqs=reqs, size=len(reqs), is_prefill=False, is_decode=True, + input_ids=torch.tensor([14] * len(reqs), dtype=torch.int64, device=device), + positions=torch.tensor([len(p) for p in prompts], dtype=torch.int64, device=device), + attn_metadata=None, + ) + with ctx.forward_batch(decode): + decode_logits = model.forward() + assert decode_logits.shape == (len(prompts), config.vocab_size) + assert torch.isfinite(decode_logits.float()).all() diff --git a/tests/models/qwen4_exp/test_weight.py b/tests/models/qwen4_exp/test_weight.py new file mode 100644 index 0000000000..b3f1f851f7 --- /dev/null +++ b/tests/models/qwen4_exp/test_weight.py @@ -0,0 +1,410 @@ +"""qwen4_exp weight loading against a synthetic checkpoint shaped like the RadixArk NVFP4 one. + +The tensors are tiny but the key names, dtypes and the fusion geometry that matters +(hc_lowrank=320 + hc_count=4 -> a 12-row zero pad) are the real ones. +""" + +from __future__ import annotations + +import random +from types import SimpleNamespace + +import pytest +import torch +from safetensors.torch import save_file + +from freetoken.distributed import set_tp_info, try_get_tp_info +from freetoken.kernel.aot_models import SUPPORTED_MODELS, expert_bank_row_bytes +from freetoken.models.qwen4_exp.weight import ( + _ZERO_CENTERED_NORM_SUFFIXES, + iter_weights, + load_ple_table, +) +from freetoken.moe.host_banks import HostBank, read_range_into + +H = 32 # hidden_size +HC = 4 # hc_count +LR = 320 # hc_lowrank; kept real so the merged HC pad is the real (-(320+4)) % 16 = 12 +HCH = HC * H # hyper-connection stream width +KH, VH, HD = 2, 6, 8 # GDN key / value heads, head dim +QH, KVH, AHD = 4, 2, 16 # QSA q / kv heads, head dim +IHD = 8 # indexer head dim +E, I = 3, 6 # routed experts, moe_intermediate_size +NGRAM_DIM, NGRAM_ROWS, NGRAM_SHARDS = 4, 7, 4 + + +@pytest.fixture(scope="session", autouse=True) +def _tp_info(): + if try_get_tp_info() is None: + set_tp_info(rank=0, size=1) + + +def _bf16(*shape: int) -> torch.Tensor: + return torch.randn(*shape).to(torch.bfloat16) + + +def _hc_weights(prefix: str, inject: bool) -> dict[str, torch.Tensor]: + w = { + f"{prefix}.hc_norm.weight": _bf16(HCH), + f"{prefix}.input_mix_weight_down.weight": _bf16(LR, HCH), + f"{prefix}.input_mix_weight_up.weight": _bf16(HCH, LR), + } + if inject: + w[f"{prefix}.block_inject_weight.weight"] = _bf16(HC, HCH) + return w + + +def _raw_checkpoint() -> dict[str, torch.Tensor]: + """Layer 0 = GDN + PLE, layer 1 = QSA; plus the mtp / visual / routed-expert noise.""" + lm = "model.language_model" + raw: dict[str, torch.Tensor] = { + f"{lm}.embed_tokens.weight": _bf16(11, H), + "lm_head.weight": _bf16(11, H), + } + raw.update(_hc_weights(f"{lm}.hyper_connection_mixer", inject=False)) + for layer in (0, 1): + raw.update(_hc_weights(f"{lm}.layers.{layer}.attn_hyper_connection", inject=True)) + raw.update(_hc_weights(f"{lm}.layers.{layer}.mlp_hyper_connection", inject=True)) + raw.update({ + f"{lm}.layers.{layer}.mlp.gate.weight": _bf16(E, H), + f"{lm}.layers.{layer}.mlp.shared_expert.gate_proj.weight": _bf16(I, H), + f"{lm}.layers.{layer}.mlp.shared_expert.up_proj.weight": _bf16(I, H), + f"{lm}.layers.{layer}.mlp.shared_expert.down_proj.weight": _bf16(H, I), + f"{lm}.layers.{layer}.mlp.shared_expert_gate.weight": _bf16(1, H), + }) + for expert in range(E): + base = f"{lm}.layers.{layer}.mlp.experts.{expert}" + for proj, out, inn in (("gate_proj", I, H), ("up_proj", I, H), ("down_proj", H, I)): + raw[f"{base}.{proj}.weight"] = torch.randint( + 0, 256, (out, inn // 2), dtype=torch.uint8 + ) + raw[f"{base}.{proj}.weight_scale"] = torch.ones( + out, inn // 16 or 1, dtype=torch.float8_e4m3fn + ) + raw[f"{base}.{proj}.weight_scale_2"] = torch.tensor(0.5) + raw[f"{base}.{proj}.input_scale"] = torch.tensor(0.25) + gdn = f"{lm}.layers.0.linear_attn" + raw.update({ + f"{gdn}.in_proj_qkv.weight": _bf16(2 * KH * HD + VH * HD, H), + f"{gdn}.in_proj_z.weight": _bf16(VH * HD, H), + f"{gdn}.in_proj_b.weight": _bf16(VH, H), + f"{gdn}.in_proj_a.weight": _bf16(VH, H), + f"{gdn}.conv1d.weight": _bf16(2 * KH * HD + VH * HD, 1, 4), + f"{gdn}.A_log": _bf16(VH), + f"{gdn}.dt_bias": _bf16(VH), + f"{gdn}.norm.weight": _bf16(HD), + f"{gdn}.out_proj.weight": _bf16(H, VH * HD), + }) + ple = f"{lm}.layers.0.ple" + raw.update({ + f"{ple}.key_proj.weight": _bf16(HCH, H), + f"{ple}.value_proj.weight": _bf16(H, H), + f"{ple}.norm_key.weight": _bf16(HCH), + f"{ple}.norm_query.weight": _bf16(HCH), + f"{ple}.norm_conv.weight": _bf16(HCH), + f"{ple}.conv1d.weight": _bf16(HCH, 1, 4), + f"{ple}.ple_embedding.layer_multipliers": torch.randint(1, 1 << 40, (3,)), + f"{ple}.ple_embedding.ngram_heads_offsets": torch.arange(4), + f"{ple}.ple_embedding.ngram_heads_vocab_sizes": torch.full((4,), 5), + }) + attn = f"{lm}.layers.1.self_attn" + raw.update({ + f"{attn}.q_proj.weight": _bf16(2 * QH * AHD, H), + f"{attn}.k_proj.weight": _bf16(KVH * AHD, H), + f"{attn}.v_proj.weight": _bf16(KVH * AHD, H), + f"{attn}.o_proj.weight": _bf16(H, QH * AHD), + f"{attn}.q_norm.weight": _bf16(AHD), + f"{attn}.k_norm.weight": _bf16(AHD), + f"{attn}.indexer.index_qk_proj.weight": _bf16(5 * IHD, H), + f"{attn}.indexer.q_layernorm.weight": _bf16(IHD), + f"{attn}.indexer.k_layernorm.weight": _bf16(IHD), + }) + raw.update({ + "mtp.hyper_connection_mixer.hc_norm.weight": _bf16(HCH), + "mtp.layers.0.self_attn.q_proj.weight": _bf16(2 * QH * AHD, H), + "mtp.layers.0.mlp.experts.gate_up_proj": _bf16(E, 2 * I, H), + "mtp.layers.0.mlp.experts.down_proj": _bf16(E, H, I), + "model.visual.blocks.0.attn.qkv.weight": _bf16(3 * H, H), + "model.visual.merger.norm.weight": _bf16(H), + }) + return raw + + +def _ngram_table() -> tuple[dict[str, torch.Tensor], torch.Tensor]: + prefix = "model.language_model.layers.0.ple.ple_embedding.ngram_embedding" + shards = { + f"{prefix}.shard_{i}.weight": ( + torch.arange(i * NGRAM_ROWS * NGRAM_DIM, (i + 1) * NGRAM_ROWS * NGRAM_DIM) + .remainder(200).to(torch.uint8).view(NGRAM_ROWS, NGRAM_DIM).view(torch.float8_e4m3fn) + ) + for i in range(NGRAM_SHARDS) + } + scale = torch.tensor([0.125], dtype=torch.bfloat16) + shards[f"{prefix}.weight_scale"] = scale + return shards, scale + + +@pytest.fixture(scope="module") +def checkpoint(tmp_path_factory) -> tuple[str, dict[str, torch.Tensor]]: + torch.manual_seed(0) + folder = tmp_path_factory.mktemp("qwen4_exp_ckpt") + raw = _raw_checkpoint() + table, _scale = _ngram_table() + # Spread the dense tensors over two shards so the fusion buffer has to survive a file + # boundary, and put the n-gram table in its own shards like the real checkpoint does. + names = sorted(raw) + save_file({n: raw[n] for n in names[::2]}, str(folder / "model-bf16-00001.safetensors")) + save_file({n: raw[n] for n in names[1::2]}, str(folder / "model-bf16-00002.safetensors")) + shard_names = sorted(table) + save_file({n: table[n] for n in shard_names[:2]}, str(folder / "model-plefp8-00000.safetensors")) + save_file({n: table[n] for n in shard_names[2:]}, str(folder / "model-plefp8-00001.safetensors")) + return str(folder), {**raw, **table} + + +@pytest.fixture(scope="module") +def loaded(checkpoint) -> dict[str, torch.Tensor]: + folder, _raw = checkpoint + return { + name: tensor.clone() + for name, tensor in iter_weights( + folder, torch.device("cpu"), include_moe_experts=True, include_non_moe=True + ) + } + + +def _expected_names() -> set[str]: + names = {"model.embed_tokens.weight", "lm_head.weight"} + names |= {f"model.hyper_connection_mixer.{leaf}" for leaf in + ("hc_norm.weight", "input_mix_weight_down.weight", "input_mix_weight_up.weight")} + for layer in (0, 1): + for hc in ("attn_hyper_connection", "mlp_hyper_connection"): + names |= {f"model.layers.{layer}.{hc}.{leaf}" for leaf in ( + "hc_norm.weight", "input_mix_weight_down_block_inject.weight", + "input_mix_weight_up.weight")} + names |= {f"model.layers.{layer}.mlp.{leaf}" for leaf in ( + "gate.weight", "shared_expert.gate_up_proj.weight", + "shared_expert.down_proj.weight", "shared_expert_gate.weight")} + names |= {f"model.layers.0.linear_attn.{leaf}" for leaf in ( + "in_proj.weight", "conv1d.weight", "A_log", "dt_bias", "norm.weight", "out_proj.weight")} + names |= {f"model.layers.0.ple.{leaf}" for leaf in ( + "key_proj.weight", "value_proj.weight", "norm_key.weight", "norm_query.weight", + "norm_conv.weight", "conv1d.weight", "ple_embedding.layer_multipliers", + "ple_embedding.ngram_heads_offsets", "ple_embedding.ngram_heads_vocab_sizes")} + names |= {f"model.layers.1.self_attn.{leaf}" for leaf in ( + "qkv_proj.weight", "o_proj.weight", "q_norm.weight", "k_norm.weight", + "indexer.index_qk_proj.weight", "indexer.q_layernorm.weight", + "indexer.k_layernorm.weight")} + return names + + +def test_key_map_is_exactly_the_model_state_dict(loaded): + assert set(loaded) == _expected_names() + + +def test_mtp_visual_experts_and_table_never_loaded(loaded): + for name in loaded: + assert not name.startswith(("mtp.", "model.visual.")) + assert ".mlp.experts." not in name + assert "ngram_embedding" not in name + assert not name.endswith((".weight_scale", ".weight_scale_2", ".input_scale")) + + +def test_hc_merge_is_down_then_inject_then_zero_pad(loaded, checkpoint): + _folder, raw = checkpoint + key = "model.layers.0.attn_hyper_connection.input_mix_weight_down_block_inject.weight" + merged = loaded[key] + assert merged.shape == (LR + HC + 12, HCH) # pad = (-(320 + 4)) % 16 + down = raw["model.language_model.layers.0.attn_hyper_connection.input_mix_weight_down.weight"] + inject = raw["model.language_model.layers.0.attn_hyper_connection.block_inject_weight.weight"] + assert torch.equal(merged[:LR], down) + assert torch.equal(merged[LR:LR + HC], inject) + assert torch.equal(merged[LR + HC:], torch.zeros(12, HCH, dtype=merged.dtype)) + + +def test_top_level_mixer_keeps_the_unmerged_down(loaded, checkpoint): + _folder, raw = checkpoint + got = loaded["model.hyper_connection_mixer.input_mix_weight_down.weight"] + assert got.shape == (LR, HCH) + assert torch.equal( + got, raw["model.language_model.hyper_connection_mixer.input_mix_weight_down.weight"] + ) + assert torch.equal( + loaded["model.hyper_connection_mixer.input_mix_weight_up.weight"], + raw["model.language_model.hyper_connection_mixer.input_mix_weight_up.weight"], + ) + + +def test_qkv_fusion_slices_back_to_q_k_v(loaded, checkpoint): + _folder, raw = checkpoint + attn = "model.language_model.layers.1.self_attn" + parts = [raw[f"{attn}.{p}_proj.weight"] for p in ("q", "k", "v")] + fused = loaded["model.layers.1.self_attn.qkv_proj.weight"] + assert fused.shape == (2 * QH * AHD + 2 * KVH * AHD, H) # q carries the output gate + for part, back in zip(parts, torch.split(fused, [p.shape[0] for p in parts], dim=0)): + assert torch.equal(part, back) + + +def test_gdn_in_proj_slices_round_trip(loaded, checkpoint): + _folder, raw = checkpoint + gdn = "model.language_model.layers.0.linear_attn" + parts = [raw[f"{gdn}.in_proj_{p}.weight"] for p in ("qkv", "z", "b", "a")] + fused = loaded["model.layers.0.linear_attn.in_proj.weight"] + assert fused.shape == (sum(p.shape[0] for p in parts), H) + splits = torch.split(fused, [p.shape[0] for p in parts], dim=0) + for part, back in zip(parts, splits): + assert torch.equal(part, back) + + +def test_shared_expert_gate_up_merge(loaded, checkpoint): + _folder, raw = checkpoint + base = "model.language_model.layers.1.mlp.shared_expert" + merged = loaded["model.layers.1.mlp.shared_expert.gate_up_proj.weight"] + assert torch.equal(merged[:I], raw[f"{base}.gate_proj.weight"]) + assert torch.equal(merged[I:], raw[f"{base}.up_proj.weight"]) + + +ZERO_CENTERED = ( + "model.layers.0.attn_hyper_connection.hc_norm.weight", + "model.layers.0.mlp_hyper_connection.hc_norm.weight", + "model.hyper_connection_mixer.hc_norm.weight", + "model.layers.0.ple.norm_key.weight", + "model.layers.0.ple.norm_query.weight", + "model.layers.0.ple.norm_conv.weight", + "model.layers.1.self_attn.q_norm.weight", + "model.layers.1.self_attn.k_norm.weight", + "model.layers.1.self_attn.indexer.q_layernorm.weight", + "model.layers.1.self_attn.indexer.k_layernorm.weight", +) + + +def test_zero_centered_norms_are_loaded_raw(loaded, checkpoint): + """(1+w) is applied at runtime in fp32, so the loader must not fold it into the bf16 weight.""" + _folder, raw = checkpoint + for name in ZERO_CENTERED: + raw_name = name.replace("model.", "model.language_model.", 1) + assert torch.equal(loaded[name], raw[raw_name]), name + + +def test_the_zero_centered_suffix_list_covers_every_such_norm(): + assert {n for n in ZERO_CENTERED if n.endswith(_ZERO_CENTERED_NORM_SUFFIXES)} == set(ZERO_CENTERED) + assert not "model.layers.0.linear_attn.norm.weight".endswith(_ZERO_CENTERED_NORM_SUFFIXES) + + +def test_gdn_gated_norm_passes_through(loaded, checkpoint): + _folder, raw = checkpoint + assert torch.equal( + loaded["model.layers.0.linear_attn.norm.weight"], + raw["model.language_model.layers.0.linear_attn.norm.weight"], + ) + + +def test_hash_constants_stay_int64(loaded): + for leaf in ("layer_multipliers", "ngram_heads_offsets", "ngram_heads_vocab_sizes"): + assert loaded[f"model.layers.0.ple.ple_embedding.{leaf}"].dtype is torch.int64 + + +def test_load_ple_table_concatenates_shards_in_index_order(checkpoint): + folder, raw = checkpoint + args = SimpleNamespace(split_ngram_parts=NGRAM_SHARDS, ngram_head_dim=NGRAM_DIM) + table = load_ple_table(folder, args, pin=False) + assert table.tensor.shape == (NGRAM_SHARDS * NGRAM_ROWS, NGRAM_DIM) + assert table.tensor.dtype is torch.float8_e4m3fn + prefix = "model.language_model.layers.0.ple.ple_embedding.ngram_embedding" + for shard in range(NGRAM_SHARDS): + rows = table.tensor[shard * NGRAM_ROWS: (shard + 1) * NGRAM_ROWS] + assert torch.equal(rows.view(torch.uint8), + raw[f"{prefix}.shard_{shard}.weight"].view(torch.uint8)) + assert table.weight_scale.dtype is torch.bfloat16 + assert float(table.weight_scale) == 0.125 + + +def test_load_ple_table_rejects_a_shard_count_mismatch(checkpoint): + folder, _raw = checkpoint + args = SimpleNamespace(split_ngram_parts=NGRAM_SHARDS + 1, ngram_head_dim=NGRAM_DIM) + with pytest.raises(ValueError, match="shards 0"): + load_ple_table(folder, args, pin=False) + + +# ====================================================================================== +# read_range_into: the O_DIRECT byte-range read the PLE table load is built on +# ====================================================================================== + + +@pytest.fixture(scope="module") +def blob(tmp_path_factory) -> tuple[str, bytes]: + data = random.Random(7).randbytes(5_000_003) + path = tmp_path_factory.mktemp("blob") / "data.bin" + path.write_bytes(data) + return str(path), data + + +@pytest.mark.parametrize("file_offset, nbytes, dest_offset", [ + (1, 4095, 0), # sub-block, unaligned source + (2239, 1_000_000, 0), # the real checkpoint's header-end phase + (4095, 4097, 1), # straddles two block boundaries + (4_999_000, 1003, 123_456), # runs to EOF +]) +def test_read_range_into_matches_the_file(blob, file_offset, nbytes, dest_offset): + path, data = blob + bank = HostBank((6_000_000,), torch.uint8) + view = bank.memoryview() + got = read_range_into(view, path, file_offset=file_offset, nbytes=nbytes, + dest_offset=dest_offset, chunk=1 << 20) + assert got == nbytes + assert bytes(view[dest_offset:dest_offset + nbytes]) == data[file_offset:file_offset + nbytes] + + +def test_read_range_into_is_chunk_and_thread_safe(blob): + path, data = blob + bank = HostBank((6_000_000,), torch.uint8) + view = bank.memoryview() + read_range_into(view, path, file_offset=2239, nbytes=4_000_000, dest_offset=1024, + workers=8, chunk=64 << 10) + assert bytes(view[1024:1024 + 4_000_000]) == data[2239:2239 + 4_000_000] + + +def test_read_range_into_rejects_a_short_destination(blob): + path, _data = blob + bank = HostBank((1024,), torch.uint8) + with pytest.raises(ValueError, match="destination holds"): + read_range_into(bank.memoryview(), path, file_offset=0, nbytes=1 << 20) + + +# ====================================================================================== +# AOT shape table +# ====================================================================================== + + +def test_aot_entry_carries_the_checkpoint_geometry(): + entry = next(m for m in SUPPORTED_MODELS + if m.architecture == "Qwen4ExpForConditionalGeneration") + assert (entry.hidden_size, entry.moe_intermediate_size, entry.top_k) == (2560, 640, 10) + assert entry.kv_groups == ((2, 256),) + rows = expert_bank_row_bytes("nvfp4", entry.hidden_size, entry.moe_intermediate_size) + assert set(rows) == {"gate_up_packed", "gate_up_scale", "gate_up_global", + "down_packed", "down_scale", "down_global"} + for name, nbytes in rows.items(): + assert nbytes % 16 == 0, name # fused multi-bank copy only engages on 16B multiples + + +def test_every_registry_architecture_is_claimed_by_an_aot_entry(): + from freetoken.models.register import _MODEL_REGISTRY + + claimed = {m.architecture for m in SUPPORTED_MODELS} + claimed |= {a for m in SUPPORTED_MODELS for a in m.arch_aliases} + assert "Qwen4ExpForConditionalGeneration" in claimed + assert set(_MODEL_REGISTRY) - claimed == set() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs cuda") +def test_fusion_pad_rides_the_tensor_device(): + """safetensors loads straight to cuda; a cpu-allocated pad row would break torch.cat.""" + from freetoken.models.qwen4_exp.weight import _try_fuse + + buf = {} + down = torch.randn(320, 64, device="cuda", dtype=torch.bfloat16) + inject = torch.randn(4, 64, device="cuda", dtype=torch.bfloat16) + assert _try_fuse("model.layers.0.attn_hyper_connection.input_mix_weight_down.weight", down, buf) == () + key, fused = _try_fuse("model.layers.0.attn_hyper_connection.block_inject_weight.weight", inject, buf) + assert fused.device.type == "cuda" and fused.shape[0] == 336 + assert torch.equal(fused[324:], torch.zeros(12, 64, device="cuda", dtype=torch.bfloat16)) diff --git a/tests/models/qwen4_exp/test_weight_ckpt.py b/tests/models/qwen4_exp/test_weight_ckpt.py new file mode 100644 index 0000000000..747d22f2af --- /dev/null +++ b/tests/models/qwen4_exp/test_weight_ckpt.py @@ -0,0 +1,338 @@ +"""qwen4_exp weight loading against the real RadixArk/Qwen3.8-Flash-Next-NVFP4 checkpoint. + +Set ``FREETOKEN_QWEN4EXP_MODEL`` to the local checkpoint directory to run these. Everything is +sampled except the PLE table, which is loaded and pinned in full once (~47.7 GiB) because that +is the only way to check the shard concatenation and the pin budget. +""" + +from __future__ import annotations + +import dataclasses +import json +import os +import random +from types import SimpleNamespace + +import pytest +import safetensors +import torch + +from freetoken.distributed import set_tp_info, try_get_tp_info +from freetoken.kernel.aot_models import expert_bank_row_bytes +from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks +from freetoken.models.qwen4_exp.config import parse_config +from freetoken.models.qwen4_exp.weight import ( + _NVFP4_SOURCE_SPEC, + _ZERO_CENTERED_NORM_SUFFIXES, + iter_weights, + load_ple_table, +) +from freetoken.moe.host_banks import HostResidency +from freetoken.utils import cached_load_hf_config + +MODEL_PATH = os.environ.get("FREETOKEN_QWEN4EXP_MODEL") +pytestmark = [ + pytest.mark.needs_weights, + pytest.mark.skipif(not MODEL_PATH, reason="FREETOKEN_QWEN4EXP_MODEL is not set"), +] + +E, H, I = 512, 2560, 640 +NUM_LAYERS = 48 +PLE_LAYER = 1 +PLE_SHARDS, PLE_ROWS_PER_SHARD, PLE_DIM = 128, 2_500_012, 160 +PLE_BYTES = PLE_SHARDS * PLE_ROWS_PER_SHARD * PLE_DIM +EXPERT_LAYER_BYTES = E * sum(expert_bank_row_bytes("nvfp4", H, I).values()) +LM = "model.language_model" + + +@pytest.fixture(scope="session", autouse=True) +def _tp_info(): + if try_get_tp_info() is None: + set_tp_info(rank=0, size=1) + + +class _Reader: + """Serves checkpoint tensors by their raw key, through the index shard map.""" + + def __init__(self, folder: str): + with open(os.path.join(folder, "model.safetensors.index.json"), encoding="utf-8") as fh: + self._map = json.load(fh)["weight_map"] + self._folder = folder + self._handles: dict = {} + + def get(self, name: str) -> torch.Tensor: + shard = self._map[name] + handle = self._handles.get(shard) + if handle is None: + handle = safetensors.safe_open( + os.path.join(self._folder, shard), framework="pt", device="cpu" + ).__enter__() + self._handles[shard] = handle + return handle.get_tensor(name) + + def close(self) -> None: + for handle in self._handles.values(): + handle.__exit__(None, None, None) + self._handles.clear() + + +def _gdn_parts(layer: int) -> list[str]: + return [f"{LM}.layers.{layer}.linear_attn.in_proj_{p}.weight" for p in ("qkv", "z", "b", "a")] + + +def _hc_parts(layer: int, hc: str) -> list[str]: + return [f"{LM}.layers.{layer}.{hc}.input_mix_weight_down.weight", + f"{LM}.layers.{layer}.{hc}.block_inject_weight.weight"] + + +def _qkv_parts(layer: int) -> list[str]: + return [f"{LM}.layers.{layer}.self_attn.{p}_proj.weight" for p in ("q", "k", "v")] + + +# (model key, checkpoint keys, mode). "cat16" additionally requires the merged rows to be +# zero-padded up to a multiple of 16. Zero-centered norms are "same": (1+w) is a runtime op. +SAMPLES: tuple[tuple[str, list[str], str], ...] = ( + ("model.embed_tokens.weight", [f"{LM}.embed_tokens.weight"], "same"), + ("lm_head.weight", ["lm_head.weight"], "same"), + ("model.layers.0.linear_attn.in_proj.weight", _gdn_parts(0), "cat"), + ("model.layers.46.linear_attn.in_proj.weight", _gdn_parts(46), "cat"), + ("model.layers.0.linear_attn.conv1d.weight", [f"{LM}.layers.0.linear_attn.conv1d.weight"], "same"), + ("model.layers.0.linear_attn.A_log", [f"{LM}.layers.0.linear_attn.A_log"], "same"), + ("model.layers.0.linear_attn.dt_bias", [f"{LM}.layers.0.linear_attn.dt_bias"], "same"), + ("model.layers.0.linear_attn.norm.weight", [f"{LM}.layers.0.linear_attn.norm.weight"], "same"), + ("model.layers.0.linear_attn.out_proj.weight", [f"{LM}.layers.0.linear_attn.out_proj.weight"], "same"), + ("model.layers.3.self_attn.qkv_proj.weight", _qkv_parts(3), "cat"), + ("model.layers.47.self_attn.qkv_proj.weight", _qkv_parts(47), "cat"), + ("model.layers.3.self_attn.o_proj.weight", [f"{LM}.layers.3.self_attn.o_proj.weight"], "same"), + ("model.layers.3.self_attn.q_norm.weight", [f"{LM}.layers.3.self_attn.q_norm.weight"], "same"), + ("model.layers.3.self_attn.k_norm.weight", [f"{LM}.layers.3.self_attn.k_norm.weight"], "same"), + ("model.layers.3.self_attn.indexer.index_qk_proj.weight", + [f"{LM}.layers.3.self_attn.indexer.index_qk_proj.weight"], "same"), + ("model.layers.3.self_attn.indexer.q_layernorm.weight", + [f"{LM}.layers.3.self_attn.indexer.q_layernorm.weight"], "same"), + ("model.layers.47.self_attn.indexer.k_layernorm.weight", + [f"{LM}.layers.47.self_attn.indexer.k_layernorm.weight"], "same"), + ("model.layers.7.attn_hyper_connection.hc_norm.weight", + [f"{LM}.layers.7.attn_hyper_connection.hc_norm.weight"], "same"), + ("model.layers.7.attn_hyper_connection.input_mix_weight_down_block_inject.weight", + _hc_parts(7, "attn_hyper_connection"), "cat16"), + ("model.layers.7.attn_hyper_connection.input_mix_weight_up.weight", + [f"{LM}.layers.7.attn_hyper_connection.input_mix_weight_up.weight"], "same"), + ("model.layers.47.mlp_hyper_connection.input_mix_weight_down_block_inject.weight", + _hc_parts(47, "mlp_hyper_connection"), "cat16"), + ("model.hyper_connection_mixer.hc_norm.weight", + [f"{LM}.hyper_connection_mixer.hc_norm.weight"], "same"), + ("model.hyper_connection_mixer.input_mix_weight_down.weight", + [f"{LM}.hyper_connection_mixer.input_mix_weight_down.weight"], "same"), + ("model.hyper_connection_mixer.input_mix_weight_up.weight", + [f"{LM}.hyper_connection_mixer.input_mix_weight_up.weight"], "same"), + (f"model.layers.{PLE_LAYER}.ple.key_proj.weight", [f"{LM}.layers.{PLE_LAYER}.ple.key_proj.weight"], "same"), + (f"model.layers.{PLE_LAYER}.ple.value_proj.weight", [f"{LM}.layers.{PLE_LAYER}.ple.value_proj.weight"], "same"), + (f"model.layers.{PLE_LAYER}.ple.norm_key.weight", [f"{LM}.layers.{PLE_LAYER}.ple.norm_key.weight"], "same"), + (f"model.layers.{PLE_LAYER}.ple.norm_query.weight", [f"{LM}.layers.{PLE_LAYER}.ple.norm_query.weight"], "same"), + (f"model.layers.{PLE_LAYER}.ple.norm_conv.weight", [f"{LM}.layers.{PLE_LAYER}.ple.norm_conv.weight"], "same"), + (f"model.layers.{PLE_LAYER}.ple.conv1d.weight", [f"{LM}.layers.{PLE_LAYER}.ple.conv1d.weight"], "same"), + (f"model.layers.{PLE_LAYER}.ple.ple_embedding.layer_multipliers", + [f"{LM}.layers.{PLE_LAYER}.ple.ple_embedding.layer_multipliers"], "same"), + (f"model.layers.{PLE_LAYER}.ple.ple_embedding.ngram_heads_offsets", + [f"{LM}.layers.{PLE_LAYER}.ple.ple_embedding.ngram_heads_offsets"], "same"), + (f"model.layers.{PLE_LAYER}.ple.ple_embedding.ngram_heads_vocab_sizes", + [f"{LM}.layers.{PLE_LAYER}.ple.ple_embedding.ngram_heads_vocab_sizes"], "same"), + ("model.layers.5.mlp.gate.weight", [f"{LM}.layers.5.mlp.gate.weight"], "same"), + ("model.layers.5.mlp.shared_expert.gate_up_proj.weight", + [f"{LM}.layers.5.mlp.shared_expert.gate_proj.weight", + f"{LM}.layers.5.mlp.shared_expert.up_proj.weight"], "cat"), + ("model.layers.5.mlp.shared_expert.down_proj.weight", + [f"{LM}.layers.5.mlp.shared_expert.down_proj.weight"], "same"), + ("model.layers.5.mlp.shared_expert_gate.weight", + [f"{LM}.layers.5.mlp.shared_expert_gate.weight"], "same"), +) + + +@pytest.fixture(scope="module") +def reader() -> _Reader: + r = _Reader(MODEL_PATH) + yield r + r.close() + + +@pytest.fixture(scope="module") +def dense_pass() -> tuple[list[str], dict[str, torch.Tensor]]: + """One full iter_weights sweep: every emitted name, plus a clone of each sampled tensor. + + The zero-centered norms are kept too -- they are tiny, and checking all of them is the + cheapest guard against the +1 creeping back into the load path.""" + wanted = {name for name, _raw, _mode in SAMPLES} + names: list[str] = [] + sampled: dict[str, torch.Tensor] = {} + for name, tensor in iter_weights( + MODEL_PATH, torch.device("cpu"), include_moe_experts=True, include_non_moe=True + ): + names.append(name) + if name in wanted or name.endswith(_ZERO_CENTERED_NORM_SUFFIXES): + sampled[name] = tensor.clone() + return names, sampled + + +def test_emitted_names_are_unique_and_complete(dense_pass): + names, _sampled = dense_pass + assert len(names) == len(set(names)) + assert len([n for n in names if n.endswith(".linear_attn.in_proj.weight")]) == 36 + assert len([n for n in names if n.endswith(".self_attn.qkv_proj.weight")]) == 12 + assert len([n for n in names + if n.endswith(".input_mix_weight_down_block_inject.weight")]) == 2 * NUM_LAYERS + assert len([n for n in names if ".ple." in n]) == 9 + assert {"model.embed_tokens.weight", "lm_head.weight", + "model.hyper_connection_mixer.input_mix_weight_down.weight"} <= set(names) + + +@pytest.fixture(scope="module") +def model_state_dict_keys() -> set[str]: + """Keys ``Qwen4ExpForCausalLM`` declares -- the authoritative target the loader must fill.""" + from freetoken.layers import rotary + from freetoken.models.qwen4_exp.model import Qwen4ExpForCausalLM + + config = parse_config(cached_load_hf_config(MODEL_PATH)) + saved = rotary._ROPE_DEVICE + rotary.set_rope_device(torch.device("cpu")) # get_rope refuses to build on meta + rotary.get_rope.cache_clear() + try: + with torch.device("meta"): + return set(Qwen4ExpForCausalLM(config).state_dict()) + finally: + rotary.set_rope_device(saved) + rotary.get_rope.cache_clear() + + +def test_emitted_names_are_the_model_state_dict(dense_pass, model_state_dict_keys): + names, _sampled = dense_pass + # The routed NVFP4 experts come from the offload source banks, never from the dense pass. + expected = {k for k in model_state_dict_keys + if not k.endswith((".mlp.experts.gate_up_proj", ".mlp.experts.down_proj"))} + assert set(names) == expected + + +def test_every_zero_centered_norm_is_present_and_raw(dense_pass, reader): + names, sampled = dense_pass + zero_centered = [n for n in names if n.endswith(_ZERO_CENTERED_NORM_SUFFIXES)] + # 2 HC per layer + the top-level mixer + 3 PLE norms + q/k_norm and indexer q/k per QSA layer + assert len(zero_centered) == 2 * NUM_LAYERS + 1 + 3 + 4 * 12 + for name in zero_centered: + assert torch.equal(sampled[name], reader.get(name.replace("model.", f"{LM}.", 1))), name + + +def test_no_mtp_visual_expert_or_table_tensor_is_loaded(dense_pass): + names, _sampled = dense_pass + for name in names: + assert not name.startswith(("mtp.", "model.visual.", "visual.")) + assert ".mlp.experts." not in name + assert "ngram_embedding" not in name + assert not name.endswith((".weight_scale", ".weight_scale_2", ".input_scale")) + + +@pytest.mark.parametrize("name, raw_names, mode", SAMPLES, ids=[s[0] for s in SAMPLES]) +def test_sampled_tensor_matches_the_checkpoint(dense_pass, reader, name, raw_names, mode): + _names, sampled = dense_pass + parts = [reader.get(raw) for raw in raw_names] + got = sampled[name] + if mode == "same": + assert torch.equal(got, parts[0]) + assert got.dtype is parts[0].dtype + else: + rows = sum(p.shape[0] for p in parts) + assert torch.equal(got[:rows], torch.cat(parts, dim=0)) + if mode == "cat16": + assert got.shape[0] == rows + (-rows) % 16 + assert not got[rows:].any() + else: + assert got.shape[0] == rows + + +@pytest.mark.slow +@pytest.mark.skipif(not torch.cuda.is_available(), reason="pinning needs CUDA") +def test_ple_table_loads_pinned_and_matches_the_checkpoint(reader): + args = parse_config(cached_load_hf_config(MODEL_PATH)).qwen4_args + table = load_ple_table(MODEL_PATH, args) + assert table.bank.residency is HostResidency.PINNED + assert table.bank.nbytes == PLE_BYTES + assert abs(PLE_BYTES / 2**30 - 47.68) < 0.05 + assert table.tensor.shape == (PLE_SHARDS * PLE_ROWS_PER_SHARD, PLE_DIM) + assert table.tensor.dtype is torch.float8_e4m3fn + + prefix = f"{LM}.layers.{PLE_LAYER}.ple.ple_embedding.ngram_embedding" + assert torch.equal(table.weight_scale.reshape(1), + reader.get(f"{prefix}.weight_scale").reshape(1)) + rows = random.Random(0).sample(range(PLE_SHARDS * PLE_ROWS_PER_SHARD), 1000) + by_shard: dict[int, list[int]] = {} + for row in rows: + by_shard.setdefault(row // PLE_ROWS_PER_SHARD, []).append(row) + got = table.tensor.view(torch.uint8) + for shard, shard_rows in by_shard.items(): + ref = reader.get(f"{prefix}.shard_{shard}.weight").view(torch.uint8) + local = torch.tensor([r - shard * PLE_ROWS_PER_SHARD for r in shard_rows]) + assert torch.equal(got[torch.tensor(shard_rows)], ref[local]) + + # The two pinned host allocations the engine must budget for. + total = PLE_BYTES + NUM_LAYERS * EXPERT_LAYER_BYTES + assert abs(total / 2**30 - 111.14) < 0.05 + + +@pytest.fixture(scope="module") +def layer0_expert_banks(): + """The real NVFP4 source-bank loader, restricted to layer 0 (1.32 GiB instead of 63.5).""" + if not torch.cuda.is_available(): + pytest.skip("expert bank pinning needs CUDA") + spec = dataclasses.replace( + _NVFP4_SOURCE_SPEC, layer_to_bank=lambda layer, config: 0 if layer == 0 else None + ) + config = SimpleNamespace(num_experts=E, hidden_size=H, moe_intermediate_size=I, + num_moe_layers=1) + return load_nvfp4_expert_source_banks( + MODEL_PATH, config, spec, drop_page_cache=lambda path: None, primary=False + ) + + +@pytest.mark.slow +def test_sampled_experts_match_the_checkpoint(layer0_expert_banks, reader): + banks = layer0_expert_banks + for expert in random.Random(1).sample(range(E), 8): + base = f"{LM}.layers.0.mlp.experts.{expert}" + assert torch.equal(banks["gate_up_packed"][0][expert, :I], + reader.get(f"{base}.gate_proj.weight")) + assert torch.equal(banks["gate_up_packed"][0][expert, I:], + reader.get(f"{base}.up_proj.weight")) + assert torch.equal(banks["down_packed"][0][expert], + reader.get(f"{base}.down_proj.weight")) + for proj, bank, rows in (("gate_proj", "gate_up_scale", slice(0, I)), + ("up_proj", "gate_up_scale", slice(I, 2 * I)), + ("down_proj", "down_scale", slice(None))): + scale = reader.get(f"{base}.{proj}.weight_scale") + assert torch.equal(banks[bank][0][expert][rows].reshape(-1).view(torch.uint8), + scale.reshape(-1).view(torch.uint8)) + gate_g = reader.get(f"{base}.gate_proj.weight_scale_2").to(torch.float16) + assert torch.equal(banks["gate_up_global"][0][expert, :I], gate_g.reshape(1).expand(I)) + + +def test_expert_bank_bytes_match_the_aot_row_table(layer0_expert_banks): + measured = sum(t[0].numel() * t[0].element_size() for t in layer0_expert_banks.values()) + assert measured == EXPERT_LAYER_BYTES + assert abs(NUM_LAYERS * EXPERT_LAYER_BYTES / 2**30 - 63.46) < 0.05 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="dummy banks are pinned") +def test_dummy_expert_sources_have_the_real_bank_shapes(layer0_expert_banks): + from freetoken.models.weight import _model_override, dummy_nvfp4_expert_sources + from freetoken.models.register import get_model_spec + + spec = get_model_spec("Qwen4ExpForConditionalGeneration") + # No dummy_* override, so --use-dummy-weight goes through the generic builders. + for hook in ("dummy_nvfp4_expert_sources", "dummy_moe_expert_sources", "dummy_q4_0_expert_sources"): + assert _model_override(spec, hook) is None + + config = SimpleNamespace(num_experts=E, hidden_size=H, moe_intermediate_size=I, + num_moe_layers=1) + dummy = dummy_nvfp4_expert_sources(config) + assert set(dummy) == set(layer0_expert_banks) + for name, banks in dummy.items(): + assert banks[0].shape == layer0_expert_banks[name][0].shape + assert banks[0].dtype is layer0_expert_banks[name][0].dtype diff --git a/tests/models/test_gemma4_mmproj_mapping.py b/tests/models/test_gemma4_mmproj_mapping.py new file mode 100644 index 0000000000..fe300bd0ef --- /dev/null +++ b/tests/models/test_gemma4_mmproj_mapping.py @@ -0,0 +1,31 @@ +"""Pure mapping regression coverage for the separate Gemma4 projector GGUF.""" + +import pytest + +from freetoken.models.gemma4.gguf import _gemma4_image_token_id, gemma4_mmproj_param_name + + +def test_gemma4_mmproj_tensor_mapping() -> None: + """Every projector tensor family maps into its corresponding vision module.""" + assert gemma4_mmproj_param_name("mm.input_projection.weight") == "embed_vision.embedding_projection.weight" + assert gemma4_mmproj_param_name("v.patch_embd.weight") == "vision_tower.patch_embedder.input_proj.weight" + assert gemma4_mmproj_param_name("v.blk.7.attn_q.weight") == "vision_tower.encoder.layers.7.self_attn.q_proj.weight" + assert gemma4_mmproj_param_name("v.blk.7.ffn_down.weight") == "vision_tower.encoder.layers.7.mlp.down_proj.weight" + assert gemma4_mmproj_param_name("v.blk.7.ln1.weight") == "vision_tower.encoder.layers.7.input_layernorm.weight" + + +def test_gemma4_mmproj_rejects_unknown_names() -> None: + """Unexpected projector data cannot silently bind to an unrelated parameter.""" + assert gemma4_mmproj_param_name("v.blk.bad.attn_q.weight") is None + assert gemma4_mmproj_param_name("unrelated.weight") is None + + +def test_gemma4_gguf_image_placeholder_uses_checkpoint_token() -> None: + """Gemma's soft image placeholder is not inferred from a fixed id.""" + assert _gemma4_image_token_id({"tokenizer.ggml.tokens": ["x", "<|image|>"]}) == 1 + + +def test_gemma4_gguf_image_placeholder_rejects_ambiguous_tokenizers() -> None: + """A conversion that carries multiple candidate placeholders must fail closed.""" + with pytest.raises(ValueError, match="exactly one image placeholder"): + _gemma4_image_token_id({"tokenizer.ggml.tokens": ["<|image|>", "<|image|>"]}) diff --git a/tests/models/test_gguf_q4_k.py b/tests/models/test_gguf_q4_k.py new file mode 100644 index 0000000000..9a69c4df2d --- /dev/null +++ b/tests/models/test_gguf_q4_k.py @@ -0,0 +1,40 @@ +"""Reference checks for the Q4_K GGUF format used by Q4_K_M model releases.""" + +import torch + +from freetoken.models.gguf.dequant import GGML_Q4_K, dequant_q4_k, row_bytes + + +def _half_bytes(value: float) -> torch.Tensor: + """Return the two little-endian bytes of one IEEE fp16 scalar.""" + return torch.tensor([value], dtype=torch.float16).view(torch.uint8) + + +def test_q4_k_row_size_matches_the_ggml_block_layout(): + """A 256-value Q4_K super-block occupies 144 bytes in the GGUF tensor table.""" + assert row_bytes(256, GGML_Q4_K) == 144 + assert row_bytes(2048, GGML_Q4_K) == 8 * 144 + + +def test_q4_k_reference_decoder_handles_scale_minimum_and_nibble_order(): + """Known packed bytes decode by the same affine rule used in llama.cpp.""" + raw = torch.zeros((1, 144), dtype=torch.uint8) + raw[0, 0:2] = _half_bytes(2.0) + raw[0, 2:4] = _half_bytes(0.5) + # The first packed 32-byte pair encodes group 0 in its low nibbles and + # group 1 in its high nibbles. Their scale/minimum fields are separate. + raw[0, 4] = 3 + raw[0, 8] = 4 + raw[0, 5] = 3 + raw[0, 9] = 4 + raw[0, 16:32] = 0xF1 # low nibble 1, high nibble 15 for the first 32-value group. + + decoded = dequant_q4_k(raw, torch.float32) + # group 0: 2 * 3 * q - 0.5 * 4. The first 32 values use low nibbles. + assert decoded[0].item() == 4.0 + assert decoded[31].item() == 4.0 + # Group 1 uses the high nibbles from the same packed byte range. + assert decoded[32].item() == 88.0 + assert decoded[63].item() == 88.0 + # The remaining groups have zero scale/minimum and therefore decode to zero. + assert torch.count_nonzero(decoded[64:]) == 0 diff --git a/tests/models/test_gguf_tokenizer_specials.py b/tests/models/test_gguf_tokenizer_specials.py new file mode 100644 index 0000000000..549c8c433a --- /dev/null +++ b/tests/models/test_gguf_tokenizer_specials.py @@ -0,0 +1,45 @@ +"""Regression coverage for GGUF tokenizer control-token registration.""" + +from freetoken.models.gguf.tokenizer import _register_embedded_special_tokens + + +class _FakeTokenizer: + """Minimal tokenizer recorder that keeps the helper test independent of model files.""" + + bos_token = "" + eos_token = "" + unk_token = "" + pad_token = "" + + def __init__(self) -> None: + self.calls: list[dict[str, list[str]]] = [] + + def add_special_tokens(self, values: dict[str, list[str]]) -> int: + """Record the exact registration request made by the GGUF helper.""" + self.calls.append(values) + return len(values["additional_special_tokens"]) + + +def test_register_embedded_control_and_user_defined_tokens() -> None: + """Qwen's thinking marker stays atomic after a GGUF tokenizer conversion.""" + tokenizer = _FakeTokenizer() + + _register_embedded_special_tokens( + tokenizer, + ["ordinary", "", "<|im_start|>", "", ""], + [1, 3, 3, 4, 3], + ) + + assert tokenizer.calls == [ + {"additional_special_tokens": ["<|im_start|>", ""]} + ] + + +def test_register_embedded_special_tokens_ignores_invalid_metadata() -> None: + """Malformed optional type metadata cannot block otherwise valid GGUF loading.""" + tokenizer = _FakeTokenizer() + + _register_embedded_special_tokens(tokenizer, [""], None) + _register_embedded_special_tokens(tokenizer, [""], [4, 4]) + + assert tokenizer.calls == [] diff --git a/tests/models/test_glm5_next_config.py b/tests/models/test_glm5_next_config.py new file mode 100644 index 0000000000..8f0bc518e5 --- /dev/null +++ b/tests/models/test_glm5_next_config.py @@ -0,0 +1,330 @@ +"""glm5_next (GLM-5.3-Flash) config parsing: attention groups, kpool spec, aliases. + +Runs off a trimmed copy of the real checkpoint config.json via RawConfigShim (the +exact object cached_load_hf_config falls back to when transformers doesn't know +``glm5_next`` yet), so the parse path under test is the one production hits. +""" + +from __future__ import annotations + +import pytest + +from freetoken.attention.base import AttnType +from freetoken.models.config import ( + FullAttentionGroupConfig, + LinearGatedDeltaGroupConfig, +) +from freetoken.models.glm5_next.args import load_args +from freetoken.models.glm5_next.config import parse_config +from freetoken.utils.hf import RawConfigShim + +_NUM_LAYERS = 45 +_DSA_IDS = tuple(range(3, _NUM_LAYERS, 4)) # 3, 7, ..., 43 +_KDA_IDS = tuple(i for i in range(_NUM_LAYERS) if i not in _DSA_IDS) + + +def _layer_types() -> list[str]: + return [ + "deepseek_sparse_attention" if i in _DSA_IDS else "linear_attention" + for i in range(_NUM_LAYERS) + ] + + +def _text_config() -> dict: + # Trimmed from zai-org/GLM-5.3-Flash config.json (text_config). + return { + "hidden_size": 4096, + "intermediate_size": 12288, + "num_hidden_layers": _NUM_LAYERS, + "num_attention_heads": 64, + "num_key_value_heads": 64, + "vocab_size": 154880, + "hidden_act": "silu", + "rms_norm_eps": 1e-5, + "max_position_embeddings": 1048576, + "tie_word_embeddings": False, + # MLA (NoPE) + "q_lora_rank": 1536, + "kv_lora_rank": 512, + "qk_nope_head_dim": 256, + "qk_rope_head_dim": 0, + "qk_head_dim": 256, + "v_head_dim": 256, + "mla_use_nope": True, + # DSA indexer + kpool + "index_n_heads": 32, + "index_head_dim": 128, + "index_topk": 2048, + "indexer_types": ["full"] * _NUM_LAYERS, + "indexer_rope_interleave": True, + "index_kpool": 4, + "index_kpool_compress": True, + "index_kpool_always_select_tail": True, + # KDA + "linear_attn_config": { + "num_heads": 64, + "head_dim": 128, + "short_conv_kernel_size": 4, + "gate_lower_bound": -5.0, + "kda_layers": list(_KDA_IDS), + "full_attn_layers": list(_DSA_IDS), + }, + # layout + "layer_types": _layer_types(), + "mlp_layer_types": ["dense"] * 3 + ["sparse"] * (_NUM_LAYERS - 3), + "first_k_dense_replace": 3, + # mHC (checkpoint spellings) + "mhc": True, + "hc_mult": 4, + "hc_eps": 1e-6, + "hc_sinkhorn_iters": 20, + # MoE + "n_routed_experts": 288, + "num_experts_per_tok": 8, + "n_shared_experts": 1, + "moe_intermediate_size": 2048, + "norm_topk_prob": True, + "routed_scaling_factor": 2.5, + "scoring_func": "sigmoid", + "topk_method": "noaux_tc", + "n_group": 1, + "topk_group": 1, + "swiglu_limit": 10.0, + "num_nextn_predict_layers": 1, + "attention_bias": False, + "model_type": "glm5_next_text", + } + + +def _hf_config(quantization_config: dict | None = None) -> RawConfigShim: + data: dict = { + "architectures": ["Glm5NextForConditionalGeneration"], + "model_type": "glm5_next", + "text_config": _text_config(), + "vision_config": {"model_type": "glm5_next_vision", "depth": 24}, + "image_token_id": 154854, + } + if quantization_config is not None: + data["quantization_config"] = quantization_config + return RawConfigShim(data) + + +_CT_NVFP4_QUANT = { + # From RedHatAI/GLM-5.3-Flash-NVFP4 (llm-compressor, experts-only calibrated NVFP4). + "quant_method": "compressed-tensors", + "format": "nvfp4-pack-quantized", + "config_groups": { + "group_0": { + "targets": ["re:.*mlp\\.experts\\..*(gate|up|down)_proj$"], + "weights": {"num_bits": 4, "type": "float", "group_size": 16}, + } + }, +} + +_CT_MIXED_QUANT = { + # From RedHatAI/GLM-5.3-Flash-NVFP4 as published: nvfp4 routed experts plus fp8 experts on the MTP layer, so the top-level format is "mixed-precision". + "quant_method": "compressed-tensors", + "format": "mixed-precision", + "config_groups": { + "group_0": { + "targets": ["re:.*\\.layers\\.(?:[3-9]|[1-3][0-9]|4[0-4])\\.mlp\\.experts\\..*(gate|up|down)_proj$"], + "weights": {"num_bits": 4, "type": "float", "group_size": 16, "strategy": "tensor_group"}, + "format": "nvfp4-pack-quantized", + }, + "group_1": { + "targets": ["re:.*\\.layers\\.45\\.mlp\\.experts\\.\\d+\\.(gate_proj|up_proj|down_proj)$"], + "weights": {"num_bits": 8, "type": "float", "strategy": "block"}, + "format": "float-quantized", + }, + }, +} + +_NVFP4_QUANT = { + # From LibertAIDAI/GLM-5.3-Flash-NVFP4 (ModelOpt weight-only NVFP4). + "quant_algo": "NVFP4", + "quant_method": "modelopt", + "config_groups": { + "group_0": { + "targets": ["Linear"], + "weights": {"num_bits": 4, "type": "float", "group_size": 16}, + } + }, + "ignore": ["lm_head", "model.visual.*"], +} + + +def test_attention_groups(): + cfg = parse_config(_hf_config()) + + assert cfg.num_layers == _NUM_LAYERS + assert len(cfg.attention_groups) == 2 + linear, full = cfg.attention_groups # ordered by first layer id (0 < 3) + + assert isinstance(linear, LinearGatedDeltaGroupConfig) + assert linear.variant == "kda" + assert linear.layer_ids == _KDA_IDS + assert (linear.num_key_heads, linear.key_head_dim) == (64, 128) + assert (linear.num_value_heads, linear.value_head_dim) == (64, 128) + assert linear.conv_kernel_dim == 4 + + assert isinstance(full, FullAttentionGroupConfig) + assert full.layer_ids == _DSA_IDS + assert full.mla is True + assert full.head_dim == 512 # bare ckv latent: kv_lora_rank + 0 rope dims + assert full.num_kv_heads == 1 + assert full.index_head_dim == 128 + assert full.num_index_layers == len(_DSA_IDS) + assert full.index_ratio == 4 + + assert cfg.has_linear_attention and cfg.has_hybrid_attention + assert cfg.attn_type_for_layer(0) == AttnType.LINEAR + assert cfg.attn_type_for_layer(3) == AttnType.DSA + + +def test_kv_cache_group_specs_skip_linear_and_carry_kpool(): + cfg = parse_config(_hf_config()) + specs = [s for s in cfg.kv_cache_group_specs() if s.num_layers > 0] + # The linear group keeps recurrent state (no paged KV); exactly one paged spec. + assert len(specs) == 1 + (spec,) = specs + assert spec.attn_type == AttnType.DSA + assert spec.layer_ids == _DSA_IDS + assert (spec.mla, spec.head_dim, spec.index_head_dim) == (True, 512, 128) + assert spec.index_ratio == 4 + assert spec.num_index_layers == len(_DSA_IDS) + + +def test_moe_and_scalars(): + cfg = parse_config(_hf_config(_NVFP4_QUANT)) + assert cfg.expert_quant == "nvfp4" + # Only DERIVED facts are pinned here; fixture echoes (num_experts == 288 + # and friends) assert nothing the parse could get wrong. + assert cfg.attn_sm_scale == pytest.approx(256**-0.5) + assert cfg.num_moe_layers == _NUM_LAYERS - 3 + assert cfg.is_moe + # Checkpoint-faithful default; the FREETOKEN_GLM5_*_FP8 env flags opt into + # the W8A16 fp8 load. + assert (cfg.attn_quant, cfg.dense_quant, cfg.lm_head_quant) == ("none",) * 3 + # Text-only serving: the vision tower is never built. + assert cfg.vision_config is None + + +def test_args_alias_folding_and_nope(): + args = load_args(_hf_config()) + # Checkpoint spellings fold into the canonical fields. + assert args.mla_nope is True # from mla_use_nope + assert args.mhc_num_residual_streams == 4 # from hc_mult + assert args.mhc_sinkhorn_iterations == 20 # from hc_sinkhorn_iters + assert args.linear_num_heads == 64 # from nested linear_attn_config + assert args.linear_lower_bound == -5.0 + # NoPE geometry. + assert args.qk_rope_head_dim == 0 + assert args.qk_head_dim == 256 + assert args.latent_dim == 512 + assert args.kda_layer_ids == _KDA_IDS + assert args.dsa_layer_ids == _DSA_IDS + # Defaults for fields the checkpoint doesn't ship. + assert args.rope_theta == 10000.0 + assert args.mhc_tau == 0.05 + assert args.mhc_post_mult_value == 2.0 + + +def test_registry_resolves_glm5_next(): + from freetoken.models.register import get_model_spec + + spec = get_model_spec("Glm5NextForConditionalGeneration") + assert spec.module == "freetoken.models.glm5_next" + assert spec.model_cls == "Glm5NextForCausalLM" + assert get_model_spec("Glm5NextForCausalLM").module == spec.module + + +def test_dev_layer_cap(monkeypatch): + monkeypatch.setenv("FREETOKEN_GLM5_MAX_LAYERS", "5") + cfg = parse_config(_hf_config()) + assert cfg.num_layers == 5 + linear, full = cfg.attention_groups + assert linear.layer_ids == (0, 1, 2, 4) + assert full.layer_ids == (3,) + assert full.num_index_layers == 1 + # 3 dense + 2 sparse under the cap. + assert cfg.first_k_dense_replace == 3 + + +def test_rejects_unknown_layer_types(): + data = _hf_config().to_dict() + data["text_config"]["layer_types"][0] = "full_attention" + with pytest.raises(ValueError, match="unsupported layer_types"): + load_args(RawConfigShim(data)) + + +def test_compressed_tensors_nvfp4_detected(): + """RedHatAI/GLM-5.3-Flash-NVFP4 (llm-compressor): expert_quant resolves to + nvfp4 off the ``format`` field (quant_algo is absent for compressed-tensors).""" + cfg = parse_config(_hf_config(_CT_NVFP4_QUANT)) + assert cfg.expert_quant == "nvfp4" + + +def test_compressed_tensors_mixed_precision_detected(): + """The published RedHatAI export says ``format: mixed-precision`` at the top and + ``nvfp4-pack-quantized`` only inside the routed-expert group; expert_quant must + still resolve to nvfp4, not to the raw quant_method.""" + cfg = parse_config(_hf_config(_CT_MIXED_QUANT)) + assert cfg.expert_quant == "nvfp4" + + +def test_compressed_tensors_mixed_precision_reads_the_expert_group(): + """A mixed export with nvfp4 dense layers but fp8 experts must not report nvfp4 + experts: the group that targets the experts decides.""" + quant = { + "quant_method": "compressed-tensors", + "format": "mixed-precision", + "config_groups": { + "group_0": { + "targets": ["re:.*self_attn.*_proj$"], + "weights": {"num_bits": 4, "type": "float", "group_size": 16, "strategy": "tensor_group"}, + "format": "nvfp4-pack-quantized", + }, + "group_1": { + "targets": ["re:.*mlp\\.experts\\..*(gate|up|down)_proj$"], + "weights": {"num_bits": 8, "type": "float", "strategy": "block"}, + "format": "float-quantized", + }, + }, + } + cfg = parse_config(_hf_config(quant)) + assert cfg.expert_quant == "compressed-tensors" + + +def test_expert_source_spec_selection(): + """quant_method picks the bank source spec: compressed-tensors maps + weight_packed/weight_global_scale onto the canonical kinds with a reciprocal + global; modelopt stays identity.""" + from freetoken.models.glm5_next.weight import ( + _NVFP4_CT_SOURCE_SPEC, + _NVFP4_SOURCE_SPEC, + ) + + ct = _NVFP4_CT_SOURCE_SPEC + m = ct.key_pattern.match( + "model.language_model.layers.5.mlp.experts.7.gate_proj.weight_packed" + ) + assert m and m.group("kind") == "weight_packed" + assert ct.kind_map["weight_packed"] == "weight" + assert ct.kind_map["weight_global_scale"] == "weight_scale_2" + assert ct.global_reciprocal + # W4A16 serving never consumes the calibrated activation scale. + assert ct.key_pattern.match( + "model.language_model.layers.5.mlp.experts.7.gate_proj.input_global_scale" + ) is None + assert _NVFP4_SOURCE_SPEC.kind_map is None + assert not _NVFP4_SOURCE_SPEC.global_reciprocal + + +def test_ingest_global_reciprocal(): + import torch + from freetoken.models.glm5_next.weight import _NVFP4_CT_SOURCE_SPEC, _NVFP4_SOURCE_SPEC + from freetoken.models.nvfp4_banks import _ingest_global + + g = torch.tensor(4.0) + assert _ingest_global(_NVFP4_CT_SOURCE_SPEC, g).item() == 0.25 + assert _ingest_global(_NVFP4_SOURCE_SPEC, g).item() == 4.0 diff --git a/tests/models/test_glm5_next_kda_op.py b/tests/models/test_glm5_next_kda_op.py new file mode 100644 index 0000000000..11adbf11ca --- /dev/null +++ b/tests/models/test_glm5_next_kda_op.py @@ -0,0 +1,258 @@ +"""Glm5NextKDA op vs an eager reference (projection/conv/gate/norm wiring). + +The kernel math itself is validated in tests/kernels/test_kda.py; this test checks +the OP-level wiring: the fused in_proj split (q|k|v|b|f_a|g_a), the merged q|k|v +depthwise causal conv (+silu) against the state pool, the low-rank f/g gates, the +sigmoid-gated output RMSNorm, and prefill -> decode state continuity through +``LinearStatePool``. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + + +@pytest.fixture(autouse=True) +def _single_rank_tp(): + from freetoken.distributed import set_tp_info, try_get_tp_info + + if try_get_tp_info() is None: + set_tp_info(rank=0, size=1) + +HIDDEN, H, D, KERNEL = 256, 4, 128, 4 +P = H * D +LOWER_BOUND = -5.0 + + +def _make_args(): + from freetoken.models.glm5_next.args import Glm5NextArgs + + n_layers = 2 + return Glm5NextArgs( + hidden_size=HIDDEN, num_heads=8, + q_lora_rank=64, kv_lora_rank=32, qk_nope_head_dim=32, qk_rope_head_dim=0, + v_head_dim=32, mla_nope=True, norm_eps=1e-5, max_position=4096, + index_n_heads=0, index_head_dim=0, index_topk=0, indexer_types=(), + indexer_rope_interleave=True, index_kpool=1, index_kpool_compress=False, + index_kpool_always_select_tail=False, + linear_num_heads=H, linear_head_dim=D, linear_conv_kernel_dim=KERNEL, + linear_lower_bound=LOWER_BOUND, + layer_types=("linear_attention",) * n_layers, + mlp_layer_types=("dense",) * n_layers, + mhc=False, mhc_num_residual_streams=1, hc_eps=1e-6, + mhc_sinkhorn_iterations=0, mhc_tau=0.05, mhc_post_mult_value=2.0, + mhc_no_norm_weight=False, swiglu_limit=None, rope_theta=10000.0, + ) + + +def _make_op(seed=0): + from freetoken.models.glm5_next.kda import Glm5NextKDA + + cfg = SimpleNamespace(glm5_args=_make_args(), attn_quant="none") + op = Glm5NextKDA(cfg, layer_id=0) + torch.manual_seed(seed) + dev, dt = "cuda", torch.bfloat16 + op.in_proj.weight = torch.randn(3 * P + H + 2 * D, HIDDEN, device=dev, dtype=dt) * 0.05 + op.f_b_proj.weight = torch.randn(P, D, device=dev, dtype=dt) * 0.05 + op.g_b_proj.weight = torch.randn(P, D, device=dev, dtype=dt) * 0.05 + op.conv1d.weight = torch.randn(3 * P, 1, KERNEL, device=dev, dtype=dt) * 0.2 + op.A_log = torch.randn(H, device=dev, dtype=torch.float32) * 0.5 + op.dt_bias = torch.randn(P, device=dev, dtype=torch.float32) * 0.5 + op.o_norm.weight = torch.randn(D, device=dev, dtype=dt) * 0.1 + 1.0 + op.o_proj.weight = torch.randn(HIDDEN, P, device=dev, dtype=dt) * 0.05 + return op + + +def _make_pool(num_slots=4): + from freetoken.kvcache.linear_state_pool import LinearStatePool + from freetoken.models.config import LinearGatedDeltaGroupConfig + + group = LinearGatedDeltaGroupConfig( + name="linear", layer_ids=(0,), + num_key_heads=H, num_value_heads=H, key_head_dim=D, value_head_dim=D, + conv_kernel_dim=KERNEL, output_gate=True, variant="kda", + ) + return LinearStatePool( + group, num_slots, dtype=torch.bfloat16, + device=torch.device("cuda"), tp_size=1, + ) + + +def _patch_ctx(monkeypatch, pool, batch): + ctx = SimpleNamespace(batch=batch, linear_state_pool=pool) + monkeypatch.setattr( + "freetoken.models.glm5_next.kda.get_global_ctx", lambda: ctx + ) + + +def _l2norm(x): + return x / torch.sqrt((x * x).sum(-1, keepdim=True) + 1e-6) + + +def _reference_forward(op, x_seq, conv_ctx=None, h0=None): + """Eager op reference for one sequence [T, HIDDEN] (fp32 where the kernels are + fp32). Returns (out [T, HIDDEN], conv_tail [3P, KERNEL-1], state [H, D, D]).""" + T = x_seq.shape[0] + proj = x_seq.to(torch.bfloat16) @ op.in_proj.weight.T + conv_in, b, f_a, g_a = torch.split(proj, [3 * P, H, D, D], dim=-1) + g1 = (f_a @ op.f_b_proj.weight.T).float() + g2 = g_a @ op.g_b_proj.weight.T + + # depthwise causal conv + silu over the merged q|k|v stream, with optional + # left-context from a previous chunk (conv state semantics). + w = op.conv1d.weight.squeeze(1).float() # [3P, KERNEL] + stream = conv_in.T.float() # [3P, T] + left = ( + conv_ctx.float() + if conv_ctx is not None + else torch.zeros(3 * P, KERNEL - 1, device=x_seq.device) + ) + padded = torch.cat([left, stream], dim=1) # [3P, KERNEL-1+T] + conv = torch.stack( + [(padded[:, t : t + KERNEL] * w).sum(-1) for t in range(T)], dim=1 + ) + mixed = torch.nn.functional.silu(conv).T # [T, 3P] + conv_tail = padded[:, -(KERNEL - 1):] + + q, k, v = (t.reshape(T, H, D) for t in torch.split(mixed, [P, P, P], dim=-1)) + # bf16 round-trip like the op (kernel inputs are bf16) + q, k, v = q.to(torch.bfloat16).float(), k.to(torch.bfloat16).float(), v.to(torch.bfloat16).float() + + h = h0.clone() if h0 is not None else torch.zeros(H, D, D, device=x_seq.device) + amp = op.A_log.exp().view(H, 1) + bias = op.dt_bias.view(H, D) + core = [] + for t in range(T): + gk = LOWER_BOUND * torch.sigmoid(amp * (g1[t].view(H, D) + bias)) + h = h * gk.exp().unsqueeze(1) + kt = _l2norm(k[t]) + v_err = (v[t] - torch.einsum("hvk,hk->hv", h, kt)) * torch.sigmoid( + b[t].float() + ).unsqueeze(-1) + h = h + torch.einsum("hv,hk->hvk", v_err, kt) + core.append(torch.einsum("hvk,hk->hv", h, _l2norm(q[t]) * D**-0.5)) + core = torch.stack(core) # [T, H, D] + + xn = core.reshape(-1, D) + rms = xn * torch.rsqrt(xn.pow(2).mean(-1, keepdim=True) + op.o_norm.eps) + gated = rms * op.o_norm.weight.float() * torch.sigmoid(g2.reshape(-1, D).float()) + out = gated.reshape(T, P).to(torch.bfloat16) @ op.o_proj.weight.T + return out.float(), conv_tail, h + + +def _fla(cu, indices, has_init=None, fresh=None): + from freetoken.attention.linear import FLAMetadata + + return FLAMetadata( + cu_seqlens=cu, cache_indices=indices, + has_initial_state=has_init, fresh_state_indices=fresh, + ) + + +def _assert_close(ours, ref, tag, atol=3e-2): + err = (ours.float() - ref.float()).abs().max().item() + scale = ref.float().abs().max().item() + 1e-8 + assert err / scale < atol, f"{tag}: max abs err {err:.5f} (ref scale {scale:.3f})" + + +def test_prefill_matches_reference(monkeypatch): + op = _make_op() + pool = _make_pool() + lens = [33, 70] + total = sum(lens) + torch.manual_seed(10) + x = torch.randn(total, HIDDEN, device="cuda", dtype=torch.bfloat16) + + cu = torch.tensor([0, *torch.tensor(lens).cumsum(0).tolist()], dtype=torch.int32, device="cuda") + indices = torch.tensor([1, 2], dtype=torch.int32, device="cuda") + has_init = torch.tensor([False, False], device="cuda") + fresh = torch.tensor([1, 2], dtype=torch.int64, device="cuda") + batch = SimpleNamespace(is_decode=False, fla_metadata=_fla(cu, indices, has_init, fresh)) + _patch_ctx(monkeypatch, pool, batch) + + out = op.forward(x) + + start = 0 + for i, ln in enumerate(lens): + sl = slice(start, start + ln) + ref_out, ref_conv, ref_h = _reference_forward(op, x[sl].float()) + _assert_close(out[sl], ref_out, f"seq{i} prefill out") + _assert_close(pool.recurrent_states[0, i + 1], ref_h, f"seq{i} state") + _assert_close(pool.conv_states[0, i + 1], ref_conv, f"seq{i} conv state") + start += ln + + +def test_prefill_then_decode_continuity(monkeypatch): + op = _make_op(seed=1) + pool = _make_pool() + T0, T1 = 40, 3 + torch.manual_seed(11) + x = torch.randn(T0 + T1, HIDDEN, device="cuda", dtype=torch.bfloat16) + ref_out, _, _ = _reference_forward(op, x.float()) + + cu = torch.tensor([0, T0], dtype=torch.int32, device="cuda") + indices = torch.tensor([1], dtype=torch.int32, device="cuda") + batch = SimpleNamespace( + is_decode=False, + fla_metadata=_fla( + cu, indices, + torch.tensor([False], device="cuda"), + torch.tensor([1], dtype=torch.int64, device="cuda"), + ), + ) + _patch_ctx(monkeypatch, pool, batch) + out0 = op.forward(x[:T0]) + _assert_close(out0, ref_out[:T0], "prefill out") + + for t in range(T0, T0 + T1): + batch = SimpleNamespace( + is_decode=True, + fla_metadata=_fla( + torch.tensor([0, 1], dtype=torch.int32, device="cuda"), indices + ), + ) + _patch_ctx(monkeypatch, pool, batch) + out_t = op.forward(x[t : t + 1]) + _assert_close(out_t[0], ref_out[t], f"decode token {t}") + + +def test_chunked_prefill_continuation(monkeypatch): + """Second prefill chunk with has_initial_state=True must continue conv AND + recurrent state exactly (the chunked-prefill path).""" + op = _make_op(seed=2) + pool = _make_pool() + T0, T1 = 64, 30 + torch.manual_seed(12) + x = torch.randn(T0 + T1, HIDDEN, device="cuda", dtype=torch.bfloat16) + ref_out, ref_conv, ref_h = _reference_forward(op, x.float()) + + indices = torch.tensor([1], dtype=torch.int32, device="cuda") + batch = SimpleNamespace( + is_decode=False, + fla_metadata=_fla( + torch.tensor([0, T0], dtype=torch.int32, device="cuda"), indices, + torch.tensor([False], device="cuda"), + torch.tensor([1], dtype=torch.int64, device="cuda"), + ), + ) + _patch_ctx(monkeypatch, pool, batch) + out0 = op.forward(x[:T0]) + _assert_close(out0, ref_out[:T0], "chunk0 out") + + batch = SimpleNamespace( + is_decode=False, + fla_metadata=_fla( + torch.tensor([0, T1], dtype=torch.int32, device="cuda"), indices, + torch.tensor([True], device="cuda"), None, + ), + ) + _patch_ctx(monkeypatch, pool, batch) + out1 = op.forward(x[T0:]) + _assert_close(out1, ref_out[T0:], "chunk1 out") + _assert_close(pool.recurrent_states[0, 1], ref_h, "final state") + _assert_close(pool.conv_states[0, 1], ref_conv, "final conv state") diff --git a/tests/models/test_glm5_next_kda_snapshot.py b/tests/models/test_glm5_next_kda_snapshot.py new file mode 100644 index 0000000000..e040b92509 --- /dev/null +++ b/tests/models/test_glm5_next_kda_snapshot.py @@ -0,0 +1,98 @@ +"""KDA hybrid-radix track-snapshot contract (prefix caching for glm5_next). + +The scheduler snapshots each request's linear state at the deepest chunk-aligned +(x64) boundary of a prefill into a donatable pool slot (FLAMetadata.track_*; the +op writes it from the chunk kernel's per-chunk h + the raw conv window). A later +request restores by copying that slot and continuing with +``has_initial_state=True``. Checks, at the KDA-op level: + +* the snapshot equals the TRUE state after exactly 64 tokens (independent run) +* restore + continuation reproduces the uninterrupted run's outputs +* the 64-boundary is kpool-aligned by construction (64 % 4 == 0) +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +from tests.models.test_glm5_next_kda_op import ( # reuse the op harness + _make_op, + _make_pool, + _patch_ctx, + _fla, + _assert_close, + HIDDEN, +) + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + +CHUNK = 64 + + +@pytest.fixture(autouse=True) +def _single_rank_tp(): + from freetoken.distributed import set_tp_info, try_get_tp_info + + if try_get_tp_info() is None: + set_tp_info(rank=0, size=1) + + +def _prefill(op, pool, monkeypatch, x, slot, t0=0, has_init=False, track=None): + t = x.shape[0] + fla = _fla( + torch.tensor([0, t], dtype=torch.int32, device="cuda"), + torch.tensor([slot], dtype=torch.int32, device="cuda"), + torch.tensor([has_init], device="cuda"), + None if has_init else torch.tensor([slot], dtype=torch.int64, device="cuda"), + ) + if track is not None: + fla.track_dst, fla.track_h_row, fla.track_conv_src = track + batch = SimpleNamespace(is_decode=False, fla_metadata=fla) + _patch_ctx(monkeypatch, pool, batch) + return op.forward(x) + + +def test_snapshot_restore_roundtrip(monkeypatch): + from freetoken.kernel.fla.index import prepare_chunk_offsets + + op = _make_op(seed=5) + pool = _make_pool(num_slots=6) + total = 100 # crosses one x64 boundary; tail 36 tokens + torch.manual_seed(20) + x = torch.randn(total, HIDDEN, device="cuda", dtype=torch.bfloat16) + + # --- ground truth: state after exactly CHUNK tokens (independent run, slot 3) + _prefill(op, pool, monkeypatch, x[:CHUNK], slot=3) + true_rec = pool.recurrent_states[0, 3].clone() + true_conv = pool.conv_states[0, 3].clone() + + # --- tracked run (slot 1, snapshot into slot 2), as _build_track_metadata would + km1 = pool.conv_states.shape[-1] + cu_host = torch.tensor([0, total], dtype=torch.int64) + boh = prepare_chunk_offsets(cu_host, CHUNK).tolist() + c = (total - 1) // CHUNK # deepest mid-chunk boundary: 1 -> position 64 + track = ( + torch.tensor([2], dtype=torch.int64, device="cuda"), + torch.tensor([boh[0] + c], dtype=torch.int64, device="cuda"), + torch.tensor([[c * CHUNK - km1 + j for j in range(km1)]], dtype=torch.int64, device="cuda"), + ) + out_full = _prefill(op, pool, monkeypatch, x, slot=1, track=track) + + _assert_close(pool.recurrent_states[0, 2], true_rec, "snapshot recurrent state") + _assert_close(pool.conv_states[0, 2], true_conv, "snapshot conv state") + + # --- restore: copy snapshot -> fresh slot 4, continue [64, 100) + pool.copy_from(2, 4) + out_cont = _prefill( + op, pool, monkeypatch, x[CHUNK:], slot=4, t0=CHUNK, has_init=True + ) + _assert_close(out_cont, out_full[CHUNK:], "restored continuation outputs") + _assert_close( + pool.recurrent_states[0, 4], pool.recurrent_states[0, 1], "final states agree" + ) + + # kpool alignment is subsumed by the x64 snapshot boundary. + assert CHUNK % 4 == 0 \ No newline at end of file diff --git a/tests/models/test_glm5_next_model.py b/tests/models/test_glm5_next_model.py new file mode 100644 index 0000000000..9a499fba4b --- /dev/null +++ b/tests/models/test_glm5_next_model.py @@ -0,0 +1,210 @@ +"""Glm5NextForCausalLM wiring smoke test (tiny random model, dense MLPs). + +The per-op math is covered elsewhere (KDA kernels/op, kpool backend, mHC); this +test checks the ASSEMBLY: a 2-layer hybrid (KDA + DSA) model with mHC threading +runs prefill and decode through the real backends/pools, and the strongest +cache invariant holds -- decoding token T after prefilling [0, T) produces the +same logits as prefilling [0, T] outright (state handoff across the KDA +recurrent pool, the MLA latent pool, and the kpool indexer cache). +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +import torch + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") + +HIDDEN, VOCAB = 64, 128 +KDA_H, KDA_D = 2, 128 # KDA kernels specialize on D=128 +IDX_H, IDX_D = 16, 64 +LATENT = 32 +DEV = "cuda" + + +def _hf_config(): + from freetoken.utils.hf import RawConfigShim + + text = { + "hidden_size": HIDDEN, "intermediate_size": 96, "num_hidden_layers": 2, + "num_attention_heads": 2, "vocab_size": VOCAB, "hidden_act": "silu", + "rms_norm_eps": 1e-5, "max_position_embeddings": 4096, + "tie_word_embeddings": False, + "q_lora_rank": 48, "kv_lora_rank": LATENT, "qk_nope_head_dim": 32, + "qk_rope_head_dim": 0, "v_head_dim": 32, "mla_use_nope": True, + "index_n_heads": IDX_H, "index_head_dim": IDX_D, "index_topk": 32, + "indexer_types": ["full", "full"], "indexer_rope_interleave": True, + "index_kpool": 4, "index_kpool_compress": True, + "index_kpool_always_select_tail": True, + "linear_attn_config": { + "num_heads": KDA_H, "head_dim": KDA_D, + "short_conv_kernel_size": 4, "gate_lower_bound": -5.0, + }, + "layer_types": ["linear_attention", "deepseek_sparse_attention"], + "mlp_layer_types": ["dense", "dense"], # no MoE machinery in this test + "first_k_dense_replace": 2, + "mhc": True, "hc_mult": 4, "hc_eps": 1e-6, "hc_sinkhorn_iters": 20, + "n_routed_experts": 8, "num_experts_per_tok": 2, "n_shared_experts": 1, + "moe_intermediate_size": 32, "norm_topk_prob": True, + "routed_scaling_factor": 2.5, "scoring_func": "sigmoid", + "n_group": 1, "topk_group": 1, "swiglu_limit": 10.0, + "attention_bias": False, "model_type": "glm5_next_text", + } + return RawConfigShim({ + "architectures": ["Glm5NextForConditionalGeneration"], + "model_type": "glm5_next", "text_config": text, + }) + + +@pytest.fixture() +def rig(monkeypatch): + from freetoken.attention.dsa_indexer_kpool import Glm5NextDSABackend + from freetoken.distributed import set_tp_info, try_get_tp_info + from freetoken.kvcache.dsa_pool import KpoolDSAKVCache + from freetoken.kvcache.linear_state_pool import LinearStatePool + from freetoken.models.glm5_next.config import parse_config + from freetoken.models.glm5_next.model import Glm5NextForCausalLM + + if try_get_tp_info() is None: + set_tp_info(rank=0, size=1) + config = parse_config(_hf_config()) + + prev_dtype = torch.get_default_dtype() + torch.set_default_dtype(torch.bfloat16) + prev_dev = torch.get_default_device() + torch.set_default_device(DEV) + try: + model = Glm5NextForCausalLM(config) + finally: + torch.set_default_dtype(prev_dtype) + torch.set_default_device(prev_dev) + + # Random weights via the state-dict round trip (keeps shapes/dtypes honest). + torch.manual_seed(0) + sd = model.state_dict() + rand = {} + for k, v in sd.items(): + t = torch.randn(v.shape, dtype=torch.float32, device=DEV) * 0.05 + if k.endswith("norm.weight") or ".o_norm.weight" in k: + t = t.abs() + 0.5 + rand[k] = t.to(v.dtype) + model.load_state_dict(rand) + + kv = KpoolDSAKVCache( + latent_dim=LATENT, num_layers=2, num_pages=4, page_size=64, + dtype=torch.bfloat16, device=torch.device(DEV), + index_head_dim=IDX_D, num_index_layers=1, + index_ratio=4, num_req_slots=4, + ) + page_table = torch.full((2, 256), -1, dtype=torch.int32, device=DEV) + page_table[0] = torch.arange(256, dtype=torch.int32, device=DEV) + linear_pool = LinearStatePool( + config.linear_attention_group(), num_slots=4, + dtype=torch.bfloat16, device=torch.device(DEV), tp_size=1, + ) + + ctx = SimpleNamespace( + kv_cache=kv, page_table=page_table, linear_state_pool=linear_pool, + attn_backend=None, batch=None, + ) + for mod in ( + "freetoken.attention.dsa.get_global_ctx", + "freetoken.models.glm5_next.kda.get_global_ctx", + "freetoken.models.glm5_next.attention.get_global_ctx", + "freetoken.models.glm5_next.model.get_global_ctx", + "freetoken.layers.embedding.get_global_ctx", + ): + monkeypatch.setattr(mod, lambda: ctx) + ctx.attn_backend = Glm5NextDSABackend(config) + return model, ctx + + +def _req(device_len, cached_len): + return SimpleNamespace( + table_idx=0, device_len=device_len, extend_len=device_len - cached_len, + cached_len=cached_len, linear_slot_idx=1, mamba_ping_pong=None, + ) + + +def _batch(ctx, ids, t0, phase): + from freetoken.attention.linear import FLAMetadata + + t1 = t0 + len(ids) + is_decode = phase == "decode" + batch = SimpleNamespace( + phase=phase, + is_prefill=not is_decode, is_decode=is_decode, size=1, + reqs=[_req(t1, t0)], padded_reqs=[_req(t1, t0)], + input_ids=torch.tensor(ids, device=DEV), + positions=torch.arange(t0, t1, device=DEV), + out_loc=torch.arange(t0, t1, device=DEV), + active_table_idx=torch.tensor([0], device=DEV) if is_decode else None, + fla_metadata=FLAMetadata( + cu_seqlens=torch.tensor([0, len(ids)], dtype=torch.int32, device=DEV), + cache_indices=torch.tensor([1], dtype=torch.int32, device=DEV), + has_initial_state=None if is_decode else torch.tensor([t0 > 0], device=DEV), + fresh_state_indices=( + None if (is_decode or t0 > 0) + else torch.tensor([1], dtype=torch.int64, device=DEV) + ), + ), + mm_embeds=None, + ) + ctx.batch = batch + ctx.attn_backend.prepare_metadata(batch) + return batch + + +def _reset(ctx): + ctx.linear_state_pool.reset(1) + ctx.kv_cache._kv_buffer.zero_() + ctx.kv_cache._index_k_buffer.zero_() + ctx.kv_cache._tail_k.zero_() + ctx.kv_cache._tail_gate.zero_() + + +def test_prefill_decode_consistency(rig): + model, ctx = rig + torch.manual_seed(1) + total = 24 + ids = torch.randint(0, VOCAB, (total,)).tolist() + + # One-shot prefill over the full sequence: last-token logits per position + # are only produced for the final token, so run it twice at different splits. + _reset(ctx) + _batch(ctx, ids, 0, "prefill") + full_logits = model.forward() # [1, VOCAB] logits of the last position + assert full_logits.shape == (1, VOCAB) + assert torch.isfinite(full_logits.float()).all() + + # Prefill [0, total-1) then decode the last token: must match the one-shot run. + _reset(ctx) + _batch(ctx, ids[:-1], 0, "prefill") + model.forward() + _batch(ctx, ids[-1:], total - 1, "decode") + dec_logits = model.forward() + err = (dec_logits.float() - full_logits.float()).abs().max().item() + scale = full_logits.float().abs().max().item() + 1e-8 + assert err / scale < 3e-2, f"decode/prefill divergence: {err} (scale {scale})" + + +def test_chunked_prefill_consistency(rig): + model, ctx = rig + torch.manual_seed(2) + total = 28 # split 16 + 12; chunk boundary pool-aligned (16 % 4 == 0) + ids = torch.randint(0, VOCAB, (total,)).tolist() + + _reset(ctx) + _batch(ctx, ids, 0, "prefill") + full_logits = model.forward() + + _reset(ctx) + _batch(ctx, ids[:16], 0, "prefill") + model.forward() + _batch(ctx, ids[16:], 16, "prefill") + chunk_logits = model.forward() + err = (chunk_logits.float() - full_logits.float()).abs().max().item() + scale = full_logits.float().abs().max().item() + 1e-8 + assert err / scale < 3e-2, f"chunked/one-shot divergence: {err} (scale {scale})" diff --git a/tests/models/test_glm_dsa.py b/tests/models/test_glm_dsa.py index 84d6c1b38a..0dceabadd0 100644 --- a/tests/models/test_glm_dsa.py +++ b/tests/models/test_glm_dsa.py @@ -298,11 +298,15 @@ def test_backend_ragged_prefill_identity_and_selection(): q_pe = torch.randn(t, h, dr, device="cuda", dtype=torch.bfloat16) c_kv = torch.randn(t, dv, device="cuda", dtype=torch.bfloat16) k_rope = torch.randn(t, dr, device="cuda", dtype=torch.bfloat16) - qkw = (torch.randn(t, idx_h, idx_d, device="cuda", dtype=torch.bfloat16), - torch.randn(t, idx_d, device="cuda", dtype=torch.bfloat16), - torch.randn(t, idx_h, device="cuda").abs()) + from freetoken.attention.dsa import DSAIndexerInputs - o0 = backend.mla_forward(q_nope, q_pe, c_kv, k_rope, 0, batch, indexer_qkw=qkw) + qkw = DSAIndexerInputs( + q=torch.randn(t, idx_h, idx_d, device="cuda", dtype=torch.bfloat16), + k=torch.randn(t, idx_d, device="cuda", dtype=torch.bfloat16), + w=torch.randn(t, idx_h, device="cuda").abs(), + ) + + o0 = backend.mla_forward(q_nope, q_pe, c_kv, k_rope, 0, batch, indexer_inputs=qkw) # request A (kv <= topk): selection covers all live -> equals dense reference q_cat = torch.cat([q_nope, q_pe], -1) @@ -313,7 +317,7 @@ def test_backend_ragged_prefill_identity_and_selection(): assert (o0[j].float() - ref).abs().max().item() < 3e-2, f"A q{j}" # request B (kv > topk): causal top-k reference from the same scoring math - q_idx, k_idx, w = qkw + q_idx, k_idx, w = qkw.q, qkw.k, qkw.w for j in (0, 11): # first and last of B's queries row = 8 + j pos = 88 + j @@ -327,7 +331,7 @@ def test_backend_ragged_prefill_identity_and_selection(): # leader/follower: layer 1 (shared, no indexer) reuses layer 0's selection; with # identical latent content its output must match layer 0's - o1 = backend.mla_forward(q_nope, q_pe, c_kv, k_rope, 1, batch, indexer_qkw=None) + o1 = backend.mla_forward(q_nope, q_pe, c_kv, k_rope, 1, batch, indexer_inputs=None) assert (o0.float() - o1.float()).abs().max().item() < 3e-2 # identity wiring (dense ablation): same batch through an MLAKVCache backend @@ -338,7 +342,7 @@ def test_backend_ragged_prefill_identity_and_selection(): batch_d = SimpleNamespace(reqs=reqs, positions=positions, out_loc=out_loc, active_table_idx=None, attn_metadata=None) backend_d.prepare_metadata(batch_d) - od = backend_d.mla_forward(q_nope, q_pe, c_kv, k_rope, 0, batch_d, indexer_qkw=None) + od = backend_d.mla_forward(q_nope, q_pe, c_kv, k_rope, 0, batch_d, indexer_inputs=None) slab_d = ctx_d.kv_cache.latent_rows(0) for j in range(8): live = ctx_d.page_table[0, : 33 + j] diff --git a/tests/models/test_minimax_m3.py b/tests/models/test_minimax_m3.py index 787661dce9..9cd8c7fbb0 100644 --- a/tests/models/test_minimax_m3.py +++ b/tests/models/test_minimax_m3.py @@ -234,7 +234,7 @@ def test_auto_backend_resolution(): cfg = parse_config(_hf_config()) required = _required_attn_types(cfg) assert required == frozenset({AttnType.BSA}) - assert _resolve_auto_attention_backend(required, False) == "m3_sparse" + assert _resolve_auto_attention_backend(required) == "m3_sparse" assert attention_backend_info("m3_sparse").page_sizes == (128,) diff --git a/tests/models/test_muse_glimmer.py b/tests/models/test_muse_glimmer.py index 4a4457e9dd..cbd579e441 100644 --- a/tests/models/test_muse_glimmer.py +++ b/tests/models/test_muse_glimmer.py @@ -153,7 +153,7 @@ def test_pool_family_and_backend_resolution(): assert required == frozenset({AttnType.FULL, AttnType.SWA}) # SWA restricts serving to the triton backend (the only one in the capability # matrix that consumes per-call sliding windows), same as gemma4. - assert _resolve_auto_attention_backend(required, False) == "triton" + assert _resolve_auto_attention_backend(required) == "triton" def test_registry_resolves_architecture(): diff --git a/tests/models/test_qwen35_gguf_config.py b/tests/models/test_qwen35_gguf_config.py new file mode 100644 index 0000000000..ad9a302970 --- /dev/null +++ b/tests/models/test_qwen35_gguf_config.py @@ -0,0 +1,150 @@ +"""Unit tests for the metadata-only Qwen3.5 MoE GGUF configuration adapter. + +These tests use the public Qwen3.6-35B-A3B GGUF geometry recorded on GMKtek EVO-X2. +They prove the parser's architecture translation without requiring a 22 GiB model +file or a GPU in the test process. +""" + +from freetoken.models.gguf.config import GgufConfigShim +from freetoken.models.qwen3_5_moe.config import parse_gguf_config + + +def _qwen35moe_shim() -> GgufConfigShim: + """Return a minimal Qwen3.6-35B-A3B GGUF metadata shim for parser coverage.""" + return GgufConfigShim( + architectures=["Qwen3_5MoeGGUFForCausalLM"], + model_path="qwen35b-a3b-q4-k-m.gguf", + model_type="qwen35moe", + metadata={ + "qwen35moe.block_count": 40, + "qwen35moe.context_length": 262144, + "qwen35moe.embedding_length": 2048, + "qwen35moe.attention.head_count": 16, + "qwen35moe.attention.head_count_kv": 2, + "qwen35moe.attention.key_length": 256, + "qwen35moe.attention.layer_norm_rms_epsilon": 1e-6, + "qwen35moe.expert_count": 256, + "qwen35moe.expert_used_count": 8, + "qwen35moe.expert_feed_forward_length": 512, + "qwen35moe.expert_shared_feed_forward_length": 512, + "qwen35moe.ssm.conv_kernel": 4, + "qwen35moe.ssm.state_size": 128, + "qwen35moe.ssm.group_count": 16, + "qwen35moe.ssm.inner_size": 4096, + "qwen35moe.full_attention_interval": 4, + "qwen35moe.rope.dimension_count": 64, + "qwen35moe.rope.freq_base": 10_000_000.0, + }, + vocab_size=151936, + tie_word_embeddings=False, + ) + + +def test_qwen35moe_gguf_metadata_maps_to_the_official_hybrid_geometry(): + """Qwen's GGUF SSM fields recreate the published Gated DeltaNet dimensions.""" + config = parse_gguf_config(_qwen35moe_shim()) + + assert (config.num_layers, config.hidden_size, config.num_experts) == (40, 2048, 256) + assert (config.num_qo_heads, config.num_kv_heads, config.head_dim) == (16, 2, 256) + assert (config.num_experts_per_tok, config.moe_intermediate_size) == (8, 512) + assert config.rotary_config.rotary_dim == 64 + assert (config.expert_quant, config.moe_weight_format) == ("q4_k_q5_k", "q4_k_q5_k") + assert config.gguf_q6_down_layer_ids == () # metadata-only shim has no tensor table + + linear, full = config.attention_groups + assert linear.layer_ids == tuple(index for index in range(40) if (index + 1) % 4) + assert full.layer_ids == tuple(index for index in range(40) if not (index + 1) % 4) + assert (linear.num_key_heads, linear.num_value_heads) == (16, 32) + assert (linear.key_head_dim, linear.value_head_dim, linear.conv_kernel_dim) == (128, 128, 4) + + +def test_qwen35moe_gguf_rejects_an_invalid_ssm_value_head_partition(): + """A malformed GGUF cannot silently create a Gated DeltaNet with fractional heads.""" + shim = _qwen35moe_shim() + metadata = dict(shim.metadata) + metadata["qwen35moe.ssm.inner_size"] = 4095 + malformed = GgufConfigShim( + architectures=shim.architectures, + model_path=shim.model_path, + model_type=shim.model_type, + metadata=metadata, + vocab_size=shim.vocab_size, + tie_word_embeddings=shim.tie_word_embeddings, + ) + + try: + parse_gguf_config(malformed) + except ValueError as exc: + assert "value-head groups" in str(exc) + else: + raise AssertionError("expected malformed Gated DeltaNet geometry to be rejected") + + +def test_qwen35_dense_gguf_metadata_maps_to_dense_hybrid_geometry(): + """Qwen3.8's qwen35 metadata selects the existing dense hybrid model branch.""" + base = _qwen35moe_shim() + metadata = { + key.replace("qwen35moe.", "qwen35.", 1): value + for key, value in base.metadata.items() + if "expert_" not in key and key != "qwen35moe.expert_count" + } + metadata.update( + { + "qwen35.block_count": 64, + "qwen35.embedding_length": 5120, + "qwen35.feed_forward_length": 17408, + "qwen35.attention.head_count": 24, + "qwen35.attention.head_count_kv": 4, + "qwen35.ssm.inner_size": 6144, + "qwen35.ssm.time_step_rank": 48, + } + ) + dense = GgufConfigShim( + architectures=["Qwen3_5MoeGGUFForCausalLM"], + model_path="qwen38-27b-q4-k-m.gguf", + model_type="qwen35", + metadata=metadata, + vocab_size=248320, + tie_word_embeddings=False, + ) + + config = parse_gguf_config(dense) + + assert (config.num_layers, config.hidden_size, config.intermediate_size) == (64, 5120, 17408) + assert config.num_experts == 0 + assert config.moe_enabled is False + assert config.moe_weight_format == "qwen35_dense" + assert config.expert_quant == "none" + +# What: define the MTP exclusion regression; why: resident Qwopus GGUFs serialize one predictor block after 40 decoder layers. +def test_qwen35moe_gguf_excludes_trailing_next_token_predictor_layers(): + # What: obtain the compatible MoE fixture; why: the test should vary only serialized predictor geometry. + base = _qwen35moe_shim() + # What: copy fixture metadata; why: shared-state mutation would make unrelated tests order-dependent. + metadata = dict(base.metadata) + # What: model the real total block count; why: the artifact contains 40 decoders plus one MTP block. + metadata["qwen35moe.block_count"] = 41 + # What: declare one predictor block; why: the parser needs explicit evidence to derive executable depth. + metadata["qwen35moe.nextn_predict_layers"] = 1 + # What: build an isolated metadata shim; why: production geometry can be tested without loading a 22 GB file. + mtp = GgufConfigShim( + # What: preserve architecture identity; why: this regression targets only the Qwen MoE path. + architectures=base.architectures, + # What: preserve the placeholder path; why: metadata-only validation must not require a checkpoint. + model_path=base.model_path, + # What: preserve model type; why: the architecture metadata prefix must remain qwen35moe. + model_type=base.model_type, + # What: supply MTP-bearing metadata; why: this is the input that previously over-counted decoders. + metadata=metadata, + # What: preserve vocabulary size; why: unrelated tokenizer geometry must remain controlled. + vocab_size=base.vocab_size, + # What: preserve embedding tying; why: unrelated weight-layout behavior must remain controlled. + tie_word_embeddings=base.tie_word_embeddings, + # What: close shim construction; why: the complete object is the parser input under test. + ) + # What: parse the MTP-bearing metadata; why: the assertion must exercise production configuration logic. + config = parse_gguf_config(mtp) + # What: require 40 executable decoders; why: the trailing predictor block must not allocate or execute. + assert config.num_layers == 40 + # What: require schedules to stop at decoder 39; why: attention groups must exclude the MTP block. + assert max(layer for group in config.attention_groups for layer in group.layer_ids) == 39 diff --git a/tests/models/test_qwen35_gguf_expert_banks.py b/tests/models/test_qwen35_gguf_expert_banks.py new file mode 100644 index 0000000000..db5640919c --- /dev/null +++ b/tests/models/test_qwen35_gguf_expert_banks.py @@ -0,0 +1,35 @@ +"""Shape and registration checks for Qwen's mixed GGUF routed-expert banks.""" + +from freetoken.models.gguf.dequant import GGML_Q4_K, GGML_Q5_K, row_bytes +from freetoken.models.qwen3_5_moe.gguf import _expert_specs +from freetoken.moe.offload_cache import _BANK_BYTES_PER_EXPERT, _BANK_SCHEMAS + + +class _Config: + """Small geometry carrier matching Qwen3.6-35B-A3B's routed MoE.""" + + num_experts = 256 + hidden_size = 2048 + moe_intermediate_size = 512 + + +def test_qwen_mixed_gguf_bank_shapes_preserve_each_tensor_encoding(): + """Gate/up and down rows keep their distinct Q4_K and Q5_K byte strides.""" + specs = _expert_specs(_Config()) + gate_shape, gate_dtype = specs["gate_up"] + down_shape, down_dtype = specs["down"] + + assert gate_shape == (256, 1024, row_bytes(2048, GGML_Q4_K)) + assert down_shape == (256, 2048, row_bytes(512, GGML_Q5_K)) + assert str(gate_dtype) == "torch.uint8" + assert str(down_dtype) == "torch.uint8" + + +def test_qwen_mixed_gguf_bank_budget_matches_the_two_exact_row_layouts(): + """Cache planning counts Q4_K gate/up bytes and Q5_K down bytes separately.""" + hidden, intermediate = 2048, 512 + expected = 2 * intermediate * row_bytes(hidden, GGML_Q4_K) + hidden * row_bytes( + intermediate, GGML_Q5_K + ) + assert _BANK_SCHEMAS["q4_k_q5_k"] == ("gate_up", "down") + assert _BANK_BYTES_PER_EXPERT["q4_k_q5_k"](hidden, intermediate) == expected diff --git a/tests/models/test_qwen35_gguf_ssm_a.py b/tests/models/test_qwen35_gguf_ssm_a.py new file mode 100644 index 0000000000..963c7b09f7 --- /dev/null +++ b/tests/models/test_qwen35_gguf_ssm_a.py @@ -0,0 +1,86 @@ +"""Regression coverage for Qwen GGUF's serialized Gated DeltaNet decay.""" + +import pytest +import torch + +from freetoken.models.qwen3_5_moe.gguf import ( + _restore_gdn_value_head_input_blocks, + _restore_gdn_value_head_order, + _restore_gdn_value_head_rows, + _ssm_a_to_a_log, +) + + +def test_qwen_gguf_ssm_a_inverts_llama_cpp_negative_exponential(): + """The loader recovers FreeToken's A_log rather than exponentiating twice.""" + a_log = torch.tensor([-3.0, -1.25, 0.0, 2.5], dtype=torch.float32) + serialized = -torch.exp(a_log) + + recovered = _ssm_a_to_a_log(serialized) + + torch.testing.assert_close(recovered, a_log) + + +@pytest.mark.parametrize( + "invalid", + [torch.tensor([0.0]), torch.tensor([1.0]), torch.tensor([float("nan")])], +) +def test_qwen_gguf_ssm_a_rejects_values_that_are_not_negative_finite_decay(invalid): + """Malformed decay coefficients cannot silently corrupt recurrent execution.""" + with pytest.raises(ValueError, match="finite negative"): + _ssm_a_to_a_log(invalid) + + +def test_qwen_gguf_gdn_value_heads_restore_grouped_llama_cpp_order(): + """Two GGUF groups become consecutive per-key-head values in FreeToken.""" + grouped = torch.tensor([[0, 1], [10, 11], [20, 21], [30, 31]]) + + restored = _restore_gdn_value_head_order(grouped, num_key_heads=2) + + torch.testing.assert_close(restored, torch.tensor([[0, 1], [20, 21], [10, 11], [30, 31]])) + + +def test_qwen_gguf_gdn_value_heads_reject_invalid_key_head_partition(): + """A malformed GGUF head layout fails before it reaches the recurrent kernel.""" + with pytest.raises(ValueError, match="incompatible"): + _restore_gdn_value_head_order(torch.zeros(3), num_key_heads=2) + + +def test_qwen_gguf_gdn_value_head_rows_restore_complete_quantized_rows(): + """A grouped Q8 projection V suffix regains Qwen's per-key-head order.""" + grouped = torch.tensor( + [[0, 0], [2, 2], [4, 4], [6, 6], [1, 1], [3, 3], [5, 5], [7, 7]], + dtype=torch.uint8, + ) + + # Each value head occupies two complete output rows. Eight heads therefore + # require sixteen rows; eight rows would describe four heads with ratio 1. + grouped = grouped.repeat_interleave(2, dim=0) + restored = _restore_gdn_value_head_rows(grouped, num_key_heads=4, head_dim=2) + + expected = torch.tensor( + [[0, 0], [1, 1], [2, 2], [3, 3], [4, 4], [5, 5], [6, 6], [7, 7]], + dtype=torch.uint8, + ) + torch.testing.assert_close(restored, expected.repeat_interleave(2, dim=0)) + + +def test_qwen_gguf_gdn_output_restores_q8_blocks_without_dequantizing(): + """Q8_0 blocks move intact when restoring GDN output-projection columns.""" + grouped_order = (0, 2, 4, 6, 1, 3, 5, 7) + grouped = torch.stack( + [ + torch.cat([torch.full((34,), head, dtype=torch.uint8) for head in grouped_order]), + torch.cat([torch.full((34,), head + 20, dtype=torch.uint8) for head in grouped_order]), + ] + ) + + restored = _restore_gdn_value_head_input_blocks(grouped, num_key_heads=4, head_dim=32) + + expected = torch.stack( + [ + torch.cat([torch.full((34,), head, dtype=torch.uint8) for head in range(8)]), + torch.cat([torch.full((34,), head + 20, dtype=torch.uint8) for head in range(8)]), + ] + ) + torch.testing.assert_close(restored, expected) diff --git a/tests/models/test_qwen36_gdn_grouped_output.py b/tests/models/test_qwen36_gdn_grouped_output.py new file mode 100644 index 0000000000..a7bc41289b --- /dev/null +++ b/tests/models/test_qwen36_gdn_grouped_output.py @@ -0,0 +1,45 @@ +import torch +import pytest +from types import SimpleNamespace + +from freetoken.models.qwen3_5_moe.gdn import Qwen3_5GatedDeltaNet +from freetoken.models.qwen3_5_moe.gguf import _restore_gdn_value_head_order + + +def test_grouped_output_activation_inverts_gguf_value_head_restore(): + """Q4_K GDN output weights remain packed while activation order is inverted.""" + grouped = torch.arange(2 * 48 * 128, dtype=torch.float32).reshape(2, 48, 128) + free_token_order = _restore_gdn_value_head_order(grouped.transpose(0, 1), 16).transpose(0, 1) + + op = Qwen3_5GatedDeltaNet.__new__(Qwen3_5GatedDeltaNet) + op.num_k_heads = 16 + op.num_v_heads = 48 + op.head_v_dim = 128 + + restored_grouped = op._gguf_group_value_heads_for_out_proj(free_token_order.reshape(2, -1)) + assert torch.equal(restored_grouped.reshape_as(grouped), grouped) + + +@pytest.mark.parametrize("qkv_type", [8, 12, 14]) +@pytest.mark.parametrize("gate_type", [8, 12, 14]) +def test_dense_gdn_uses_each_tensor_descriptor(qkv_type, gate_type): + from freetoken.distributed import set_tp_info, try_get_tp_info + from freetoken.models.gguf.dequant import row_bytes + + if try_get_tp_info() is None: + set_tp_info(rank=0, size=1) + config = SimpleNamespace(gguf_tensor_types=( + ("blk.0.attn_qkv.weight", qkv_type), + ("blk.0.attn_gate.weight", gate_type), + )) + with torch.device("meta"): + op = Qwen3_5GatedDeltaNet( + hidden_size=4096, num_k_heads=16, num_v_heads=32, + head_k_dim=128, head_v_dim=128, conv_kernel_size=4, + rms_norm_eps=1e-6, layer_id=0, attn_quant="gguf_mixed", config=config, + ) + assert op.in_proj_qkv._quant_type == qkv_type + assert op.in_proj_z._quant_type == gate_type + assert op.in_proj_qkv.qweight.shape[-1] == row_bytes(4096, qkv_type) + assert op.in_proj_z.qweight.shape[-1] == row_bytes(4096, gate_type) + assert not hasattr(op, "in_proj_qkvz") diff --git a/tests/moe/test_fused_moe.py b/tests/moe/test_fused_moe.py index 1fd0f2e523..e2c4c30f20 100644 --- a/tests/moe/test_fused_moe.py +++ b/tests/moe/test_fused_moe.py @@ -2,6 +2,70 @@ import torch +def test_fused_topk_keeps_reference_router_on_rocm(monkeypatch): + """HIP retains the exact PyTorch router until end-to-end parity is proven.""" + from freetoken.kernel import backend + from freetoken.moe import fused + + weights = torch.tensor([[0.7, 0.3]], dtype=torch.float32) + ids = torch.tensor([[4, 9]], dtype=torch.int32) + calls = [] + + monkeypatch.setattr(backend, "is_rocm_runtime", lambda: True) + monkeypatch.setattr( + fused, + "_torch_fused_topk", + lambda logits, topk, renormalize, limit: ( + calls.append((logits, topk, renormalize, limit)) or (weights, ids) + ), + ) + + got_weights, got_ids = fused.fused_topk( + torch.empty((1, 3)), torch.empty((1, 16)), topk=2, renormalize=True + ) + + assert len(calls) == 1 + assert calls[0][1:] == (2, True, None) + assert got_weights is weights + assert got_ids is ids + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") +def test_rocm_grouped_moe_alignment_assigns_each_routed_expert(): + """The ROCm alignment path must not collapse all grouped blocks onto expert zero.""" + from freetoken.kernel.backend import is_rocm_runtime + + if not is_rocm_runtime(): + pytest.skip("ROCm-specific grouped-MoE regression") + + from freetoken.moe.fused import moe_align_block_size + + # Each flattened route names a distinct expert. With a 16-row grouped + # block each route consumes exactly one padded block, making ownership + # unambiguous and exposing the former gfx1151 small-alignment defect. + topk_ids = torch.tensor( + [[31, 4, 18, 0], [29, 7, 35, 12], [6, 21, 1, 33], [16, 3, 28, 9]], + device="cuda", + dtype=torch.int32, + ) + sorted_ids, expert_ids, num_tokens_post_padded = moe_align_block_size( + topk_ids, block_size=16, num_experts=37 + ) + torch.cuda.synchronize() + + padded = int(num_tokens_post_padded.cpu()) + routed_experts = sorted(set(topk_ids.flatten().cpu().tolist())) + block_experts = expert_ids[: padded // 16].cpu().tolist() + routed_tokens = sorted_ids[:padded].cpu() + + assert padded == topk_ids.numel() * 16 + assert block_experts == routed_experts + assert sorted(routed_tokens[routed_tokens < topk_ids.numel()].tolist()) == list( + range(topk_ids.numel()) + ) + assert torch.all(routed_tokens[(routed_tokens >= topk_ids.numel())] == topk_ids.numel()) + + def _activation_and_mul(gate_up: torch.Tensor, activation: str) -> torch.Tensor: gate, up = gate_up.chunk(2, dim=-1) if activation == "silu": @@ -274,3 +338,98 @@ def test_fused_experts_decode_activation_and_router_weight_modes( torch.cuda.synchronize() torch.testing.assert_close(output, expected, rtol=5e-2, atol=5e-2) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") +def test_fused_topk_handles_non_power_of_2_k(): + """A top-10 router (qwen4_exp) must route like any other k.""" + from freetoken.moe.fused import _torch_fused_topk, fused_topk + + gating = torch.randn(5, 64, device="cuda") + hidden = torch.randn(5, 8, device="cuda") + weights, ids = fused_topk(hidden, gating, 10, renormalize=True) + ref_w, ref_i = _torch_fused_topk(gating, 10, True, None) + assert torch.equal(ids, ref_i) + torch.testing.assert_close(weights, ref_w, rtol=1e-5, atol=1e-6) + + +# The in-repo triton router behind fused_topk; fp32 logits keep the reference top-k tie-free. +# Ties get their own case below, because torch.topk does not break them by expert id. +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") +@pytest.mark.parametrize("renormalize", [True, False]) +@pytest.mark.parametrize( + "num_tokens,num_experts,topk", + [(1, 512, 10), (7, 512, 10), (129, 512, 10), (33, 512, 6), (4, 64, 3)], +) +def test_fused_topk_softmax_matches_torch_reference(num_tokens, num_experts, topk, renormalize): + from freetoken.kernel.triton.moe_router import fused_topk_softmax + from freetoken.moe.fused import _torch_fused_topk + + gen = torch.Generator(device="cuda").manual_seed(num_tokens * 31 + topk) + gating = torch.randn(num_tokens, num_experts, generator=gen, device="cuda") + + weights, ids = fused_topk_softmax(gating, topk, renormalize) + ref_w, ref_i = _torch_fused_topk(gating, topk, renormalize, None) + + assert weights.dtype == torch.float32 and ids.dtype == torch.int32 + assert torch.equal(ids, ref_i) + torch.testing.assert_close(weights, ref_w, rtol=1e-5, atol=1e-6) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") +def test_fused_topk_softmax_ties_pick_the_lowest_expert_id(): + from freetoken.kernel.triton.moe_router import fused_topk_softmax + + gating = torch.full((2, 8), -10.0, device="cuda") + gating[0, [6, 2, 5]] = 1.0 # three-way tie for two slots + gating[1] = 0.0 # whole row tied + + weights, ids = fused_topk_softmax(gating, 3, renormalize=True) + + assert ids[0].tolist() == [2, 5, 6] + assert ids[1].tolist() == [0, 1, 2] + torch.testing.assert_close(weights, torch.full((2, 3), 1.0 / 3.0, device="cuda")) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") +@pytest.mark.parametrize("limit_dtype", [torch.int32, torch.int64]) +def test_fused_topk_softmax_masks_padded_rows(limit_dtype): + from freetoken.kernel.triton.moe_router import fused_topk_softmax + from freetoken.moe.fused import _torch_fused_topk + + gen = torch.Generator(device="cuda").manual_seed(5) + gating = torch.randn(16, 512, generator=gen, device="cuda") + limit = torch.tensor(5, dtype=limit_dtype, device="cuda") + + weights, ids = fused_topk_softmax(gating, 10, True, limit) + ref_w, ref_i = _torch_fused_topk(gating, 10, True, limit) + + assert (ids[5:] == -1).all() + assert torch.equal(ids, ref_i) + torch.testing.assert_close(weights, ref_w, rtol=1e-5, atol=1e-6) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA is required") +def test_fused_topk_softmax_is_cuda_graph_capturable(): + """The padded-row limit must come off the device tensor, not a host read baked into the graph.""" + from freetoken.kernel.triton.moe_router import fused_topk_softmax + + gen = torch.Generator(device="cuda").manual_seed(11) + gating = torch.randn(8, 512, generator=gen, device="cuda") + limit = torch.tensor(8, dtype=torch.int32, device="cuda") + + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + fused_topk_softmax(gating, 10, True, limit) + torch.cuda.current_stream().wait_stream(stream) + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + _, ids = fused_topk_softmax(gating, 10, True, limit) + + limit.fill_(3) + graph.replay() + torch.cuda.synchronize() + assert (ids[:3] != -1).all() + assert (ids[3:] == -1).all() diff --git a/tests/reproduce/test_collect_host_manifest.py b/tests/reproduce/test_collect_host_manifest.py new file mode 100644 index 0000000000..fc24bce11b --- /dev/null +++ b/tests/reproduce/test_collect_host_manifest.py @@ -0,0 +1,48 @@ +"""Static safety checks for the portable public manifest collector.""" + +from __future__ import annotations + +import unittest +from pathlib import Path + + +class CollectHostManifestTests(unittest.TestCase): + """Keep the public collector portable, privacy-conscious, and HIP-only.""" + + @classmethod + def setUpClass(cls) -> None: + root = Path(__file__).resolve().parents[2] + cls.script = (root / "scripts" / "reproduce" / "collect_host_manifest.sh").read_text( + encoding="utf-8" + ) + + def test_requires_a_native_hip_pytorch_device(self) -> None: + self.assertIn("PyTorch does not report a HIP runtime", self.script) + self.assertIn("PyTorch HIP device is unavailable", self.script) + + def test_default_manifest_redacts_hostname_and_omits_sensitive_inventory(self) -> None: + self.assertIn('PUBLIC_HOSTNAME="redacted"', self.script) + self.assertNotIn("uname -a", self.script) + self.assertIn('printf \'kernel_system=%s\\n\'', self.script) + self.assertNotIn("ps -eo", self.script) + self.assertNotIn("lsblk -o NAME,MODEL,SERIAL", self.script) + + def test_public_collector_has_no_host_identifier_or_personal_path_dependency(self) -> None: + forbidden_host = "lan" + "-" + "223" + self.assertNotIn(forbidden_host, self.script.lower()) + self.assertNotRegex(self.script, r"/home/[A-Za-z][A-Za-z0-9_-]+") + + def test_collector_accepts_a_git_worktree_and_requires_a_new_artifact_directory(self) -> None: + self.assertIn('git -C "${SOURCE_DIR}" rev-parse --is-inside-work-tree', self.script) + self.assertNotIn('[[ ! -d "${SOURCE_DIR}/.git" ]]', self.script) + self.assertIn('if [[ -e "${ARTIFACT_DIR}" ]]', self.script) + self.assertIn("artifact directory already exists", self.script) + + def test_checksums_cover_the_raw_reports_and_manifest(self) -> None: + self.assertIn("sha256sum system.txt source-state.txt python-hip.json rocminfo.txt rocm-smi.txt", self.script) + self.assertIn('"manifest.json"', self.script) + self.assertIn('"SHA256SUMS"', self.script) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/reproduce/test_run_local_api_benchmark.py b/tests/reproduce/test_run_local_api_benchmark.py new file mode 100644 index 0000000000..2e31e1abfd --- /dev/null +++ b/tests/reproduce/test_run_local_api_benchmark.py @@ -0,0 +1,34 @@ +"""Safety tests for the portable loopback-only API benchmark client.""" + +from __future__ import annotations + +import contextlib +import io +import unittest + +from benchmarks.reproduce.run_local_api_benchmark import parse_args, require_loopback_url + + +class LoopbackUrlTests(unittest.TestCase): + def test_accepts_localhost_variants(self) -> None: + self.assertEqual(require_loopback_url("http://127.0.0.1:8000/v1"), "http://127.0.0.1:8000/v1") + self.assertEqual(require_loopback_url("https://localhost/v1/"), "https://localhost/v1") + + def test_rejects_remote_target(self) -> None: + with self.assertRaisesRegex(ValueError, "loopback"): + require_loopback_url("http://192.168." + "1.223:1919/v1") + + def test_quality_mode_requires_visible_text_gate(self) -> None: + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + parse_args(["--model", "model", "--tokenizer", "tokenizer", "--artifact-dir", "artifact", "--prompt", "hello"]) + + def test_throughput_requires_at_least_two_tokens(self) -> None: + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + parse_args([ + "--model", "model", "--tokenizer", "tokenizer", "--artifact-dir", "artifact", + "--prompt", "hello", "--mode", "throughput", "--max-tokens", "1", + ]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/scheduler/test_abort_inflight_prefill.py b/tests/scheduler/test_abort_inflight_prefill.py index 1c9a9fba51..a50d0d3230 100644 --- a/tests/scheduler/test_abort_inflight_prefill.py +++ b/tests/scheduler/test_abort_inflight_prefill.py @@ -37,7 +37,7 @@ def _pool(num_slots=16): g = LinearGatedDeltaGroupConfig( name="linear", layer_ids=(0,), num_key_heads=2, num_value_heads=4, - key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True, + key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu", ) return LinearStatePool(group=g, num_slots=num_slots, dtype=torch.bfloat16, device=torch.device("cpu"), tp_size=1) diff --git a/tests/scheduler/test_hybrid_cache_manager.py b/tests/scheduler/test_hybrid_cache_manager.py index 19be56f3cd..080999addf 100644 --- a/tests/scheduler/test_hybrid_cache_manager.py +++ b/tests/scheduler/test_hybrid_cache_manager.py @@ -16,7 +16,7 @@ def _pool(num_slots=16): g = LinearGatedDeltaGroupConfig( name="linear", layer_ids=(0,), num_key_heads=2, num_value_heads=4, - key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True, + key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu", ) return LinearStatePool(group=g, num_slots=num_slots, dtype=torch.bfloat16, device=torch.device("cpu"), tp_size=1) @@ -116,6 +116,43 @@ def test_rebuild_reclaims_donated_gdn_slots(): assert pool.num_free_slots == pool.num_slots - 1 # all GDN slots reclaimed (no leak) +def test_prefill_chunk_ends_on_a_page_boundary(): + """A hybrid chunk must end page-aligned: the snapshot commit skips any other boundary.""" + from freetoken.scheduler.prefill import ChunkedReq, PrefillAdder + from freetoken.scheduler.table import TableManager + from freetoken.scheduler.utils import PendingReq + + pool = _pool() + pt = torch.zeros(4, 512, dtype=torch.int32) + cm = CacheManager(64, 64, pt, "hybrid_radix", linear_state_pool=pool) + assert cm.prefill_chunk_align == 64 + tm = TableManager(max_running_reqs=4, page_table=pt) + pending = PendingReq(0, torch.arange(300, dtype=torch.int32), SamplingParams(max_tokens=1)) + + adder = PrefillAdder(token_budget=100, reserved_size=0, cache_manager=cm, table_manager=tm) + req = adder.try_add_one(pending) + assert isinstance(req, ChunkedReq) and req.extend_len == 64 + + # a budget below one page keeps the unaligned chunk rather than stalling the request + adder = PrefillAdder(token_budget=40, reserved_size=0, cache_manager=cm, table_manager=tm) + assert adder.try_add_one(pending).extend_len == 40 + + +def test_naive_cache_does_not_align_prefill_chunks(): + """The alignment hook is hybrid-only; every other cache keeps the raw budget chunk.""" + from freetoken.scheduler.prefill import PrefillAdder + from freetoken.scheduler.table import TableManager + from freetoken.scheduler.utils import PendingReq + + pt = torch.zeros(4, 512, dtype=torch.int32) + cm = CacheManager(64, 64, pt, "radix") + assert cm.prefill_chunk_align == 1 + tm = TableManager(max_running_reqs=4, page_table=pt) + adder = PrefillAdder(token_budget=100, reserved_size=0, cache_manager=cm, table_manager=tm) + pending = PendingReq(0, torch.arange(300, dtype=torch.int32), SamplingParams(max_tokens=1)) + assert adder.try_add_one(pending).extend_len == 100 + + def test_pool_sizing_covers_4mr_floor(): """C6: pool must reserve the 4-slot-per-request non-evictable floor even at a tiny ratio.""" from types import SimpleNamespace diff --git a/tests/server/test_message_wire.py b/tests/server/test_message_wire.py index 3bd7cc6ba5..625849996b 100644 --- a/tests/server/test_message_wire.py +++ b/tests/server/test_message_wire.py @@ -7,6 +7,8 @@ from __future__ import annotations +import torch + from freetoken.message import ( BaseBackendMsg, DetokenizeMsg, @@ -16,8 +18,13 @@ CacheRebuildMsg, CacheRebuildReply, CacheRebuildResultMsg, + CacheStatsBackendMsg, + CacheStatsMsg, + CacheStatsReply, + CacheStatsResultMsg, PromptAdmittedMsg, TokenizeMsg, + UserMsg, UserReply, ) from freetoken.core import SamplingParams @@ -53,6 +60,30 @@ def test_cache_rebuild_reply_roundtrip(): assert (out.request_id, out.status, out.error) == ("r3", "failed", "boom") +def test_cache_stats_messages_roundtrip(): + """The read-only cache-statistics control path preserves nested counter data on every hop.""" + + request = CacheStatsMsg(request_id="stats-request") + backend = CacheStatsBackendMsg(request_id="stats-request") + result = CacheStatsResultMsg( + request_id="stats-request", + stats={"available": True, "summary": {"miss_rate": 0.25}}, + ) + reply = CacheStatsReply( + request_id="stats-request", + stats={"available": True, "summary": {"miss_rate": 0.25}}, + ) + + assert isinstance(BaseTokenizerMsg.decoder(BaseTokenizerMsg.encoder(request)), CacheStatsMsg) + assert isinstance(BaseBackendMsg.decoder(backend.encoder()), CacheStatsBackendMsg) + decoded_result = BaseTokenizerMsg.decoder(BaseTokenizerMsg.encoder(result)) + decoded_reply = BaseFrontendMsg.decoder(BaseFrontendMsg.encoder(reply)) + assert isinstance(decoded_result, CacheStatsResultMsg) + assert isinstance(decoded_reply, CacheStatsReply) + assert decoded_result.stats == reply.stats + assert decoded_reply.stats == reply.stats + + def test_prompt_admitted_msg_roundtrip(): msg = PromptAdmittedMsg(uid=42, prompt_tokens=1234, cached_tokens=500) out = BaseTokenizerMsg.decoder(BaseTokenizerMsg.encoder(msg)) @@ -122,3 +153,27 @@ def test_client_dicts_with_the_wire_tag_key_survive_intact(): assert isinstance(out, TokenizeMsg) assert out.chat_template_kwargs == payload assert out.tools[0]["function"]["parameters"] == payload + + +def test_backend_wire_preserves_multidimensional_cpu_tensors(): + """Gemma 4 patch data and image positions survive the tokenizer scheduler ZMQ hop.""" + msg = UserMsg( + uid=9, + input_ids=torch.tensor([1, 2, 3], dtype=torch.int32), + sampling_params=SamplingParams(), + # ``mm_embeds`` is used by the in-process offline path. Online requests + # instead move these CPU tensors to the scheduler, where its ROCm-owned + # model instance runs the vision tower and projector. + mm_pixel_values=torch.arange(24, dtype=torch.float32).reshape(1, 2, 12), + mm_image_position_ids=torch.tensor([[[0, 0], [0, 1]]], dtype=torch.int64), + ) + decoded = BaseBackendMsg.decoder(msg.encoder()) + assert isinstance(decoded, UserMsg) + assert decoded.mm_pixel_values is not None + assert decoded.mm_image_position_ids is not None + assert decoded.mm_pixel_values.shape == (1, 2, 12) + assert decoded.mm_pixel_values.dtype == torch.float32 + assert decoded.mm_image_position_ids.shape == (1, 2, 2) + assert decoded.mm_image_position_ids.dtype == torch.int64 + assert torch.equal(decoded.mm_pixel_values, msg.mm_pixel_values) + assert torch.equal(decoded.mm_image_position_ids, msg.mm_image_position_ids) diff --git a/tests/server/test_openai_image_input.py b/tests/server/test_openai_image_input.py new file mode 100644 index 0000000000..73e89a07e6 --- /dev/null +++ b/tests/server/test_openai_image_input.py @@ -0,0 +1,25 @@ +"""OpenAI image_url extraction stays aligned with generation's content markers.""" + +from freetoken.server.generation import render_messages +from freetoken.server.openai_api import _openai_image_urls + + +def test_openai_images_extract_in_rendered_marker_order() -> None: + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "first"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}, + {"type": "text", "text": "second"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,BB=="}}, + ], + } + ] + assert _openai_image_urls(messages) == [ + {"url": "data:image/png;base64,AA=="}, + {"url": "data:image/png;base64,BB=="}, + ] + assert render_messages(messages)[0]["content"] == ( + "first<|freetoken-image|>second<|freetoken-image|>" + ) diff --git a/tests/tokenizer/test_gemma4_image.py b/tests/tokenizer/test_gemma4_image.py new file mode 100644 index 0000000000..85f73da2a4 --- /dev/null +++ b/tests/tokenizer/test_gemma4_image.py @@ -0,0 +1,101 @@ +"""CPU-only regression coverage for Gemma4 OpenAI image preprocessing.""" + +from __future__ import annotations + +import base64 +import io + +import torch +from PIL import Image + +from freetoken.core import SamplingParams +from freetoken.message import TokenizeMsg +from freetoken.tokenizer.gemma4_image import decode_openai_image_data_url, gemma4_image_inputs +from freetoken.tokenizer.tokenize import TokenizeManager + + +def _png_data_url() -> str: + image = Image.new("RGB", (16, 16), (255, 0, 0)) + buf = io.BytesIO() + image.save(buf, format="PNG") + return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode("ascii") + + +def test_data_url_becomes_gemma4_patch_and_position_tensors() -> None: + """A tiny image scales to a valid grid and preserves convolution channel planes.""" + inputs = gemma4_image_inputs(decode_openai_image_data_url({"url": _png_data_url()})) + assert inputs.pixel_values.shape == (2520, 768) + assert inputs.image_position_ids.shape == (2520, 2) + assert inputs.soft_token_count == 256 + assert torch.equal(inputs.image_position_ids[0], torch.tensor([0, 0])) + assert torch.equal(inputs.image_position_ids[1], torch.tensor([1, 0])) + assert torch.equal(inputs.image_position_ids[2304], torch.tensor([-1, -1])) + # v.patch_embd.weight has the standard [out, channel, patch_y, patch_x] + # convolution layout. The first vector is therefore all red samples, + # followed by green and blue, not one RGB triplet per source pixel. + channel_size = 16 * 16 + first_patch = inputs.pixel_values[0] + assert torch.equal(first_patch[:channel_size], torch.ones(channel_size)) + assert torch.equal(first_patch[channel_size : 2 * channel_size], torch.zeros(channel_size)) + assert torch.equal(first_patch[2 * channel_size :], torch.zeros(channel_size)) + + +def test_remote_image_url_is_rejected_without_network_fetching() -> None: + """The local inference endpoint must not become an arbitrary network client.""" + try: + decode_openai_image_data_url("https://example.com/image.png") + except ValueError as exc: + assert "only data:image URLs" in str(exc) + else: # pragma: no cover - keeps the failure obvious if the security policy regresses + raise AssertionError("remote image URL was unexpectedly accepted") + + +def test_tokenizer_expands_one_image_marker_and_stages_shaped_cpu_tensors() -> None: + """The online tokenizer computes the placeholder count from the same processed image.""" + msg = TokenizeMsg( + uid=1, + text="unused", + sampling_params=SamplingParams(), + image_urls=[{"url": _png_data_url()}], + ) + manager = TokenizeManager.__new__(TokenizeManager) + prompt = manager._expand_gemma4_images(msg, "before<|freetoken-image|>after") + assert prompt == "before<|image>" + "<|image|>" * 256 + "after" + assert msg.mm_pixel_values is not None + assert msg.mm_image_position_ids is not None + assert msg.mm_pixel_values.shape == (1, 2520, 768) + assert msg.mm_image_position_ids.shape == (1, 2520, 2) + + +def test_preflight_expansion_uses_the_same_single_marker_pass_as_worker() -> None: + """A streamed request can validate image tensors without changing worker semantics.""" + + msg = TokenizeMsg( + uid=1, + text=[{"role": "user", "content": "image request"}], + sampling_params=SamplingParams(), + image_urls=[{"url": _png_data_url()}], + ) + manager = TokenizeManager.__new__(TokenizeManager) + manager._sanitize_effort = lambda kwargs: kwargs + manager._render = lambda _messages, _tools, _kwargs: "before<|freetoken-image|>after" + + rendered = manager.render_prompt(msg) + prompt = manager._expand_gemma4_images(msg, rendered) + + assert prompt == "before<|image>" + "<|image|>" * 256 + "after" + assert msg.mm_pixel_values is not None + assert msg.mm_image_position_ids is not None + +def test_extreme_aspect_ratio_is_rejected_before_large_resize() -> None: + # Create a narrow image with modest source pixels but a resize target that would exceed the patch budget. + image = Image.new("RGB", (1, 100_000), (0, 0, 0)) + # Capture the expected validation error so the test proves the allocation guard is active. + try: + gemma4_image_inputs(image) + except ValueError as exc: + # Require the public error to identify aspect geometry rather than an incidental Pillow memory failure. + assert "aspect ratio" in str(exc) + else: + # Fail loudly if the dangerous resize path becomes reachable again. + raise AssertionError("extreme-aspect image unexpectedly passed the fixed patch budget") diff --git a/tests/utils/test_rocm_runtime.py b/tests/utils/test_rocm_runtime.py new file mode 100644 index 0000000000..7f6a69bfe9 --- /dev/null +++ b/tests/utils/test_rocm_runtime.py @@ -0,0 +1,69 @@ +"""Regression coverage for the CUDA-namespace compatibility boundary on ROCm. + +PyTorch exposes AMD devices through ``torch.cuda`` so CUDA-oriented Python +programs can run on HIP. FreeToken must not mistake a ``gfx11xx`` capability +for a newer NVIDIA SM capability, nor select optional CUDA binaries merely +because a stale package happens to be installed in the environment. +""" + +import torch +from pathlib import Path + +from freetoken.kernel import backend +from freetoken.utils import arch + + +def test_rocm_never_satisfies_nvidia_sm_gates(monkeypatch): + """HIP hardware is excluded before numerical NVIDIA capability comparison.""" + monkeypatch.setattr(torch.version, "hip", "7.15") + monkeypatch.setattr(torch.version, "cuda", None) + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.cuda, "get_device_capability", lambda: (11, 5)) + arch.is_rocm_runtime.cache_clear() + arch._get_torch_cuda_version.cache_clear() + + try: + assert arch.is_rocm_runtime() is True + assert arch._get_torch_cuda_version() is None + assert arch.is_sm90_supported() is False + assert arch.is_sm100_supported() is False + finally: + # Cached runtime detection must not leak the synthetic HIP state into + # unrelated test modules that run later in the same interpreter. + arch.is_rocm_runtime.cache_clear() + arch._get_torch_cuda_version.cache_clear() + + +def test_rocm_disables_cuda_only_optional_backends(monkeypatch): + """Triton remains available, while CUDA binary packages are bypassed on HIP.""" + monkeypatch.setattr(backend, "is_rocm_runtime", lambda: True) + monkeypatch.setattr(backend, "_importable", lambda _name: True) + backend.is_flashinfer_installed.cache_clear() + backend.is_sgl_kernel_installed.cache_clear() + backend.is_triton_kernels_installed.cache_clear() + backend.driver_cuda_version.cache_clear() + + try: + assert backend.is_flashinfer_installed() is False + assert backend.is_sgl_kernel_installed() is False + assert backend.is_triton_kernels_installed() is False + assert backend.driver_cuda_version() is None + finally: + backend.is_flashinfer_installed.cache_clear() + backend.is_sgl_kernel_installed.cache_clear() + backend.is_triton_kernels_installed.cache_clear() + backend.driver_cuda_version.cache_clear() + + +def test_native_extension_build_uses_torch_hip_and_explicit_host_macro(): + """ROCm host C++ builds must not infer their ABI from CUDA toolkit presence.""" + + root = Path(__file__).resolve().parents[2] + setup_source = (root / "setup.py").read_text(encoding="utf-8") + compat_source = (root / "python/freetoken/kernel/csrc/hip_compat.h").read_text( + encoding="utf-8" + ) + + assert "IS_ROCM = torch.version.hip is not None" in setup_source + assert 'GPU_RUNTIME_MACROS = [("FREETOKEN_USE_ROCM", "1")]' in setup_source + assert "defined(FREETOKEN_USE_ROCM)" in compat_source