diff --git a/.github/workflows/ci-ready.yml b/.github/workflows/ci-ready.yml index 4f84c3dac..63de236d4 100644 --- a/.github/workflows/ci-ready.yml +++ b/.github/workflows/ci-ready.yml @@ -68,6 +68,37 @@ jobs: PR_BODY: ${{ github.event.pull_request.body }} run: python3 tools/check_pr_references.py + benchmark-artifact-controls: + # Software correctness only: no model, secrets, device or performance gates. + runs-on: ubuntu-22.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Install isolated artifact test dependencies + run: | + sudo apt-get update + sudo apt-get install --yes --no-install-recommends bubblewrap + sudo /usr/bin/python3 -m pip install "numpy>=1.24" + /usr/bin/python3 -m pip install "pytest>=7.4" "PyYAML>=6.0" + /usr/bin/python3 -I -c "import numpy; print(numpy.__version__)" + - name: Real NumPy sandbox and authored two-arm software controls + env: + INFERRSI_REQUIRE_BENCHMARK_SANDBOX: "1" + run: | + /usr/bin/python3 -m pytest tests/test_skill_eval_benchmark.py tests/test_skill_eval_benchmark_compare.py --junitxml=benchmark-controls.xml + - name: Retain software-control results including failures + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: benchmark-artifact-controls-${{ github.run_id }}-${{ github.run_attempt }} + path: benchmark-controls.xml + if-no-files-found: error + retention-days: 14 + kv-task-qualification: # Bounded software-control qualification; not a branch-protection or efficacy gate. if: github.event.pull_request.draft == false diff --git a/docs/architecture.md b/docs/architecture.md index 6589edcc9..8033dbdab 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,5 +1,20 @@ # Modular architecture +The [shared engineering benchmark adapter](engineering-benchmarks.md) lives in +`harness/benchmarks`, with task data in `data/engineering_benchmarks`. It binds +task-owned oracles and workloads to the existing Workbench runner and offline +SkillEval verifier contract. It adds neither a search controller nor another +execution platform. The current device pilot runs trusted affine RMSNorm on +Ascend through Workbench; the formal artifact sandbox remains CPU NumPy only. +Each task's actual qualification and limits are recorded separately. + +The [two-rank MoE scenario](moe-forward-benchmark.md) has its own distributed +task/grader contract under `data/engineering_scenarios` and reuses +`harness.scenarios.external.execute_trial`. Its paired, slowest-rank timing is +not the single-operation TaskPackage/SkillEval adapter. Historical Agent runs +and their published evidence remain distinct from the portable runner's current +software and device qualification. + The local experiment path now has concrete Harness, Model and Infra modules. The CLI, RunSpec, knowledge retrieval and existing command contracts remain stable. The diagrams below describe implemented code, with the historical compatibility diff --git a/docs/engineering-benchmarks.md b/docs/engineering-benchmarks.md new file mode 100644 index 000000000..08a9d435c --- /dev/null +++ b/docs/engineering-benchmarks.md @@ -0,0 +1,155 @@ +# Shared engineering benchmarks + +The `benchmarks` commands check and measure an implementation against a versioned +task package. Packages define semantics, inputs, correctness rules and timing; +the existing Workbench runs paired comparisons and retains evidence. Agent +execution, knowledge selection and optimization search remain caller-owned. + +For a complete distributed layer rather than a single-operation task, see the +[two-NPU MoE forward scenario](moe-forward-benchmark.md). Its local paired runner +uses the existing external executor with a distributed grader. It has a separate +task contract and is not accepted by the single-operation `benchmarks` adapter. +The [six-session MoE comparison](../evidence/moe-agent-comparison-20261104/README.md) +publishes historical measurements, candidate snapshots, token costs and curves; +it does not establish device validation of the newly packaged runner. + +| Task | Candidate entry point | Execution boundary | +| --- | --- | --- | +| [Ascend affine RMSNorm](../src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md) | `compute(x, weight, *, eps)` | Reviewed source on one caller-allocated NPU; no formal device sandbox | +| [CPU Softmax](../src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md) | `compute(x, *, eps)` | Reviewed NumPy source; a separate SkillEval artifact sandbox also supports this task | + +The task READMEs retain source attribution, licenses and workload limits. To +optimize an implementation, copy the task's `baseline.py` to a separate +`implementation.py`. The packaged `candidate.py` is a reference control, not an +Agent-produced optimization. Keep task, oracle and baseline fixed while changing +the implementation. Review source before using the trusted worker: content +hashes detect drift, not malicious code. + +## Run an NPU implementation + +Discovery imports neither torch nor torch_npu. Execution requires the target's +compatible installed torch/torch_npu/CANN stack; InferRSI does not install or +replace that runtime. Enter a caller-managed single-NPU allocation before any +device execution. The worker requires one `ASCEND_RT_VISIBLE_DEVICES` entry, +uses logical device 0 and never falls back to CPU. + +```bash +inferrsi benchmarks list +inferrsi benchmarks show rmsnorm-npu +inferrsi benchmarks check rmsnorm-npu --candidate --output-dir --timeout-seconds +inferrsi benchmarks compare rmsnorm-npu --candidate --repo --output-dir --repetitions --max-minutes --min-improvement-pct +``` + +RMSNorm uses contiguous 2D float32 inputs and nonuniform weights. Input and +weight generation, scalar double/math.fsum oracle computation and transfers +occur outside timing. The clock covers eager dispatch and completion +synchronization after an untimed pre-synchronization. It measures synchronized +eager-call latency, not pure kernel time. Shape, dtype, device, numerical values +and input/weight preservation are checked outside the timer for every call. + +Profiling is optional and separate from measurement: + +```bash +inferrsi benchmarks profile rmsnorm-npu --candidate --output-dir --timeout-seconds +``` + +This retains real device events in `npu-trace.json` and a path-free kernel +summary, with no performance verdict. Distinguish verification copies from +compute kernels. Review raw traces for private paths/identifiers before sharing. + +## Use the CPU software control + +Install the development checkout with `pip install -e ".[dev,benchmarks]"`. +Discovery and planning do not import NumPy; CPU execution requires it. + +```bash +inferrsi benchmarks show softmax-cpu +inferrsi benchmarks check softmax-cpu --candidate --output-dir --timeout-seconds +inferrsi benchmarks measure softmax-cpu --candidate --output-dir --timeout-seconds +inferrsi benchmarks grade softmax-cpu --candidate --output-dir --timeout-seconds +inferrsi benchmarks plan softmax-cpu --candidate --repo --output-dir --repetitions --max-minutes --min-improvement-pct +``` + +The Softmax contract reduces axis 1 and includes strided and boundary inputs. +The independent oracle uses scalar double precision and `math.fsum`. `check` +evaluates the base cases and the same varying-input sequence used by measurement. +Every timed input has distinct retained objects and bytes, so an earlier call +cannot prime an identity/content memo for a later sample. This public suite does +not prevent hostile implementations or establish held-out generalization. + +Input generation, oracle evaluation, validation, startup and warmups are outside +timing; allocations inside `compute` are included. The score sums per-case +medians for the task's selected workloads. Inspect individual cases as well as +the aggregate. The zero-input case checks correctness but contributes no timing +score. CPU `profile` retains `profile.txt`, not a device timeline or speed score. + +## Comparison and retained results + +Replace `plan` with `compare` to execute through Workbench. `plan` prints a +RunSpec; `compare` constructs its own RunSpec rather than consuming a previously +printed one. Freeze the actual inputs and gates before candidate search using +the [acceptance procedure](contributing.md#acceptance-checklist). All placeholders +above come from that experiment, not repository defaults. Single-action output +directories must be new. + +`TaskPackage` freezes task and grader identities. Commands also bind candidate +bytes and runtime fingerprints, checked before and after worker execution. The +Git checkout supplied by `--repo` identifies source context; task and candidate +files have additional hashes. Runtime fingerprints alone do not establish an +uncontended host, equivalent device/CANN installations or native dependencies. + +Results retain the planned `input_sequence`, actual `executed_calls` and sample +input hashes. Failed candidates may execute only a prefix. Wrong outputs produce +no usable performance metric. Preserve unsuccessful trials and inspect +`execution.json`, per-trial `result.json`, `evaluation.json`, `status.json` and +source-integrity records. + +`compare` returns `run_dir` and the retained status. Its exit codes are PASS = 0, +FAIL = 1 and INCONCLUSIVE = 2. A complete comparison missing its gain gate is +FAIL. A wrong candidate preventing measurement leaves the comparison +INCONCLUSIVE, while independent artifact correctness can be FAIL. Exceptions, +missing dependencies, drift and timeouts must not become successful comparisons. +Workbench statistics are descriptive; these results do not establish service +gains, statistical significance or Agent/knowledge efficacy. + +`grade` returns artifact-correctness PASS/FAIL/INCONCLUSIVE with SkillEval's +`passed`/`assertions` structure. The lower-level +`benchmarks.skill_verifier.task_descriptor` binds an already reviewed artifact +for `skill_evaluation.run_trial`; its answer supplies `candidate_sha256`. +For actual CPU file handoff through `tools.skill_eval_compare`, see +[engineering task artifacts](engineering-task-artifacts.md). That formal adapter +checks correctness only and rejects NPU tasks. It does not automatically send +generated NPU code to this trusted worker. Provider generation mode and the +offline harness-improvement controller do not yet support this task adapter. + +## Qualify the tools with authored controls + +The NPU utility exercises correct, wrong, replay, same-code and reference +implementations through the worker and Workbench: + +```bash +python -m tools.npu_benchmark_validate --repo --output-dir --repetitions --min-improvement-pct --max-aa-difference-pct --max-minutes --timeout-seconds +``` + +It freezes policy and implementation identities before execution, retains all +samples and checks actual device/CANN/runtime consistency. The utility includes +profiling as a tool control; this does not make profiling mandatory for ordinary +optimization. `--timeout-seconds` caps its standalone actions, while paired +Workbench trials use the comparison deadline. It does not enforce a separate +whole-utility budget or the standalone cap on each paired trial. Establish the +required outer allocation/process deadlines and verify cleanup before relying +on those budget boundaries. + +The CPU integration utility dynamically selects the packaged NumPy task: + +```bash +python -m tools.engineering_benchmark_validate --repo --output-dir --repetitions +``` + +It retains profiles and AB/BA samples and runs the existing SkillEval trial path +with correct and wrong authored controls. Neither utility generates Agent +candidates. Their reports separate software controls from campaign acceptance. +Preserve experiment plans and original results with their tested revisions using +the [evidence conventions](contributing.md#evidence-and-claims); review/redact +copies of raw logs before publication. A successful control run does not finish +the Agent-to-NPU handoff or prove an optimization benefit. diff --git a/docs/engineering-task-artifacts.md b/docs/engineering-task-artifacts.md new file mode 100644 index 000000000..75f66063c --- /dev/null +++ b/docs/engineering-task-artifacts.md @@ -0,0 +1,115 @@ +# Engineering task artifacts in SkillEval + +`engineering-benchmark-v1` connects the packaged `softmax-cpu` task to +`tools.skill_eval_compare` in **update mode**. It freezes two skill versions, +retains each producer's actual `implementation.py`, and grades correctness with +the shared task oracle. It does not measure speed. Reviewed implementations can +later enter a separate [Workbench comparison](engineering-benchmarks.md). + +## Prepare the task and comparison + +Run from a development checkout. Generate a canonical case rather than editing +its contract or relying on a same-named directory in the working tree: + +```bash +python -c "from inferrsi.harness.skill_eval_benchmark import task_descriptor; from inferrsi.infra.artifacts import write_json; from pathlib import Path; write_json(Path('cases.json'), [task_descriptor('softmax-cpu')])" +``` + +Create `config.json` beside `cases.json`: + +```json +{ + "adapter": "engineering-benchmark-v1", + "mode": "update", + "skill": "", + "skill_root": "", + "cases": "cases.json", + "snapshots": { + "baseline": {"git_revision": ""}, + "candidate": {"git_revision": ""} + }, + "model": {"name": "", "effort": ""}, + "limits": { + "repeats": 1, + "max_task_calls": 2, + "timeout_seconds": 60, + "campaign_seconds": 180, + "token_stop_new_trials": 10000 + } +} +``` + +The example limits are illustrative; choose and freeze the actual bounds before +execution. Use full commits available in the local repository. Do not add a +provider client, custom schema or grader. See [comparison configuration](skill-eval-comparison.md) +for Git prerequisites, snapshot rules and budget semantics. + +```bash +python -m tools.skill_eval_compare prepare --config config.json --repo --output +``` + +Preparation freezes task, starter, schema, verifier/controllers and skill +bundles. Formal loading always uses canonical packaged resources. Retained task +copies must match that descriptor. The supplied CPU starter is already correct; +leaving it unchanged can pass, so correctness alone cannot identify optimization. + +## Hand off and retain actual files + +Stage a fresh workspace outside the evidence tree for one arm: + +```bash +python -m tools.skill_eval_compare execute --output --authorize --stage-local --task-id softmax-cpu --condition baseline --repeat 1 --current-head +``` + +Stage does not launch an Agent. Run the producer under the caller's established +permissions and budget. Only `implementation.py` may change; keep task, +instructions and assigned knowledge frozen. The file implements +`compute(x, *, eps)` with self-contained NumPy/math helpers. Its final answer is: + +```json +{"artifact": "implementation.py", "summary": "Describe the actual change"} +``` + +Stop the producer before capture. Retain `stdout.log`, `stderr.log` and +`execution.json` in a separate submission directory, following the exact fields +in [author-run handoff](local-skill-evaluation.md#author-run-handoff). +Model, access, resources and usage in these files are author-reported, not +authenticated observations. Missing usage stays unknown. + +```bash +python -m tools.skill_eval_compare execute --output --authorize --capture-local --submission --task-id softmax-cpu --condition baseline --repeat 1 --current-head +``` + +Repeat stage/produce/capture for `--condition candidate` with distinct workspace +and log directories, then run: + +```bash +python -m tools.skill_eval_compare report --output +``` + +The plan hash binds this attempt; it does not authorize model spending. Changed +inputs require a new plan. Capture preserves actual source bytes and hashes; +report rechecks retained artifacts without rerunning the producer. Failed or +incomplete attempts remain recorded and cannot be overwritten as clean results. + +## Execution boundary and interpretation + +The trusted parent owns inputs, the scalar oracle and assertions. Submitted +source executes in a Linux bubblewrap child with system Python, matching NumPy +and a bounded runner; the repository, grader and other arm are not mounted. +Every returned array is checked by the parent. The public suite does not prove +hidden-test generalization, hostile-protocol resistance or independent tracing +of every candidate operation. Producer permissions are a separate boundary. + +Windows, missing bubblewrap/namespace support or an incompatible NumPy runtime +produce `artifact_sandbox_unavailable` / INCONCLUSIVE without host fallback. +NPU tasks are explicitly rejected. CPU replay qualification does not establish +NPU isolation, and the default unrestricted Agent client is refused for automatic +engineering runs. Provider generation and the offline improvement controller do +not support this adapter. + +A PASS means correctness on the public input sequence, not faster code. A local +handoff report showing `artifact_improved` establishes a difference in retained +artifact checks, not a causal skill benefit or verified total cost. Performance, +Agent efficacy and service claims need their own controlled evidence. Profiling +is optional and is not invoked by this adapter. diff --git a/docs/moe-forward-benchmark.md b/docs/moe-forward-benchmark.md new file mode 100644 index 000000000..2aa80cb45 --- /dev/null +++ b/docs/moe-forward-benchmark.md @@ -0,0 +1,124 @@ +# 双 NPU MoE forward：任务与独立评测入口 + +这道题让 Agent 优化一层真实结构的计算：路由 → 跨卡派发 → 本地专家 MLP → 合并结果， +加一个共享专家。Agent 交付 `compute(x, params, *, config, group)` 的 Python 实现； +评测入口检查两张卡的输出，再在相同设备上交错测量基线和候选。 +它提供一次候选的事实反馈，调用方或 Agent 根据反馈决定下一次改动。 + +任务允许研究路由和重排开销、融合、冗余计算、通信组织及同步消除； +例如比较 AllGather 和 AllToAll 派发，但某一种优化并不保证有效。 +完整定义在 [TASK.md](../src/inferrsi/data/engineering_scenarios/moe-forward-npu/TASK.md)， +固定形状和误差容差在 [task.json](../src/inferrsi/data/engineering_scenarios/moe-forward-npu/task.json)。 +题目参考所引用版本的 vLLM-Ascend MoE 结构，代码独立编写，按 Apache-2.0 分发。 +它不包含完整 vLLM 服务，也不能据此声称整模型或服务获得相同加速。 + +## 怎么运行 + +需要已经配置好的 Linux、PyTorch、torch_npu、CANN/HCCL 和两张可用 Ascend NPU。 +在该环境安装当前 InferRSI checkout（例如 `python -m pip install -e .`）； +入口不安装或修改共享设备软件。候选是一个经审查的自包含 Python 文件， +可以导入当前环境已有的运行库。可先复制题目内 `baseline.py` 作为候选，验证接线。 + +由调用方申请两张空闲设备，不固定卡号。例如在仓库根目录： + +```bash +npu run --gpus 2 --timeout 20m --wait 30m -- \ + python -m tools.moe_benchmark \ + --candidate ./implementation.py \ + --output-dir ./outputs/moe/round-01 \ + --pairs 2 --timeout-seconds 900 +``` + +`npu run` 是环境已有的分配器，不由 InferRSI 安装。它必须在子进程中设置 +`ASCEND_RT_VISIBLE_DEVICES` 为两张不同的已分配卡。入口将它们映射为逻辑 rank 0/1， +用本机 `torch.distributed.run --standalone --nproc-per-node=2` 启动 HCCL。 +分配等待在入口计时之外；分配器超时应为本次评测的超时和清理留出余量。 +没有该分配器时，也可在已经获得的两卡分配内直接执行相同 Python 命令。 + +安装 wheel 后，无需 `tools/` checkout，可用同一参数运行： + +```bash +python -m inferrsi.harness.scenarios.moe \ + --candidate ./implementation.py --output-dir ./outputs/moe/round-01 \ + --pairs 2 --timeout-seconds 900 +``` + +每个候选必须使用**新的输出目录**，已有目录会拒绝覆盖。`--pairs` 和 +`--timeout-seconds` 都必须显式指定；两组配对为每实现每性能 case 16 个计时样本， +四组为 32 个。超时包括入口准备和执行，之后最多有额外的有界进程清理等待。 +入口先冻结题目、候选和 grader 的副本及 SHA256，再通过现有 +`harness.scenarios.external.execute_trial` 执行。它额外清理本次唯一环境标记的 +torchrun 子进程，覆盖子进程创建独立 session 的情况；不会按宽泛进程名杀进程。 + +## 每个结果代表什么 + +| 产物 | 用途 | +| --- | --- | +| `result.json` | 本次权威结果：状态、正确性、可用加速比、执行时间、清理结果 | +| `identity.json`、`task/`、`candidate/`、`runner/`、`request.json` | 本次输入、代码与采样次数的冻结副本和哈希 | +| `results/rank-0.json`、`results/rank-1.json` | 每次正确性、预热、正式调用、输入哈希、两卡耗时和准备开销 | +| `results/summary.json` | rank 0 的原始汇总，必须经过外层完整性检查才可用 | +| `execution/` | 通用执行器记录、原始结果、stdout 和 stderr | + +只在命令成功退出、两 rank 完整通过、全部预期调用和样本齐全、汇总可从原始样本重算、 +代码未变化且清理完成时，`result.json` 才包含可用 `speedup`。 +候选导入/执行失败、候选超时或输出错误记为 `candidate_failure / FAIL`， +`round_consumed=true`，不当作免费的基础设施重试。 +基线、设备初始化、证据完整性或清理失败记为 `infrastructure_failure / INCONCLUSIVE`。 +若已经观察到候选执行，随后发生基础设施或清理失败，仍保留 `round_consumed=true`； +已识别的候选失败也不会被后续清理问题改写为免费重试。清理未确认时应停止后续运行。 +失败均无有效速度分数；原始日志中即使曾出现局部 PASS 或速度也不能替代最终结果。 +`round_consumed=false` 只表示未发现候选执行记录；证据不完整时，不能据此认定可以免费重试。 +退出码分别为 0（完成）、2（候选失败）、1（基础设施或参数失败）。 + +每个样本取两 rank 的完成时间最大值；每个 case 取 baseline/candidate 中位数之比； +最终 `speedup` 是三个 case 加速比的几何平均。 +`metrics.suite_latency_ms` 是三个候选中位数之和，是供现有执行器读取的单独指标， +不应拿它代替上述几何平均加速比。 + +计时覆盖完整 forward 的 host 调度、路由、重排、通信、MLP 和设备完成。 +CPU 输入/oracle 准备、计时前 barrier、验收、形状预热不在 forward 分数中，开销另存。 +预热使用数值和输出都不同、每 token 专家集合相同的输入；正式样本输入持续变化， +每次都验算并清零输出。首次调用包含候选自己的权重排布/编译等准备，但同进程共享 +底层库缓存，因此不是两者各自在全新进程中的冷启动对照。 +峰值显存是 baseline/candidate **共享进程峰值**，不是候选单独峰值。 + +## 在 RSI 优化循环里如何使用 + +```text +用户目标与题目约束 → Agent 改候选 → 本入口验算并测量 → Agent 看证据再修改 + ↓ + 调用方保留每轮结果、选最佳版本、独立复测 +``` + +调用一次只评一个候选，不启动 Agent、不自动执行若干轮、不统计模型 tokens， +也不选择搜索策略。这是一个独立的分布式 scenario；当前不能把它的 ID 传给 +单算子 `inferrsi benchmarks list/show/check/compare` 或当作已经接好的 SkillEval adapter。 +准备了其他 Agent/RSI 执行方式的调用方可调用此命令，把 `result.json` 作为反馈。 +知识库 ON/OFF 或不同 Agent 的差异由调用方控制，测量协议保持一致。 +profiling 可作为 Agent 诊断工具，评测入口不要求先 profiling。 + +可给新 Agent 的简短任务指令： + +> 阅读 `src/inferrsi/data/engineering_scenarios/moe-forward-npu/TASK.md`， +> 优化 `./implementation.py` 的完整双 NPU MoE forward。遵守输入只读、专家所有权、 +> 正确性和计时边界。每次候选使用新的输出目录，按已约定的配对数、超时和实验预算， +> 通过 `python -m tools.moe_benchmark` 取得结果；保留失败，不以自报数字替代结果。 +> 最终交付最佳候选、逐轮证据和独立复测记录。 + +例如要做 8 轮再复测，由调用方为 R1…R8 和 final 分配不同目录并保存候选快照。 +轮数是该次实验的预算，不能把它当成通用默认值。优化曲线按每轮有效分数及历史最佳分数 +绘制；失败轮保留空分数并消耗预算。累计 tokens 需从对应 Agent/session 的原始 usage +记录汇总；本入口的 `elapsed_seconds` 是评测墙钟，不是 Agent 搜索墙钟或模型费用。 + +## 验证范围与限制 + +历史冻结题目、baseline、oracle、采样逻辑已有双 NPU 实验；本次发布保留了这些数学和 +计时语义，加入便携包装、源码冻结、双 rank 证据复核和独立 session 清理。 +**新增的便携入口尚未在 NPU 上重新跑通**，CPU 测试和 wheel 资源检查只验证接线、 +失败处理及证据完整性，不能当作新入口的 NPU 验收或性能结论。 + +当前权重更新用例验证原位更新后的结果，但两次访问之间还运行其他参数用例； +它没有穷尽“相同缓存项命中后立即原位更新”等所有缓存失效路径。 +候选源码需要审查；输入/oracle 隔离是协议边界，不是恶意代码沙箱。 +本题与历史实验都不能证明所有 MoE 形状、硬件或服务负载上的收益。 diff --git a/docs/operations.md b/docs/operations.md index 9614db4f2..10deb12d9 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,5 +1,15 @@ # Operation guide +Shared engineering tasks use `inferrsi benchmarks list|show|plan` for read-only +inspection and `check|measure|profile|grade|compare` for explicit local execution. +See the [engineering benchmark contract](engineering-benchmarks.md) for trusted +source requirements, task identities, Ascend/CPU coverage and offline SkillEval reuse. + +The separate [two-NPU MoE forward scenario](moe-forward-benchmark.md) uses +`python -m tools.moe_benchmark` inside a caller-managed two-device allocation. +It compares a supplied implementation with the task baseline through the +existing external executor; it does not launch an Agent or retrieve knowledge. + One concise map from an agent's intent to the supported entry point. This guide is navigation only: detailed contracts stay with their owning documents, and it is not a third primary matrix. The two authoritative matrices remain the diff --git a/docs/skill-eval-comparison.md b/docs/skill-eval-comparison.md index 41b0aa4c9..3736bce86 100644 --- a/docs/skill-eval-comparison.md +++ b/docs/skill-eval-comparison.md @@ -10,6 +10,15 @@ authorization and retained-result publication around these interfaces. The [offline improvement MVP](harness-improvement.md) reuses the same interfaces for bounded INV/DOM probes, successor rerun and a one-shot held-out gate. +Shared Softmax CPU artifacts use the `engineering-benchmark-v1` adapter +in **update mode**, with the same prepare/execute/report and local stage/capture +interfaces. The trusted parent grades output from a Linux bubblewrap NumPy +worker; unsupported sandbox/runtime leaves INCONCLUSIVE without host fallback. +This is correctness-only artifact evaluation, not timing or skill-efficacy +qualification. See the [task contract and handoff example](engineering-task-artifacts.md). +The provider generation mode and offline improvement controller do not support +this adapter yet. + ```mermaid flowchart LR A["New: small JSON config"] --> B["New: fixed Git snapshots"] diff --git a/evidence/moe-agent-comparison-20261104/README.md b/evidence/moe-agent-comparison-20261104/README.md new file mode 100644 index 000000000..cbb4b1223 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/README.md @@ -0,0 +1,99 @@ +# 六组 Agent 的双 NPU MoE 优化结果 + +本目录发布已完成历史实验的**脱敏字段提取、源码快照与图**，对应 +[需求 #252](https://github.com/InferMatrixLab/InferRSI/issues/252) 和 +[PR #338](https://github.com/InferMatrixLab/InferRSI/pull/338)。六组依次执行， +每组一个全新会话、8 轮探索、一次独立 final：共 **48 轮探索（46 PASS、2 FAIL）**， +**6 次 final 全部 PASS**。PASS 的范围是冻结任务下的正确性与有效测量， +不代表生产服务验收或知识库效果的因果结论。 + +**历史实验直接调用既有 `inferrsi.harness.scenarios.external.execute_trial`， +没有使用 PR 新增的 TaskPackage 入口。** 本次发布不重新运行设备；不能用这些 +历史成绩声称新入口已经完成 NPU 验证。 + +任务是一层完整、两卡 EP=2 的 MoE forward：8 个路由专家（每卡 4 个）加共享 +专家，涵盖路由、重排、专家 MLP、归并和跨卡通信。基线是本题独立编写的可读 +AllGather/ReduceScatter 实现，不是 vLLM 最新生产实现。三类性能场景是 +64/256/1024 token 每卡、hidden 512/1024/2048，另有正确性边界用例。 +完整数学语义、冻结门槛及参考来源见 [TASK.md](validator/task/TASK.md)。 + +| 模型（均 xhigh） | 知识库 | 入选轮次 | 独立 final 加速比 | 8 轮累计 token | final 增量 | 报告尾部 | 完整会话 token | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | +| GPT-6.1 Sol | ON | R5 | 2.216346× | 8,131,818 | 1,673,613 | 577,259 | 10,382,690 | +| GPT-6.1 Sol | OFF | R6 | 2.380696× | 8,202,212 | 1,665,783 | 533,633 | 10,401,628 | +| GPT-5.6 Sol | ON | R8 | 1.092386× | 13,016,364 | 1,716,556 | 3,155,714 | 17,888,634 | +| GPT-5.6 Sol | OFF | R8 | 1.207114× | 4,904,359 | 1,151,107 | 1,238,744 | 7,294,210 | +| GPT-6 Astra | ON | R7 | 2.367067× | 19,551,765 | 2,678,476 | 950,283 | 23,180,524 | +| GPT-6 Astra | OFF | R3 | 2.339809× | 7,774,716 | 2,039,682 | 600,255 | 10,414,653 | + +总计 **79,562,339 token**。token 是实际请求 input+output 的累计数，已经包含 +cached input；reasoning 是 output 的子集，不能再加一次。它不是货币账单。 +每轮边界是完成评测的工具结果返回之后、下一条模型生成之前的延迟遥测快照， +包含此前检索、失败、等待交互等全部会话工作；final 与报告尾部单列。 + +![六组优化进度与累计 token](results/comparison-overview.png) + +阶梯线是截至当轮保留的最佳有效分数，慢轮和失败仍计成本；菱形是独立 final， +不回填探索曲线。两组 Astra 的 R4 都因当前 `npu_moe_finalize_routing` 不接受 +`k` 参数而在候选正确性调用中失败，没有正式候选计时样本；分数为 null， +不是 0× 或 1×。两组均在下一轮删除该参数后通过。 + +**最终实现及搜索差异。** 六组均省去独立 logits AllGather，收集 BF16 输入后 +重算复制 router；最终均保留 AllGather/ReduceScatter，没有交付 All2All 或 +自定义 Triton kernel。6.1 两组采用原生路由、两次 grouped GEMM 与 FP32 finalize； +ON 回选 R5，仍有路由 count 主机读取；OFF R6 使用静态容量、设备端专家前缀、 +显式清零无效尾部，并保留 shared 与异步 ReduceScatter 的交错。 +5.6 ON 保留逐专家循环，OFF 使用批量 BMM 与 one-hot 前缀打包。 +Astra ON 采用原生路由/GMM/finalize，仍读取有效计数；OFF 回选静态容量路由、 +设备前缀 GMM、手工 FP32 combine 的 R3。详细轨迹见 [逐轮改动](results/changes.md)。 + +**能解释到哪里。** 每条件只有一条轨迹。OFF 仍使用 RSI 的执行、验收与证据链路， +只是禁用优化知识库正文；两边均允许公开技术资料。ON 确实增加了检索/阅读， +并有采纳记录，但没有逐份知识的消融实验。失败查询、材料宽泛、具体 API 从 +公开资料获取，以及固定 8 轮下的搜索方向选择，都是观察到的过程；不能把 +最终高低直接归因于知识库好坏。6.1 的两组总 token 实际基本相同。 + +6.1 final 的 2.381× 对 2.216×是**各自配对基线下的分数**，不是 OFF 代码直接 +快 7.4%。两组 final 基线延迟几何平均相差约 6.54%；跨时段候选原始延迟几何 +平均只差约 0.82%，且 medium 场景 OFF 更慢。这也不是控制时段后的直接对照。 +异步安排和显式 wait 可从源码确认，实际隐藏多少通信时间没有独立时间线证明。 + +Astra ON 累计排队 **2,485.094 秒（41.42 分钟）**，OFF 约 1.280 秒；等待中的 +模型交互 token 全部保留。共享机时段和物理卡不同，虽是相同型号/软件且使用 +配对 baseline，也不能声称运行条件完全相同。分配进程墙钟包含初始化、CPU +oracle、预热、验证和计时，不是 NPU 忙时。公开资料允许访问但未全面冻结； +Astra 两组取到的上游提交不同，不过三份共同 API 文档的保存内容哈希相同。 + +**公开文件与复核入口。** + +- [条件协议及来源](plan/README.md)、[脱敏计划](plan/publication.json)、 + [原件身份与省略范围](plan/provenance.json)。 +- [54 次紧凑原始计时提取](results/measurements.jsonl.gz)：52 次成功的逐 case + baseline/candidate 样本数组，2 次失败保留 null；数值未舍入。这里的样本已经 + 按两个 rank 的较慢值聚合，不是完整逐 rank 原日志。 +- [逐轮分数与 token](results/rounds.csv)、[会话成本](results/session-usage.csv)、 + [图所用摘要](results/summary.json)、[审计字段提取](results/audit-extract.json)。 +- [候选清单](patch/candidates.csv)：48 份探索源码在确定性 ZIP 中，6 份最终源码 + 直接可读；均保留原字节和 SHA256，包括失败、较慢及未入选候选。 +- [冻结裁判说明](validator/README.md)、[完整 bundle 哈希](plan/integrity.json)。 + +从仓库根执行 CPU 复核，无须 NPU、模型调用或私有日志： + +```bash +python evidence/moe-agent-comparison-20261104/verify.py +python tools/validate_evidence.py +``` + +重新绘制同一组数值（需要 Matplotlib；便携脚本用英文，不依赖 CJK 字体）： + +```bash +python evidence/moe-agent-comparison-20261104/render_overview.py --output /tmp/moe-overview +``` + +输出目录可替换为本机临时目录。中文原 PNG 直接复制自历史结果，重新绘制不覆盖它； +便携版保留相同三图与数值，英文文案、字体和 Matplotlib 版本会改变像素布局。 +生成脚本仅使用本目录公开摘要。 +`build_public.py --source --output ` 可由有权 +访问原始归档的人重新提取数据；该归档未公开，普通读者无法独立重放完整条件/ +会话访问审计。CPU 复核证明公开数据内部一致、可用字节及统计可重算，不等于 +重新测量性能，也不把外部审计原件变成已公开可验证材料。 diff --git a/evidence/moe-agent-comparison-20261104/build_public.py b/evidence/moe-agent-comparison-20261104/build_public.py new file mode 100644 index 000000000..52a95fe5e --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/build_public.py @@ -0,0 +1,328 @@ +"""Extract the historical campaign without running an agent or a device. + +The source is a private, already audited archive. This script selects public +fields; it does not sanitize arbitrary logs. Review source text before sharing. +""" + +import argparse +import csv +import gzip +import hashlib +import json +import math +import shutil +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +OBS_FIELDS = ( + "role round_index candidate_sha256 task_sha256 grader_sha256 source_revision " + "source_bundle_sha256 correctness valid_measurement aggregation speedup " + "effective_speedup best_exploration_speedup cleanup_verified queue_seconds " + "token_boundary_status cumulative_input_tokens cumulative_cached_input_tokens " + "cumulative_output_tokens cumulative_total_tokens cumulative_uncached_input_tokens " + "cumulative_reasoning_output_tokens round_input_tokens round_cached_input_tokens " + "round_output_tokens round_total_tokens round_uncached_input_tokens" +).split() +USAGE_FIELDS = ( + "request_records input_tokens cached_input_tokens output_tokens total_tokens " + "uncached_input_tokens reasoning_output_tokens monetary_cost" +).split() +AUDIT_FIELDS = ( + "candidate_outcome expected_calls_per_rank timed_samples_per_label_per_case " + "max_rank_samples_checked pair_geometric_speedups audit_scope audit_status " + "frozen_source_files_verified cleanup_verified" +).split() +RANK_FIELDS = ( + "rank calls timed_samples local_failed_calls max_errors_by_label " + "minimum_warm_input_rms_difference minimum_warm_output_nrmse_difference" +).split() + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def read(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def select(obj, fields): + return {key: obj[key] for key in fields if key in obj} + + +def write(path, data): + path.write_text( + json.dumps(data, ensure_ascii=False, separators=(",", ":")) + "\n", + encoding="utf-8", + newline="\n", + ) + + +def csv_write(path, rows): + with path.open("w", encoding="utf-8", newline="") as stream: + writer = csv.DictWriter(stream, fieldnames=list(rows[0]), lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + + +def build(source, target): + source, target = source.resolve(), target.resolve() + if target.is_relative_to(source) or source.is_relative_to(target): + raise ValueError("Source archive and publication destination must not overlap") + for part in ("plan", "results", "patch/final", "validator/task", "validator/runner"): + (target / part).mkdir(parents=True, exist_ok=True) + original = read(source / "results/summary.json") + plan = read(source / "common/plan.json") + extension = read(source / "extension-plan.json") + assert original["campaign_complete"] and original["condition_comparison_verified"] + assert len(original["sessions"]) == 6 + public = select( + original, + ( + "schema_version", + "campaign_id", + "expected_session_count", + "campaign_complete", + "condition_comparison_verified", + ), + ) + public["publication_type"] = "redacted field extract; not the original summary" + public["sessions"] = [] + measurements, rows, usage_rows, candidates, audits = [], [], [], [], [] + originals = {} + for name in ( + "results/summary.json", + "common/plan.json", + "extension-plan.json", + "results/comparison-overview.png", + "analysis/render_overview.py", + ): + originals[name] = { + "sha256": digest(source / name), + "bytes": (source / name).stat().st_size, + "availability": "external original; no private paths published", + } + originals["results/comparison-overview.png"]["availability"] = ( + "published byte-identical copy at results/comparison-overview.png" + ) + archive_path = target / "patch/exploration-candidates.zip" + with zipfile.ZipFile( + archive_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=9 + ) as archive: + for index, session in enumerate(original["sessions"]): + key = session["key"] + audit_path = source / f"analysis/raw-audit-{index}.json" + raw_audit = read(audit_path) + assert raw_audit["status"] == "PASS" and raw_audit["acceptance_ready"] + condition_audit = read(source / f"analysis/condition-audit-{index}.json") + assert condition_audit["status"] == "verified" + for kind, suffix in ( + ("raw-audit", "json"), + ("condition-audit", "json"), + ("source-audit", "md"), + ): + name = f"analysis/{kind}-{index}.{suffix}" + originals[name] = { + "sha256": digest(source / name), + "availability": "external audit original", + } + s = select( + session, + ( + "key", + "label", + "model", + "knowledge", + "status", + "rounds_completed", + "rounds_expected", + "validation_status", + "best_exploration_speedup", + "final_speedup", + "post_measurement_reporting_tokens", + ), + ) + s["reasoning_effort"] = session["usage"]["efforts"] + s["usage"] = select(session["usage"], USAGE_FIELDS) + s["queue_seconds"] = session["recorded_queue_seconds"] + s["allocation_process_seconds"] = session["recorded_device_active_seconds"] + s["observations"] = [] + for observation in session["observations"]: + original_path = Path(observation["observation_path"]) + actual = read(original_path) + remote_path = original_path.parent / "remote/results/summary.json" + remote_summary = read(remote_path) if remote_path.exists() else {} + assert remote_path.exists() or observation["correctness"] == "FAIL" + assert observation["cases"] == actual.get("cases") == remote_summary.get("cases") + assert ( + observation["speedup"] is None and actual.get("speedup") is None + ) or math.isclose(observation["speedup"], actual["speedup"], rel_tol=1e-12) + implementation = original_path.parent / "implementation.py" + assert digest(implementation) == observation["candidate_sha256"] + o = select(observation, OBS_FIELDS) + o["allocation_process_seconds"] = observation["device_active_seconds"] + o["original_observation_sha256"] = digest(original_path) + o["original_device_summary_sha256"] = ( + digest(remote_path) if remote_path.exists() else None + ) + o["attempt"] = actual["attempt"] + o["round_consumed"] = actual["round_consumed"] + o["execution_status"] = actual["status"] + o["performance_verdict"] = "PASS" if o["valid_measurement"] else "INCONCLUSIVE" + o["failure_reason"] = ( + None + if o["correctness"] == "PASS" + else ( + "Candidate rejected on both ranks before timing: installed " + "npu_moe_finalize_routing does not accept keyword argument 'k'." + ) + ) + s["observations"].append(o) + measurements.append({"session_key": key, **o, "cases": observation["cases"]}) + rows.append({"session_key": key, **o}) + audit = next( + a + for a in raw_audit["attempts"] + if a["role"] == o["role"] and a["round_index"] == o["round_index"] + ) + audits.append( + { + "session_key": key, + "role": o["role"], + "round_index": o["round_index"], + **select(audit, AUDIT_FIELDS), + "ranks": [select(rank, RANK_FIELDS) for rank in audit["ranks"]], + } + ) + if o["role"] == "exploratory": + name = f"{key}/round-{o['round_index']:02d}.py" + entry = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + entry.compress_type = zipfile.ZIP_DEFLATED + entry.external_attr = 0o644 << 16 + archive.writestr(entry, implementation.read_bytes()) + location = f"patch/exploration-candidates.zip!{name}" + else: + location = f"patch/final/{key}.py" + shutil.copyfile(implementation, target / location) + s["selected_round"] = next( + r["round_index"] + for r in s["observations"] + if r["role"] == "exploratory" + and r["candidate_sha256"] == o["candidate_sha256"] + ) + s["final_candidate_sha256"] = o["candidate_sha256"] + candidates.append( + { + "session_key": key, + "role": o["role"], + "round_index": o["round_index"], + "file": location, + "sha256": digest(implementation), + "bytes": implementation.stat().st_size, + } + ) + public["sessions"].append(s) + usage_rows.append( + { + "session_key": key, + "model": s["model"], + "knowledge": s["knowledge"], + "exploration_tokens": s["observations"][7]["cumulative_total_tokens"], + "final_stage_tokens": s["observations"][8]["round_total_tokens"], + "report_tail_tokens": s["post_measurement_reporting_tokens"], + **s["usage"], + "queue_seconds": s["queue_seconds"], + "allocation_process_seconds": s["allocation_process_seconds"], + } + ) + write(target / "results/summary.json", public) + with (target / "results/measurements.jsonl.gz").open("wb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as stream: + for measurement in measurements: + stream.write((json.dumps(measurement, separators=(",", ":")) + "\n").encode()) + csv_write(target / "results/rounds.csv", rows) + csv_write(target / "results/session-usage.csv", usage_rows) + csv_write(target / "patch/candidates.csv", candidates) + write( + target / "results/audit-extract.json", + {"type": "selected fields from original audits", "attempts": audits}, + ) + public_plan = select( + plan, + ( + "schema_version campaign_id status frozen_at_utc task_id task_contract qualification " + "rounds_per_session sessions execution_order round_definition final_definition " + "source_revision source_hashes bundle_lock_sha256 execution_adapter " + "runtime_observed measurement tolerance knowledge_sources_on common_sources " + "knowledge_access_isolation forbidden_cross_session_sources " + "collection_recovery token_accounting resource_accounting inference_limit" + ).split(), + ) + public_plan["publication_type"] = "redacted field extract; private launch configuration omitted" + public_plan["allocation"] = select( + plan["allocation"], + ( + "gpus", + "automatic", + "inner_timeout_seconds", + "allocation_timeout_seconds", + "queue_bound_seconds", + ), + ) + public_plan["extension"] = select( + extension, + ( + "registered_at_utc base_plan_sha256 base_source_revision expected_session_count " + "existing_session_keys rounds_per_session final_remeasurement execution_order " + "unchanged prompt_change isolation interpretation" + ).split(), + ) + public_plan["extension"]["additional_conditions"] = [ + select(c, ("key", "label", "model", "thinking", "knowledge")) + for c in extension["additional_conditions"] + ] + write(target / "plan/publication.json", public_plan) + write( + target / "plan/provenance.json", + { + "original_artifacts": originals, + "omissions": [ + "Private connection settings, personal paths and identities, " + "chat identifiers, raw session logs", + "Raw rank calls and full original audits remain outside this repository", + "Host-specific launchers and frozen dependency bytes " + "are identified by source hashes only", + ], + "extracts_have_separate_hashes": True, + }, + ) + for name in ("baseline.py", "oracle.py", "task.json", "TASK.md", "workload.py"): + src = source / "common/task" / name + assert digest(src) == plan["source_hashes"][f"task/{name}"] + shutil.copyfile(src, target / "validator/task" / name) + grader = source / "common/runner/grade.py" + assert digest(grader) == plan["source_hashes"]["runner/grade.py"] + shutil.copyfile(grader, target / "validator/runner/grade.py") + shutil.copyfile( + source / "results/comparison-overview.png", target / "results/comparison-overview.png" + ) + files = { + p.relative_to(target).as_posix(): digest(p) + for p in sorted(target.rglob("*")) + if p.is_file() and "__pycache__" not in p.parts and p.name != "integrity.json" + } + write(target / "plan/integrity.json", files) + print( + f"Extracted {len(measurements)} attempts and {len(candidates)} immutable source identities." + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--source", required=True, type=Path, help="Authorized local campaign archive" + ) + parser.add_argument("--output", type=Path, default=ROOT, help="Public bundle destination") + args = parser.parse_args() + build(args.source, args.output) diff --git a/evidence/moe-agent-comparison-20261104/patch/README.md b/evidence/moe-agent-comparison-20261104/patch/README.md new file mode 100644 index 000000000..d25286ee0 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/patch/README.md @@ -0,0 +1,14 @@ +# Immutable candidate sources + +`candidates.csv` identifies all 54 submitted sources: 48 exploratory candidates +inside `exploration-candidates.zip` and six readable final sources in `final/`. +The ZIP contains only relative source names, with fixed timestamps; each member +retains its original bytes. SHA256 and byte counts are checked by `../verify.py`. +Final sources equal the selected exploratory round byte for byte. Failed and +slower candidates remain included. These snapshots are historical evidence, +not a dependency of the current library or a request to execute their code. + +Some inherited source comments say padding does not execute an expert MLP. +After native routing this is not necessarily true: padded slots can enter the +GMM, but their probabilities are zero and they contribute no output. Original +comments remain unchanged to preserve source identity; the audit follows code. diff --git a/evidence/moe-agent-comparison-20261104/patch/candidates.csv b/evidence/moe-agent-comparison-20261104/patch/candidates.csv new file mode 100644 index 000000000..f811ec181 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/patch/candidates.csv @@ -0,0 +1,55 @@ +session_key,role,round_index,file,sha256,bytes +gpt61-kb-on,exploratory,1,patch/exploration-candidates.zip!gpt61-kb-on/round-01.py,0d0f392255b28699940db896c4eafe9a859e587dd7d7bacbbe68859ebbffb383,3065 +gpt61-kb-on,exploratory,2,patch/exploration-candidates.zip!gpt61-kb-on/round-02.py,2657e679b9f374d6e0aee01c889cf69cf23d7a2354ca58921ba731052cd3f452,3783 +gpt61-kb-on,exploratory,3,patch/exploration-candidates.zip!gpt61-kb-on/round-03.py,393a0cd1b4affdd79ab49920a4a6724f2b53d1f7ce2d45efe585874247d04c0a,4150 +gpt61-kb-on,exploratory,4,patch/exploration-candidates.zip!gpt61-kb-on/round-04.py,8fc66ba373f034b851d292a7e26097dcf814307b2c922c9fb1a732b931b95d68,3822 +gpt61-kb-on,exploratory,5,patch/exploration-candidates.zip!gpt61-kb-on/round-05.py,507d265ffde8d7b1d3348e37c46ab4f696503e0dafbfef1bbbb7cd3606e25d0e,3992 +gpt61-kb-on,exploratory,6,patch/exploration-candidates.zip!gpt61-kb-on/round-06.py,81a9b68ef4108e90279e99e9a323f7dba8ecdd211c909a67b115225adceb26d7,4221 +gpt61-kb-on,exploratory,7,patch/exploration-candidates.zip!gpt61-kb-on/round-07.py,0cf406d0472202fa9ac35390ab28993194e512c2414696a1840e19a4efc304fa,5036 +gpt61-kb-on,exploratory,8,patch/exploration-candidates.zip!gpt61-kb-on/round-08.py,c896e3e99aa4be3789526c41f74ca5bf34fa310a5532407638e12df13de4e832,4761 +gpt61-kb-on,final,,patch/final/gpt61-kb-on.py,507d265ffde8d7b1d3348e37c46ab4f696503e0dafbfef1bbbb7cd3606e25d0e,3992 +gpt61-kb-off,exploratory,1,patch/exploration-candidates.zip!gpt61-kb-off/round-01.py,74c9cbb8ac2be1b9aac1f45625d2bb77fa951c5e65cc0d3e0e7900db87ca2a42,2997 +gpt61-kb-off,exploratory,2,patch/exploration-candidates.zip!gpt61-kb-off/round-02.py,f163d015875b80f90d67b8f50dd246f59fb937299ea2a43f0279036f574e1437,3776 +gpt61-kb-off,exploratory,3,patch/exploration-candidates.zip!gpt61-kb-off/round-03.py,9cdee85e0b487703a92a85a3d4d78ad3bd11749b434dbae00ece0f03fc92db08,3955 +gpt61-kb-off,exploratory,4,patch/exploration-candidates.zip!gpt61-kb-off/round-04.py,10d7bd7d74c147f476e050bd5e23e347dd51096d83575e802d13d150df923f4c,4061 +gpt61-kb-off,exploratory,5,patch/exploration-candidates.zip!gpt61-kb-off/round-05.py,69efab972a84228f7e31cf827e615b23777316a0586cc0624214edea8e64e2eb,4162 +gpt61-kb-off,exploratory,6,patch/exploration-candidates.zip!gpt61-kb-off/round-06.py,ac521b38f2555639bcde827f35b9b205767136b91ccf170e3a3f72c1c06de4f2,4016 +gpt61-kb-off,exploratory,7,patch/exploration-candidates.zip!gpt61-kb-off/round-07.py,67bb00dd912136041066cbcb1949ff2f9b5397be54973dffca81dc55ad4b0b17,4215 +gpt61-kb-off,exploratory,8,patch/exploration-candidates.zip!gpt61-kb-off/round-08.py,36a9ac623ee1c3c2cf0717d69bd48d5915edbd4a828832a1693b83bc2632ccc7,4445 +gpt61-kb-off,final,,patch/final/gpt61-kb-off.py,ac521b38f2555639bcde827f35b9b205767136b91ccf170e3a3f72c1c06de4f2,4016 +gpt56-kb-on,exploratory,1,patch/exploration-candidates.zip!gpt56-kb-on/round-01.py,aedc1c7d99e2a5615f18443acdccdd1285befbcfd341a7aedb25586718e9e354,3026 +gpt56-kb-on,exploratory,2,patch/exploration-candidates.zip!gpt56-kb-on/round-02.py,e9324fb9b5def09b0313528a516b39cf5bdcec1a5cb8f77c62a07800bc18955f,3248 +gpt56-kb-on,exploratory,3,patch/exploration-candidates.zip!gpt56-kb-on/round-03.py,464916a52eacbed262279c369da3ecdf04c4c862a0ab8f44bac1ca6368a4d862,3367 +gpt56-kb-on,exploratory,4,patch/exploration-candidates.zip!gpt56-kb-on/round-04.py,3f14efc39ace2ba129a048214d66eec2f624f522bc4b9baa76b509bad188f3ca,3231 +gpt56-kb-on,exploratory,5,patch/exploration-candidates.zip!gpt56-kb-on/round-05.py,8d52e2f95ae28b6beeaba607868a492a5a2ab2b647d806d19e8155ff1b94c9cf,3771 +gpt56-kb-on,exploratory,6,patch/exploration-candidates.zip!gpt56-kb-on/round-06.py,a3762e1d73ef5b4f1e672b727e708b4d12fa6d2be53a1c8755db3c8829c80348,3333 +gpt56-kb-on,exploratory,7,patch/exploration-candidates.zip!gpt56-kb-on/round-07.py,7a3080fb599fbe18f58bc2c86f558740d4843e95e72b189d8b790f7460128483,2947 +gpt56-kb-on,exploratory,8,patch/exploration-candidates.zip!gpt56-kb-on/round-08.py,4815b5f5fbae5bcf2f357ab51196301a57330e8ee391ef530fc9324202564f4e,3014 +gpt56-kb-on,final,,patch/final/gpt56-kb-on.py,4815b5f5fbae5bcf2f357ab51196301a57330e8ee391ef530fc9324202564f4e,3014 +gpt56-kb-off,exploratory,1,patch/exploration-candidates.zip!gpt56-kb-off/round-01.py,b868abaac4b60f655b9360c2af5276ba53986c2c5da9e13b029685431244edb6,3012 +gpt56-kb-off,exploratory,2,patch/exploration-candidates.zip!gpt56-kb-off/round-02.py,c323e4c03a3129c3817eed5c4ece012337c6c0336d28585eb2a37653bffb3c81,3213 +gpt56-kb-off,exploratory,3,patch/exploration-candidates.zip!gpt56-kb-off/round-03.py,88fe22030b7344cf1207adc5939a3954c5b1935c95aa83f25454c19ddf6ce579,2995 +gpt56-kb-off,exploratory,4,patch/exploration-candidates.zip!gpt56-kb-off/round-04.py,f2518462b20b01cd36b206948e28613041357aae2c58386b3fa902b36d5a3290,4052 +gpt56-kb-off,exploratory,5,patch/exploration-candidates.zip!gpt56-kb-off/round-05.py,dc4de6caa9c81aa388255f2ec886f804bd15ced5495feadb72400d60a559f3cd,4323 +gpt56-kb-off,exploratory,6,patch/exploration-candidates.zip!gpt56-kb-off/round-06.py,3f61231245363d3e5029148c1761fee4fe3bf8e64573f7bd50bd31fccccc6d52,4389 +gpt56-kb-off,exploratory,7,patch/exploration-candidates.zip!gpt56-kb-off/round-07.py,7090b96e2442cffa124c58fb1b25749dce228b7907ce8c968d827344fca8e594,4552 +gpt56-kb-off,exploratory,8,patch/exploration-candidates.zip!gpt56-kb-off/round-08.py,07d0a19761a91c5e3b3a080c14c4ddcaf262f9e0499de8ad3d898d94bd98046b,4231 +gpt56-kb-off,final,,patch/final/gpt56-kb-off.py,07d0a19761a91c5e3b3a080c14c4ddcaf262f9e0499de8ad3d898d94bd98046b,4231 +gpt6astra-kb-on,exploratory,1,patch/exploration-candidates.zip!gpt6astra-kb-on/round-01.py,68c5f4a4cef7d800aab923af778431bdf0f35dde432717aac53b524b3c3e41d7,2965 +gpt6astra-kb-on,exploratory,2,patch/exploration-candidates.zip!gpt6astra-kb-on/round-02.py,854773d84fdbd027ffbe2368dbda499c0e3e76b148958f36f64c54964b4d8b87,3653 +gpt6astra-kb-on,exploratory,3,patch/exploration-candidates.zip!gpt6astra-kb-on/round-03.py,993a923efd56399775efc9596dfe3344ff68478a135de42fadf953bf21e5b884,3542 +gpt6astra-kb-on,exploratory,4,patch/exploration-candidates.zip!gpt6astra-kb-on/round-04.py,0016258c26a8ac4b9ababeed1b8fcf13871c0578f52965df3b05eec4a55746ac,3659 +gpt6astra-kb-on,exploratory,5,patch/exploration-candidates.zip!gpt6astra-kb-on/round-05.py,73a72a0c29de1a0961e874a66173324002d3c80e0702cfc11f907258090713b1,3640 +gpt6astra-kb-on,exploratory,6,patch/exploration-candidates.zip!gpt6astra-kb-on/round-06.py,b0bede9598ec1737ae9932d636dda6067f8a1073c7711bb523b67015d3eb4539,4370 +gpt6astra-kb-on,exploratory,7,patch/exploration-candidates.zip!gpt6astra-kb-on/round-07.py,1561c1fb8f1bfdcd625802e69146b24d1b69958b22dee3d855921fe4eceb657c,3690 +gpt6astra-kb-on,exploratory,8,patch/exploration-candidates.zip!gpt6astra-kb-on/round-08.py,aea3a29f087b3ff6459b5e799f67c0aba829aed03da3da2f03af93cecc022f12,3747 +gpt6astra-kb-on,final,,patch/final/gpt6astra-kb-on.py,1561c1fb8f1bfdcd625802e69146b24d1b69958b22dee3d855921fe4eceb657c,3690 +gpt6astra-kb-off,exploratory,1,patch/exploration-candidates.zip!gpt6astra-kb-off/round-01.py,c7b20dfbc23f611feb5022f08e0d40e428e2b365062b74229d6a64ca7946958b,2959 +gpt6astra-kb-off,exploratory,2,patch/exploration-candidates.zip!gpt6astra-kb-off/round-02.py,b0f9d88f3fa94668744c188a7f7964160aced4bd1f3e1f9ada924b397292e536,3677 +gpt6astra-kb-off,exploratory,3,patch/exploration-candidates.zip!gpt6astra-kb-off/round-03.py,ef5c55d92cd74bd904f3130eba476a682c3d17d4113e4163a41c0663a89735e9,3369 +gpt6astra-kb-off,exploratory,4,patch/exploration-candidates.zip!gpt6astra-kb-off/round-04.py,252af26dddc8856a78f6ad112e939c640cf229028820b453d66d46cc838d696e,3668 +gpt6astra-kb-off,exploratory,5,patch/exploration-candidates.zip!gpt6astra-kb-off/round-05.py,38ac84ca35b444caf4f71854bbe62211f4c359b0bd07bc95192539a1d3d0cb0e,3649 +gpt6astra-kb-off,exploratory,6,patch/exploration-candidates.zip!gpt6astra-kb-off/round-06.py,84ec3d844910f2ac40079165f5ab0bbe4263a7b53315fbe1040fc645c63170d2,3375 +gpt6astra-kb-off,exploratory,7,patch/exploration-candidates.zip!gpt6astra-kb-off/round-07.py,b84a624eb193224e3fd1f8c1bd087d080b2d2ac46c713c5cd808543ae64da0c3,3397 +gpt6astra-kb-off,exploratory,8,patch/exploration-candidates.zip!gpt6astra-kb-off/round-08.py,0d784712015edc1ad94393e15ee19152e660b2288ad66213d886b4ea7972c4f0,4163 +gpt6astra-kb-off,final,,patch/final/gpt6astra-kb-off.py,ef5c55d92cd74bd904f3130eba476a682c3d17d4113e4163a41c0663a89735e9,3369 diff --git a/evidence/moe-agent-comparison-20261104/patch/exploration-candidates.zip b/evidence/moe-agent-comparison-20261104/patch/exploration-candidates.zip new file mode 100644 index 000000000..e29097b09 Binary files /dev/null and b/evidence/moe-agent-comparison-20261104/patch/exploration-candidates.zip differ diff --git a/evidence/moe-agent-comparison-20261104/patch/final/gpt56-kb-off.py b/evidence/moe-agent-comparison-20261104/patch/final/gpt56-kb-off.py new file mode 100644 index 000000000..10677c987 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/patch/final/gpt56-kb-off.py @@ -0,0 +1,103 @@ +"""EP=2 MoE + shared expert: AllGather/local experts/ReduceScatter baseline. + +This is a self-contained, newly written implementation of the common +AllGather-based expert-parallel MoE workload architecture, not a copy of the +vLLM serving stack. It uses a fused gate/up projection and NPU SwiGLU. +""" + +import torch +import torch.distributed as dist +import torch.nn.functional as F + + +def _mlp(x, gate_up, down): + if x.shape[0] == 0: + return x.new_empty((0, down.shape[0])) + projected = F.linear(x, gate_up) + if x.device.type == "npu": + import torch_npu + activated = torch_npu.npu_swiglu(projected, dim=-1) + else: + gate, up = projected.chunk(2, dim=-1) + activated = F.silu(gate) * up + return F.linear(activated, down) + + +def compute(x, params, *, config, group): + world_size, rank = dist.get_world_size(group), dist.get_rank(group) + if world_size != 2 or rank != config["rank"]: + raise ValueError("Expected the configured two-rank process group") + max_tokens = max(config["rank_tokens"]) + hidden = config["hidden"] + padded_x = F.pad(x, (0, 0, 0, max_tokens - x.shape[0])) + all_x = torch.empty((2 * max_tokens, hidden), dtype=x.dtype, device=x.device) + dist.all_gather_into_tensor(all_x, padded_x.contiguous(), group=group) + # Router parameters are replicated, so recomputing the small router on the + # gathered tokens avoids a separate FP32 logits collective. + all_logits = F.linear( + all_x.float(), params["router_weight"], params["router_bias"] + ) + selected_logits, selected_experts = all_logits.topk( + config["top_k"], dim=-1, sorted=True + ) + probabilities = selected_logits.softmax(dim=-1) + # Padded slots have zero weight and never perform an expert MLP. + for source_rank, count in enumerate(config["rank_tokens"]): + if count < max_tokens: + probabilities[source_rank * max_tokens + count:(source_rank + 1) * max_tokens] = 0 + local_contribution = torch.zeros( + (2 * max_tokens, hidden), dtype=torch.float32, device=x.device + ) + flat_experts = selected_experts.reshape(-1) + flat_probabilities = probabilities.reshape(-1) + flat_token_ids = ( + torch.arange(selected_experts.shape[0], device=x.device) + .unsqueeze(1) + .expand(-1, config["top_k"]) + .reshape(-1) + ) + local_mask = ( + (flat_experts >= rank * 4) + & (flat_experts < (rank + 1) * 4) + & (flat_probabilities > 0) + ) + local_experts = flat_experts[local_mask] - rank * 4 + token_ids = flat_token_ids[local_mask] + route_probabilities = flat_probabilities[local_mask] + if local_experts.numel(): + assignment_prefix = F.one_hot(local_experts, num_classes=4).cumsum(0) - 1 + packed_rows = assignment_prefix.gather( + 1, local_experts.unsqueeze(1) + ).squeeze(1) + counts = torch.bincount(local_experts, minlength=4) + max_expert_tokens = int(counts.max().item()) + expert_inputs = torch.zeros( + (4, max_expert_tokens, hidden), dtype=x.dtype, device=x.device + ) + expert_inputs[local_experts, packed_rows] = all_x.index_select(0, token_ids) + projected = torch.bmm( + expert_inputs, params["expert_gate_up"].transpose(1, 2) + ) + if x.device.type == "npu": + import torch_npu + activated = torch_npu.npu_swiglu(projected, dim=-1) + else: + gate, up = projected.chunk(2, dim=-1) + activated = F.silu(gate) * up + expert_outputs = torch.bmm( + activated, params["expert_down"].transpose(1, 2) + ) + weighted = ( + expert_outputs[local_experts, packed_rows].float() + * route_probabilities.unsqueeze(-1) + ) + local_contribution.index_add_(0, token_ids, weighted) + local_routed = torch.empty( + (max_tokens, hidden), dtype=torch.float32, device=x.device + ) + reduce_work = dist.reduce_scatter_tensor( + local_routed, local_contribution, group=group, async_op=True + ) + shared = _mlp(x, params["shared_gate_up"], params["shared_down"]) + reduce_work.wait() + return (local_routed[:x.shape[0]] + shared.float()).to(dtype=x.dtype) diff --git a/evidence/moe-agent-comparison-20261104/patch/final/gpt56-kb-on.py b/evidence/moe-agent-comparison-20261104/patch/final/gpt56-kb-on.py new file mode 100644 index 000000000..df9769ec8 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/patch/final/gpt56-kb-on.py @@ -0,0 +1,72 @@ +"""EP=2 MoE + shared expert: AllGather/local experts/ReduceScatter baseline. + +This is a self-contained, newly written implementation of the common +AllGather-based expert-parallel MoE workload architecture, not a copy of the +vLLM serving stack. It uses a fused gate/up projection and NPU SwiGLU. +""" + +import torch +import torch.distributed as dist +import torch.nn.functional as F + + +def _mlp(x, gate_up, down): + if x.shape[0] == 0: + return x.new_empty((0, down.shape[0])) + projected = F.linear(x, gate_up) + if x.device.type == "npu": + import torch_npu + activated = torch_npu.npu_swiglu(projected, dim=-1) + else: + gate, up = projected.chunk(2, dim=-1) + activated = F.silu(gate) * up + return F.linear(activated, down) + + +def compute(x, params, *, config, group): + world_size, rank = dist.get_world_size(group), dist.get_rank(group) + if world_size != 2 or rank != config["rank"]: + raise ValueError("Expected the configured two-rank process group") + max_tokens = max(config["rank_tokens"]) + hidden = config["hidden"] + padded_x = ( + x + if x.shape[0] == max_tokens + else F.pad(x, (0, 0, 0, max_tokens - x.shape[0])) + ) + all_x = torch.empty((2 * max_tokens, hidden), dtype=x.dtype, device=x.device) + gather_work = dist.all_gather_into_tensor( + all_x, padded_x.contiguous(), group=group, async_op=True + ) + shared = _mlp(x, params["shared_gate_up"], params["shared_down"]) + gather_work.wait() + all_logits = F.linear( + all_x.float(), params["router_weight"], params["router_bias"] + ) + selected_logits, selected_experts = all_logits.topk( + config["top_k"], dim=-1, sorted=True + ) + probabilities = selected_logits.softmax(dim=-1) + # Exclude padded rows once, so each expert mask needs only one comparison. + for source_rank, count in enumerate(config["rank_tokens"]): + if count < max_tokens: + selected_experts[ + source_rank * max_tokens + count:(source_rank + 1) * max_tokens + ] = -1 + local_contribution = torch.zeros((2 * max_tokens, hidden), dtype=torch.float32, device=x.device) + for local_expert in range(4): + expert_id = rank * 4 + local_expert + positions = torch.nonzero(selected_experts == expert_id, as_tuple=False) + if positions.shape[0] == 0: + continue + token_ids, slot_ids = positions[:, 0], positions[:, 1] + values = _mlp( + all_x.index_select(0, token_ids), + params["expert_gate_up"][local_expert], + params["expert_down"][local_expert], + ) + weighted = values.float() * probabilities[token_ids, slot_ids].unsqueeze(-1) + local_contribution.index_add_(0, token_ids, weighted) + local_routed = torch.empty((max_tokens, hidden), dtype=torch.float32, device=x.device) + dist.reduce_scatter_tensor(local_routed, local_contribution, group=group) + return (local_routed[:x.shape[0]] + shared.float()).to(dtype=x.dtype) diff --git a/evidence/moe-agent-comparison-20261104/patch/final/gpt61-kb-off.py b/evidence/moe-agent-comparison-20261104/patch/final/gpt61-kb-off.py new file mode 100644 index 000000000..1b66633f5 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/patch/final/gpt61-kb-off.py @@ -0,0 +1,81 @@ +"""EP=2 MoE + shared expert: AllGather/local experts/ReduceScatter baseline. + +This is a self-contained, newly written implementation of the common +AllGather-based expert-parallel MoE workload architecture, not a copy of the +vLLM serving stack. It uses a fused gate/up projection and NPU SwiGLU. +""" + +import torch +import torch.distributed as dist +import torch.nn.functional as F + + +def _mlp(x, gate_up, down): + if x.shape[0] == 0: + return x.new_empty((0, down.shape[0])) + projected = F.linear(x, gate_up) + if x.device.type == "npu": + import torch_npu + activated = torch_npu.npu_swiglu(projected, dim=-1) + else: + gate, up = projected.chunk(2, dim=-1) + activated = F.silu(gate) * up + return F.linear(activated, down) + + +def compute(x, params, *, config, group): + world_size, rank = dist.get_world_size(group), dist.get_rank(group) + if world_size != 2 or rank != config["rank"]: + raise ValueError("Expected the configured two-rank process group") + max_tokens = max(config["rank_tokens"]) + hidden = config["hidden"] + padded_x = x if x.shape[0] == max_tokens else F.pad(x, (0, 0, 0, max_tokens - x.shape[0])) + all_x = torch.empty((2 * max_tokens, hidden), dtype=x.dtype, device=x.device) + dist.all_gather_into_tensor(all_x, padded_x.contiguous(), group=group) + # Router is replicated. Recompute it after gathering tokens to remove a + # second collective while preserving FP32 routing arithmetic. + all_logits = F.linear(all_x.float(), params["router_weight"], params["router_bias"]) + selected_logits, selected_experts = all_logits.topk(config["top_k"], dim=-1, sorted=True) + probabilities = selected_logits.softmax(dim=-1) + # Padded slots have zero weight in the final routed contribution. + for source_rank, count in enumerate(config["rank_tokens"]): + if count < max_tokens: + probabilities[source_rank * max_tokens + count:(source_rank + 1) * max_tokens] = 0 + import torch_npu + # Native dropless dispatch provides both the grouped input and inverse map. + expert_x, inverse_map, groups, _ = torch_npu.npu_moe_init_routing_v2( + all_x, selected_experts.to(torch.int32), + active_num=2 * max_tokens * config["top_k"], expert_num=8, + drop_pad_mode=0, expert_tokens_num_type=0, + expert_tokens_num_flag=True, quant_mode=-1, + active_expert_range=[rank * 4, rank * 4 + 4], row_idx_type=0, + ) + # Test the runtime's prefix-group support with static dispatch capacity. + # Group boundaries remain device tensors and contain every local route. + projected = torch_npu.npu_grouped_matmul( + [expert_x], [params["expert_gate_up"].transpose(1, 2)], + group_list=groups, split_item=3, group_type=0, + group_list_type=0, output_dtype=x.dtype, + )[0] + activated = torch_npu.npu_swiglu(projected, dim=-1) + values = torch_npu.npu_grouped_matmul( + [activated], [params["expert_down"].transpose(1, 2)], + group_list=groups, split_item=3, group_type=0, + group_list_type=0, output_dtype=x.dtype, + )[0] + # Kernels may leave inactive capacity uninitialized. Clear it before + # finalize, including the fully empty-local-expert case. + inactive = torch.arange(values.shape[0], device=x.device) >= groups[-1] + full_values = values.masked_fill(inactive.unsqueeze(-1), 0).float() + local_probabilities = probabilities * (inverse_map.view_as(probabilities) >= 0) + local_contribution = torch_npu.npu_moe_finalize_routing( + full_values, None, None, None, local_probabilities, + inverse_map.clamp_min(0), None, drop_pad_mode=2, + ) + local_routed = torch.empty((max_tokens, hidden), dtype=torch.float32, device=x.device) + routed_work = dist.reduce_scatter_tensor( + local_routed, local_contribution, group=group, async_op=True + ) + shared = _mlp(x, params["shared_gate_up"], params["shared_down"]) + routed_work.wait() + return (local_routed[:x.shape[0]] + shared.float()).to(dtype=x.dtype) diff --git a/evidence/moe-agent-comparison-20261104/patch/final/gpt61-kb-on.py b/evidence/moe-agent-comparison-20261104/patch/final/gpt61-kb-on.py new file mode 100644 index 000000000..218e66cc0 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/patch/final/gpt61-kb-on.py @@ -0,0 +1,76 @@ +"""EP=2 MoE + shared expert: AllGather/local experts/ReduceScatter baseline. + +This is a self-contained, newly written implementation of the common +AllGather-based expert-parallel MoE workload architecture, not a copy of the +vLLM serving stack. It uses a fused gate/up projection and NPU SwiGLU. +""" + +import torch +import torch.distributed as dist +import torch.nn.functional as F + + +def _mlp(x, gate_up, down): + if x.shape[0] == 0: + return x.new_empty((0, down.shape[0])) + projected = F.linear(x, gate_up) + if x.device.type == "npu": + import torch_npu + activated = torch_npu.npu_swiglu(projected, dim=-1) + else: + gate, up = projected.chunk(2, dim=-1) + activated = F.silu(gate) * up + return F.linear(activated, down) + + +def compute(x, params, *, config, group): + world_size, rank = dist.get_world_size(group), dist.get_rank(group) + if world_size != 2 or rank != config["rank"]: + raise ValueError("Expected the configured two-rank process group") + max_tokens = max(config["rank_tokens"]) + hidden = config["hidden"] + padded_x = F.pad(x, (0, 0, 0, max_tokens - x.shape[0])) + all_x = torch.empty((2 * max_tokens, hidden), dtype=x.dtype, device=x.device) + dist.all_gather_into_tensor(all_x, padded_x.contiguous(), group=group) + # The router is replicated: recompute the small FP32 projection after + # gathering BF16 activations and avoid a separate logits collective. + all_logits = F.linear(all_x.float(), params["router_weight"], params["router_bias"]) + selected_logits, selected_experts = all_logits.topk(config["top_k"], dim=-1, sorted=True) + probabilities = selected_logits.softmax(dim=-1) + # Padded slots have zero weight and never perform an expert MLP. + for source_rank, count in enumerate(config["rank_tokens"]): + if count < max_tokens: + probabilities[source_rank * max_tokens + count:(source_rank + 1) * max_tokens] = 0 + local_contribution = torch.zeros((2 * max_tokens, hidden), dtype=torch.float32, device=x.device) + import torch_npu + # Native dropless routing produces expert-sorted rows and a row-major + # inverse mapping. Only this rank's expert range is processed here. + expanded_x, row_map, group_counts, _ = torch_npu.npu_moe_init_routing_v2( + all_x, selected_experts.to(torch.int32), expert_num=8, + active_expert_range=[rank * 4, rank * 4 + 4], + expert_tokens_num_type=1, expert_tokens_num_flag=True, + drop_pad_mode=0, quant_mode=-1, row_idx_type=0 + ) + routed_count = int(group_counts.sum().item()) + if routed_count: + projected = torch_npu.npu_grouped_matmul( + [expanded_x[:routed_count]], [params["expert_gate_up"].transpose(1, 2)], + group_list=group_counts, split_item=3, group_type=0, group_list_type=1 + )[0] + activated = torch_npu.npu_swiglu(projected, dim=-1) + values = torch_npu.npu_grouped_matmul( + [activated], [params["expert_down"].transpose(1, 2)], + group_list=group_counts, split_item=3, group_type=0, group_list_type=1 + )[0] + # Finalize in FP32. Map non-local slots to an explicitly zero padded + # row, so the dropless row-major finalize never sees negative indices. + padded_values = F.pad(values.float(), (0, 0, 0, 2 * max_tokens * config["top_k"] - routed_count)) + safe_map = torch.where(row_map >= 0, row_map, routed_count) + local_contribution = torch_npu.npu_moe_finalize_routing( + padded_values, None, None, None, probabilities, + safe_map, None, drop_pad_mode=2 + ) + local_routed = torch.empty((max_tokens, hidden), dtype=torch.float32, device=x.device) + dist.reduce_scatter_tensor(local_routed, local_contribution, group=group) + shared = _mlp(x, params["shared_gate_up"], params["shared_down"]) + return (local_routed[:x.shape[0]] + shared.float()).to(dtype=x.dtype) diff --git a/evidence/moe-agent-comparison-20261104/patch/final/gpt6astra-kb-off.py b/evidence/moe-agent-comparison-20261104/patch/final/gpt6astra-kb-off.py new file mode 100644 index 000000000..91e71572e --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/patch/final/gpt6astra-kb-off.py @@ -0,0 +1,66 @@ +"""EP=2 MoE + shared expert: AllGather/local experts/ReduceScatter baseline. + +This is a self-contained, newly written implementation of the common +AllGather-based expert-parallel MoE workload architecture, not a copy of the +vLLM serving stack. It uses a fused gate/up projection and NPU SwiGLU. +""" + +import torch +import torch.distributed as dist +import torch.nn.functional as F + + +def _mlp(x, gate_up, down): + if x.shape[0] == 0: + return x.new_empty((0, down.shape[0])) + projected = F.linear(x, gate_up) + if x.device.type == "npu": + import torch_npu + activated = torch_npu.npu_swiglu(projected, dim=-1) + else: + gate, up = projected.chunk(2, dim=-1) + activated = F.silu(gate) * up + return F.linear(activated, down) + + +def compute(x, params, *, config, group): + world_size, rank = dist.get_world_size(group), dist.get_rank(group) + if world_size != 2 or rank != config["rank"]: + raise ValueError("Expected the configured two-rank process group") + max_tokens = max(config["rank_tokens"]) + hidden = config["hidden"] + padded_x = F.pad(x, (0, 0, 0, max_tokens - x.shape[0])) + all_x = torch.empty((2 * max_tokens, hidden), dtype=x.dtype, device=x.device) + gather = dist.all_gather_into_tensor(all_x, padded_x.contiguous(), group=group, async_op=True) + shared = _mlp(x, params["shared_gate_up"], params["shared_down"]) + gather.wait() + all_logits = F.linear(all_x.float(), params["router_weight"], params["router_bias"]) + selected_logits, selected_experts = all_logits.topk(config["top_k"], dim=-1, sorted=True) + probabilities = selected_logits.softmax(dim=-1) + # Padded slots have zero weight and never perform an expert MLP. + for source_rank, count in enumerate(config["rank_tokens"]): + if count < max_tokens: + probabilities[source_rank * max_tokens + count:(source_rank + 1) * max_tokens] = 0 + import torch_npu + # Static-capacity routing keeps counts on device; only local experts execute. + packed, inverse, boundaries, _ = torch_npu.npu_moe_init_routing_v2( + all_x, selected_experts.to(torch.int32), active_num=-1, expert_num=8, + drop_pad_mode=0, expert_tokens_num_type=0, expert_tokens_num_flag=True, + active_expert_range=[rank * 4, rank * 4 + 4], quant_mode=-1, row_idx_type=0, + ) + projected = torch_npu.npu_grouped_matmul( + [packed], [params["expert_gate_up"].transpose(1, 2)], + group_list=boundaries, split_item=2, group_type=0, group_list_type=0, + )[0] + activated = torch_npu.npu_swiglu(projected, dim=-1) + values = torch_npu.npu_grouped_matmul( + [activated], [params["expert_down"].transpose(1, 2)], + group_list=boundaries, split_item=2, group_type=0, group_list_type=0, + )[0] + expanded = values.index_select(0, inverse.clamp_min(0).long()) + expanded = torch.where((inverse >= 0).unsqueeze(-1), expanded, 0) + expanded = expanded.view(2 * max_tokens, config["top_k"], hidden).float() + local_contribution = (expanded * probabilities.unsqueeze(-1)).sum(dim=1) + local_routed = torch.empty((max_tokens, hidden), dtype=torch.float32, device=x.device) + dist.reduce_scatter_tensor(local_routed, local_contribution, group=group) + return (local_routed[:x.shape[0]] + shared.float()).to(dtype=x.dtype) diff --git a/evidence/moe-agent-comparison-20261104/patch/final/gpt6astra-kb-on.py b/evidence/moe-agent-comparison-20261104/patch/final/gpt6astra-kb-on.py new file mode 100644 index 000000000..5d2438e9c --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/patch/final/gpt6astra-kb-on.py @@ -0,0 +1,70 @@ +"""EP=2 MoE + shared expert: AllGather/local experts/ReduceScatter baseline. + +This is a self-contained, newly written implementation of the common +AllGather-based expert-parallel MoE workload architecture, not a copy of the +vLLM serving stack. It uses a fused gate/up projection and NPU SwiGLU. +""" + +import torch +import torch_npu +import torch.distributed as dist +import torch.nn.functional as F + + +def _mlp(x, gate_up, down): + if x.shape[0] == 0: + return x.new_empty((0, down.shape[0])) + projected = F.linear(x, gate_up) + if x.device.type == "npu": + import torch_npu + activated = torch_npu.npu_swiglu(projected, dim=-1) + else: + gate, up = projected.chunk(2, dim=-1) + activated = F.silu(gate) * up + return F.linear(activated, down) + + +def compute(x, params, *, config, group): + world_size, rank = dist.get_world_size(group), dist.get_rank(group) + if world_size != 2 or rank != config["rank"]: + raise ValueError("Expected the configured two-rank process group") + max_tokens = max(config["rank_tokens"]) + hidden = config["hidden"] + padded_x = x if x.shape[0] == max_tokens else F.pad(x, (0, 0, 0, max_tokens - x.shape[0])) + all_x = torch.empty((2 * max_tokens, hidden), dtype=x.dtype, device=x.device) + gathering = dist.all_gather_into_tensor(all_x, padded_x.contiguous(), group=group, async_op=True) + shared = _mlp(x, params["shared_gate_up"], params["shared_down"]) + gathering.wait() + all_logits = F.linear(all_x.float(), params["router_weight"], params["router_bias"]) + selected_logits, selected_experts = all_logits.topk(config["top_k"], dim=-1, sorted=True) + probabilities = selected_logits.softmax(dim=-1) + # Padded slots have zero weight; zero token rows make no contribution. + for source_rank, count in enumerate(config["rank_tokens"]): + if count < max_tokens: + probabilities[source_rank * max_tokens + count:(source_rank + 1) * max_tokens] = 0 + dispatched, inverse_positions, groups, _ = torch_npu.npu_moe_init_routing_v2( + all_x, selected_experts.to(torch.int32), expert_num=8, + active_num=-1, drop_pad_mode=0, expert_tokens_num_type=0, + expert_tokens_num_flag=True, quant_mode=-1, + active_expert_range=[rank * 4, (rank + 1) * 4], row_idx_type=0) + local_count = int(groups[-1].item()) + if local_count: + dispatched = dispatched[:local_count] + projected = torch_npu.npu_grouped_matmul( + [dispatched], [params["expert_gate_up"].transpose(1, 2)], + group_list=groups, split_item=3, group_type=0)[0] + activated = torch_npu.npu_swiglu(projected, dim=-1) + values = torch_npu.npu_grouped_matmul( + [activated], [params["expert_down"].transpose(1, 2)], + group_list=groups, split_item=3, group_type=0)[0] + # The fusion API requires a full expanded buffer and nonnegative indices. + expanded = F.pad(values.float(), (0, 0, 0, selected_experts.numel() - local_count)) + scales = probabilities * (inverse_positions >= 0).reshape_as(probabilities) + local_contribution = torch_npu.npu_moe_finalize_routing( + expanded, None, None, None, scales, inverse_positions.clamp_min(0), + None, drop_pad_mode=2) + else: + local_contribution = torch.zeros((2 * max_tokens, hidden), dtype=torch.float32, device=x.device) + local_routed = torch.empty((max_tokens, hidden), dtype=torch.float32, device=x.device) + dist.reduce_scatter_tensor(local_routed, local_contribution, group=group) + return (local_routed[:x.shape[0]] + shared.float()).to(dtype=x.dtype) diff --git a/evidence/moe-agent-comparison-20261104/plan/README.md b/evidence/moe-agent-comparison-20261104/plan/README.md new file mode 100644 index 000000000..5fde71ac2 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/plan/README.md @@ -0,0 +1,41 @@ +# 历史条件协议与原件身份 + +这是历史提示协议的脱敏提取，不是逐字复制私有会话提示。 +原四组在搜索前冻结计划，后追加 Astra ON/OFF，沿用任务、门槛、测量、源包和 +访问约束。两份原件分别由 `provenance.json` 中的 SHA256 标识； +`publication.json` 是另有身份的公开字段提取，不冒充原件。 + +每组新建 projectless 会话，不 fork、不 resume、不复制父会话优化上下文。 +实际模型及 xhigh 从会话 turn context 核验；每条件仅运行一个 Agent，不调 +子 Agent、不换模型。按 6.1 ON、6.1 OFF、5.6 ON、5.6 OFF、Astra ON、Astra OFF +串行执行；前一组 8 轮、独立 final、资源清理全部结束后才开始下一组。 + +共同指令协议：读取冻结任务及必要执行接口，交付 `compute` 实现；每轮提交 +不可变源码并调用共同评测器，根据正确性、逐 case 延迟和证据选择下一轮。 +不正确、抛异常或更慢的候选仍消耗一轮；不得因收集失败重跑已完成设备实验。 +恰好消耗 8 轮后，从本组已测有效候选或 baseline 中预选最终实现，再独立测 +一次 4 对 AB/BA final。final 不参与事后重选、不回填探索最优值。 +**8 轮是本次历史协议，不是 InferRSI 的全局默认值。** + +ON 允许查询冻结 InferRSI revision 中的 skill_library、knowledge_packs、 +repo_packs，通过 collect/resource 获取材料，记录返回 identity/revision、 +实际可见正文和采纳理由。OFF 不读取这些优化知识库正文。两边均可读冻结任务、 +执行器必要接口、共同 NPU 执行规则和公开技术文档；不得读其他会话/候选/结果、 +协调分析、私有预实验、全局优化技能/插件或外部记忆记录。实际访问记录另行审计； +这是指令和访问审计约束，不是硬文件系统隔离，也不能证明模型没有训练先验。 + +原四组及追加组实际使用相同冻结 `source_revision`、task/grader 与 79 个源文件 +hash。这些 campaign pins 只存在本证据及验收 issue 中，不进入通用默认值。 +历史执行入口是 `inferrsi.harness.scenarios.external.execute_trial`； +host-specific 启动配置和其余执行源字节未随此 bundle 公布，清单身份保留。 + +计时：完整 forward、host 调度和全部通信在区间内，返回后设备同步；每次 +两个 rank 取最大值。探索各 case 每实现 16 样本，final 32 样本，交替 AB/BA； +每个样本换输入，并使用数值不同但专家矩阵形状相同的预热输入。 +case 分数为 baseline 中位延迟 / candidate 中位延迟，总分是三 case 几何平均。 +错误样本不构造虚假速度，失败 round 保留之前 best。 + +冻结容差为逐元素 atol/rtol 0.03、NRMSE 0.01；检查双 rank、只读输入/参数、 +有限性、独立 CPU FP32 oracle，以及 top-k=1/2、不等 token、空专家、零输入 +rank、原位参数变化。参数更新用例不穷举所有缓存失效分支；最终六实现均没有 +权重布局缓存。完整准确门槛与输入生成方式见 `../validator/task/`。 diff --git a/evidence/moe-agent-comparison-20261104/plan/integrity.json b/evidence/moe-agent-comparison-20261104/plan/integrity.json new file mode 100644 index 000000000..5da8b8baa --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/plan/integrity.json @@ -0,0 +1 @@ +{"build_public.py":"40e5deb2bcdb1dc5ec4a04bd0bd652e634766eadd803b44c49b089caf088f203","patch/candidates.csv":"9b94e28b09c2e3618b68bce614f4f3e7b754aaac2d6b89397c840f832dd56b1f","patch/exploration-candidates.zip":"aa0dbdab4b41d1df4045e080ee22529fc1c3633a37d4fb26703548eb4a5b9e4d","patch/final/gpt56-kb-off.py":"07d0a19761a91c5e3b3a080c14c4ddcaf262f9e0499de8ad3d898d94bd98046b","patch/final/gpt56-kb-on.py":"4815b5f5fbae5bcf2f357ab51196301a57330e8ee391ef530fc9324202564f4e","patch/final/gpt61-kb-off.py":"ac521b38f2555639bcde827f35b9b205767136b91ccf170e3a3f72c1c06de4f2","patch/final/gpt61-kb-on.py":"507d265ffde8d7b1d3348e37c46ab4f696503e0dafbfef1bbbb7cd3606e25d0e","patch/final/gpt6astra-kb-off.py":"ef5c55d92cd74bd904f3130eba476a682c3d17d4113e4163a41c0663a89735e9","patch/final/gpt6astra-kb-on.py":"1561c1fb8f1bfdcd625802e69146b24d1b69958b22dee3d855921fe4eceb657c","patch/README.md":"015b9443d2cee28a8c476f2df95c03377d8ae7dbffb67c107f945bbc60e03749","plan/provenance.json":"f449bc63140a4d91ef2fc76b959208bda098506a538f9ea19aafbb9d365f1416","plan/publication.json":"77b2768fc016388d05aad0d17411492bac76cc4b28facf7f27f7a7d84874e10a","plan/README.md":"5a25aac6c4d46e6f71975e3dbee0640d7fa7a1373a090e9ebae0c924d91de805","README.md":"a420c19f49678b8c79e822d4d1e3921587ed6966f7d1e8911b43ce5a1649eb13","render_overview.py":"0aedc7744de629cdfae76c64035c6b3d9d68652c1579b97d2fdea4331541d754","results/audit-extract.json":"edc03a58b194c32886a92b09eceba5b9c6245de117a633d5f712eff34368e7dd","results/changes.md":"00998c27244df0ed88044ef52c20ce3f455731e02aa7da72be6f4bee8fa766d7","results/comparison-overview.png":"b2ac1d4eaf5156a166c4e519d86b556ce6a658c0bacac81059cdc36322af6b8a","results/measurements.jsonl.gz":"6882f62bb64a3645b2af9eeffe466589fefafec9bb02d6b60b0858ddc352ac1b","results/rounds.csv":"b245b2ff42d5883b65d222b752922ad9b6d1057ecf80bf5e036ba9dbf0d5ea5b","results/session-usage.csv":"3c86648bbc0e133bb5a42437d540045cf67fe710f9dca8fe936eaadcd667b12b","results/summary.json":"ded02f37fc872f3b38e7994133aa3c678951ca35e5bba61dc3ea8f28393788db","validator/README.md":"f5eb17713039590062d468f6349bc64e805f10beb7f0bcff530dae5a8bddbfe1","validator/runner/grade.py":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","validator/task/baseline.py":"ee41e4c926a7f0484848012e5bf22a31b3685daa2385c7bfe34a89d75f5568c6","validator/task/oracle.py":"6da20f29449da57d7eac508ea0a5732aacbb49ee9d755b63438a8afb3b4208d6","validator/task/task.json":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","validator/task/TASK.md":"07c4f9915040399a584520e79516ae237d16327c4c8615143fa846aa31a72730","validator/task/workload.py":"4a87ee7ee473de49467d164cb1c0b872a5515fee5a3d656c5bac1c7fe10a4463","verify.py":"1890e101079cd05a202d7d949f8c977c860ee4b849082812684b1e2739665f9a"} diff --git a/evidence/moe-agent-comparison-20261104/plan/provenance.json b/evidence/moe-agent-comparison-20261104/plan/provenance.json new file mode 100644 index 000000000..a498e7f54 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/plan/provenance.json @@ -0,0 +1 @@ +{"original_artifacts":{"results/summary.json":{"sha256":"5507e6f5683acf5d888dddad3e7f6ac42a8ddbe16731cb5ea008bb1a11244227","bytes":660443,"availability":"external original; no private paths published"},"common/plan.json":{"sha256":"fdf792c46212b1a31c7b72923847a6c3716d5b435f580688a4115b03653233b7","bytes":13800,"availability":"external original; no private paths published"},"extension-plan.json":{"sha256":"3c98fbb3db5b6f3fcdb72c0a7a985fc0eaac83d844f316dd5c4a4d59387f1e1e","bytes":2261,"availability":"external original; no private paths published"},"results/comparison-overview.png":{"sha256":"b2ac1d4eaf5156a166c4e519d86b556ce6a658c0bacac81059cdc36322af6b8a","bytes":418219,"availability":"published byte-identical copy at results/comparison-overview.png"},"analysis/render_overview.py":{"sha256":"543aa3423357a5e70c4cac9061ebfe42dbc9de01fa1a5fc01cd34976ce980ab4","bytes":7866,"availability":"external original; no private paths published"},"analysis/raw-audit-0.json":{"sha256":"de1ceea082b19681fe2182c394f6b629dc7c7befa612409be9bf0720a7b94dca","availability":"external audit original"},"analysis/condition-audit-0.json":{"sha256":"19f627691015325705d6cee9d0afbbb0c5018e272cae0e383d12f17285db557b","availability":"external audit original"},"analysis/source-audit-0.md":{"sha256":"be0c84e00541df96957ea8f11fe6faf796e3b31da5e9b8188c788966c7602465","availability":"external audit original"},"analysis/raw-audit-1.json":{"sha256":"928ea61018bab8560093797ea2b81ef86be4d44f0ec8f6a43a1d1a0f6aa61db7","availability":"external audit original"},"analysis/condition-audit-1.json":{"sha256":"8cf7c2a022236be8965e7213f2c3bd16a2c2a81f1d8c8f32486d0fe9993ce324","availability":"external audit original"},"analysis/source-audit-1.md":{"sha256":"8149cd05bd2fa072d200bf20f174a558c65d8f831504855fe12a0b1117c1368f","availability":"external audit original"},"analysis/raw-audit-2.json":{"sha256":"110ffe88a9a324de7982b6472a48dd9df7b1a72589cb635b70ab4560277c4c6a","availability":"external audit original"},"analysis/condition-audit-2.json":{"sha256":"ace3d0cbc3061f59642243669204dc493a13e913be807507386579d8e9b8593f","availability":"external audit original"},"analysis/source-audit-2.md":{"sha256":"26663a17ea4a8120075605680066aea48ed5e5c15a248eabaff29ed67bdfe522","availability":"external audit original"},"analysis/raw-audit-3.json":{"sha256":"3c5f8c1f0506583723da02d5baf5abb8fbdafe6c6798faace52fff888afe1216","availability":"external audit original"},"analysis/condition-audit-3.json":{"sha256":"792cd2fe0e20e4f2e9ea7e2044d35195b78308a7b1007943fca84f04e527f3d3","availability":"external audit original"},"analysis/source-audit-3.md":{"sha256":"a941e1a0b8ef83ec6a3fb0bd826ed22bb21ccc20a7b7b910a515bf26a12d86be","availability":"external audit original"},"analysis/raw-audit-4.json":{"sha256":"ec5f7df683efde746103d5598d531c01355791d6085f91e64805bae6802e2d35","availability":"external audit original"},"analysis/condition-audit-4.json":{"sha256":"b4ed4d5b6f81c4f2f1f560ae39cee1a810ce9731f19d4bacdca59b9b6ed12b02","availability":"external audit original"},"analysis/source-audit-4.md":{"sha256":"dcd88b3374901ef25d16efc223e3f46077c93c8abef53eb861669c20e6238d3a","availability":"external audit original"},"analysis/raw-audit-5.json":{"sha256":"cb509258450d0fcd09b13365d72b2f4d3de545b831983feca0f91564f0eada56","availability":"external audit original"},"analysis/condition-audit-5.json":{"sha256":"ec0aa32895722b8c50a50d8cc319fd6ee42bac4a850dbcaf64b12edaf1d2363d","availability":"external audit original"},"analysis/source-audit-5.md":{"sha256":"6c79ed9294678c3a1417872bc44b1711a05137d93c04e9822e57f06a8cb4c385","availability":"external audit original"}},"omissions":["Private connection settings, personal paths and identities, chat identifiers, raw session logs","Raw rank calls and full original audits remain outside this repository","Host-specific launchers and frozen dependency bytes are identified by source hashes only"],"extracts_have_separate_hashes":true} diff --git a/evidence/moe-agent-comparison-20261104/plan/publication.json b/evidence/moe-agent-comparison-20261104/plan/publication.json new file mode 100644 index 000000000..42b644edd --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/plan/publication.json @@ -0,0 +1 @@ +{"schema_version":1,"campaign_id":"moe-serial-20261104","status":"frozen","frozen_at_utc":"2026-11-04T15:08:54.987340+00:00","task_id":"moe-shared-ep2-npu-v1","task_contract":"TASK.md and task.json; compute(x, params, *, config, group)","qualification":"Baseline, independent oracle, A/A timing, legitimate optimization and three negative controls reviewed before formal sessions; private preparation evidence is not an optimization source.","rounds_per_session":8,"sessions":[{"key":"gpt61-kb-on","model":"gpt-6.1-sol","thinking":"xhigh","knowledge":"on"},{"key":"gpt61-kb-off","model":"gpt-6.1-sol","thinking":"xhigh","knowledge":"off"},{"key":"gpt56-kb-on","model":"gpt-5.6-sol","thinking":"xhigh","knowledge":"on"},{"key":"gpt56-kb-off","model":"gpt-5.6-sol","thinking":"xhigh","knowledge":"off"}],"execution_order":"fresh sessions run sequentially; prior eight rounds, final and cleanup must complete before the next starts","round_definition":"one immutable candidate plus independent evaluation; incorrect candidates and candidate failures consume rounds","final_definition":"one independent 4-pair remeasurement of a valid explored candidate or the baseline; separate from eight exploration rounds","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_hashes":{"tool/src/inferrsi/__init__.py":"60a710d283ff4f5cfa71c6872163aed4d3899145ad736602d35f88ed6ba4b6f2","tool/src/inferrsi/__main__.py":"935a1c1166b0c1ea35a82256345000bf2c73ded718d77773bc27a71ecce28f7d","tool/src/inferrsi/_skill_selection.py":"acbec7c91bfa05bc2008b939e009fe600a40518051229fb7618c56b092ad36b9","tool/src/inferrsi/arguments.py":"214f84169bcb0ce2006d920909e8e69f8140a66aa2814adb7960948b2faccd96","tool/src/inferrsi/catalog.py":"da5d5b01fed6a4a7027c7249a5514c7fdefbeabd9911846add1f69ae3ce3fbe5","tool/src/inferrsi/claim.py":"5f24c4fa91b48f0bb4d4554531607763fc8fc108e90f332816f06cfb520b5cc1","tool/src/inferrsi/cli.py":"046a78bfeed317ed1007980c6534be7f20e5caf9690e9d2a89c97b3beb2c36bb","tool/src/inferrsi/contracts.py":"03000cb9246ce8c0967147533f7ad25df918ba2674839801453ffb870f496490","tool/src/inferrsi/evaluation.py":"482c40710d428206c59fe3696703248cd86c4b3c7d19ad41017c1369bd6d780e","tool/src/inferrsi/evidence_paths.py":"cc46231e0b8334ef5742defdda20e0093e22b30c86af59df1fb3ae4b5caa70fa","tool/src/inferrsi/execution.py":"f43d8cbb4d43a72a7ff8a10b29a5fb9cf4cb22af54dd35943283c843fba0da92","tool/src/inferrsi/harness/__init__.py":"1006e4713003f44bfb32b2e385a918e1652b9e9833801e4555674a71e57fb0d5","tool/src/inferrsi/harness/benchmarks/__init__.py":"1f4acfa3d93fab2333242a6e31010a00f5fc0d563ce72494bc863be19bc0979f","tool/src/inferrsi/harness/benchmarks/cli.py":"faf930caad63d865677a9a9fc448d240a705261e2948f822d2b56c48d01bb382","tool/src/inferrsi/harness/benchmarks/consumers.py":"1ee18efb35ae43c5aff7867dbc24b4738acdbb02cee3602e2cfd07fd7e8f69ac","tool/src/inferrsi/harness/benchmarks/contract.py":"99a0b45b538b4841d66ed436cf3cbcce1a7b1edd4b6995887cc9aa615838013e","tool/src/inferrsi/harness/benchmarks/npu_task.py":"9af490c8e1f578136631a6ced88ce315bde11c570ff508fdbe89b217aea87bad","tool/src/inferrsi/harness/benchmarks/numpy_task.py":"48cbe2a8b7761f5f9762b23f0926854e4db37a17454512ee68d81bd87034ae0a","tool/src/inferrsi/harness/benchmarks/skill_verifier.py":"7384b45a86a671a72666994656c78cc282e5de466b849fd1fb885afa2086d882","tool/src/inferrsi/harness/benchmarks/worker.py":"21e646888f784f53f465ad9acfb78285ab5b10d7696f41fe62dd9a76096609a9","tool/src/inferrsi/harness/evaluation.py":"20d0c52145600a5d6b1e513b61f2fffaa459b83db194876be40726bd4f343a59","tool/src/inferrsi/harness/improvement.py":"3faa0f5928a707a98f35f45d1ca18e226c898941e6a07b3829eebeb53e70aeb8","tool/src/inferrsi/harness/kv_edit_workbench.py":"bc1f64c8aae39534b8052fdd9a6f3a6b8e63f7a9e9c93cd1077c37ef99cc6f69","tool/src/inferrsi/harness/kv_reclamation.py":"56878c4d1bdda096ac681cadf2d99377bec59d47839802fdf94cfbd9066dc8b9","tool/src/inferrsi/harness/reporting.py":"38c58f02c44e8c8aeacf46527e142be0e3142a83e4b2693246d0ca76c0ddd7c3","tool/src/inferrsi/harness/runner.py":"d290fc87710e77c8e6069e95bb0d347016a393707592d2eed98f07951f2d0865","tool/src/inferrsi/harness/scenarios/__init__.py":"4e97913c806077f0cadfeb29d986d2fa75b55ab8766cb86e6e3e9a195a9a1f5a","tool/src/inferrsi/harness/scenarios/external.py":"f3f8d4fc516ebce81c26d82248e83f79b299eed0fc17ce73e2844995aea9c3f9","tool/src/inferrsi/harness/scenarios/media.py":"67d979a3ed6f13a8e3e1e40a304f78e5a3e6c2556b3aa8a11c41156745d7050a","tool/src/inferrsi/harness/skill_eval_benchmark.py":"f05a265012b840746abf46c9cf681ab2b46f7dfcddd51a14c8832489ca1ef8a9","tool/src/inferrsi/harness/skill_eval_diagnostic.py":"e25cfee943052aaefbddf3f545fe6d56d95ab1edb9723cd9353feeabc954ea9a","tool/src/inferrsi/harness/skill_eval_engineering.py":"0d1f9e6551d11cf9e25eab94035a45ff6dbff6a71f571608ed6d683691c6daa4","tool/src/inferrsi/harness/skill_evaluation.py":"1b64c373170ab10fe9e51c50df1495708cd4fe1b2ca632e4ad11d150f4249c84","tool/src/inferrsi/infra/__init__.py":"8779897733779266841b0f79b84b1dc43f0fa57c013855c906c809829ecfc77c","tool/src/inferrsi/infra/artifacts.py":"bcf3adf325e34e10a4ecb4baf0af855ddca3d77eff76de5b66e493baaa3d7eaf","tool/src/inferrsi/infra/campaign.py":"be505fba1592611bd84ceaca55afbac027205413a2639baefa0c3bb343230209","tool/src/inferrsi/infra/commands.py":"1890f115b3a2c5c6832173bf60432ad595bd6bf1c69783ecd460c6f8961194f1","tool/src/inferrsi/infra/environment.py":"23a6716089c542617761b5a3fc67dde0af9e483212fc003f8577757fbc4ed2db","tool/src/inferrsi/infra/executors/__init__.py":"327c3cd5c821c832ff260a41dba0ee1e439cdabefc9edfb9cad1cd38d14bc87a","tool/src/inferrsi/infra/executors/local.py":"37bcbca9f3261c8c58a8bf806c9da40720b18d0676c82e8ed24c3901ac9bbe77","tool/src/inferrsi/infra/http.py":"f65d6c2b55b6a89694caaf6dcdf7d74f417e9ced103eca1b3f218e305c6b6cb2","tool/src/inferrsi/infra/inspection.py":"d1c66deb0cced529bca996d2f88b60f51fa3417ff1cae365c650fec808946ecd","tool/src/inferrsi/infra/platforms/__init__.py":"dac68eb6edbff995f8afe477396d81efd5db93128978ceedaea1babf0dd3db08","tool/src/inferrsi/infra/platforms/cuda.py":"9e0e4276e1b2984605493b14b82fba3e86f6635204719c318302e624f66f8b6f","tool/src/inferrsi/infra/platforms/mlx.py":"f3b81470db28bd6183ccca2e763c5e079686bba6cdfe6c08ebe2cdf64216f2e3","tool/src/inferrsi/infra/processes.py":"3b96ebc7a50913e78e86ff3dcba74c8ee8d03d547d9591ca6488f390ac1e6189","tool/src/inferrsi/infra/targets/__init__.py":"1f3ea75dded7c6cc967754c7b6d4f30ef00da8191a749938aabc3c6bb8f86eb3","tool/src/inferrsi/infra/targets/git.py":"abb93f9332db336298102e28c06b9a7553597d4d03b957358602f60d61b1aa65","tool/src/inferrsi/inspection.py":"afa44bf4cdfbb58a318c21f9b292ca142d1bbb22750e35a4cc8244da5350ebb3","tool/src/inferrsi/knowledge_operations.py":"3725781bcedb308af6ad9edc0b970c63f82accf6b04ebee4008c1081b63bed9c","tool/src/inferrsi/knowledge_store.py":"70dfca059b0a682579206d4376d191667934b3ca869ae847098c4c6964d86abe","tool/src/inferrsi/model/__init__.py":"3c1dba4f5159fd9fbc8d4d96e2a83c8441491c1b1dc7b42031a3fc1782ed9d9f","tool/src/inferrsi/model/anthropic_agent.py":"284d52dd379b2948dd45ede167175010fe26615e5a307f848efcbe90c0ca5646","tool/src/inferrsi/model/anthropic_edit_agent.py":"3db20467b07657c1a083f7968cfd996bbbffbe04f987bf4283a8d1456f679cf2","tool/src/inferrsi/model/protocols/__init__.py":"5a2f54483b164de04e309dbebb87a700b213682b0f44a50d89a752dd71a4ce47","tool/src/inferrsi/model/protocols/image.py":"251a1f8f885da2cd8e9d3baba766c525afe293091606b4e8668c43ba2cb7cb5f","tool/src/inferrsi/model/protocols/metrics.py":"5b32e296a746b6de30cb0c89f23f730296b0087884c1acc2b6127c1295d4db7a","tool/src/inferrsi/model/request_guard.py":"c2f8a2f822f32785327a2138ee7b0609896b180819d9d1cbe851534aa6198628","tool/src/inferrsi/model/validators/__init__.py":"1d4b6ffa968e283219ddcc8a09de9280238a8d65ae2dc8cd2b60889877a8eb0b","tool/src/inferrsi/model/validators/flare.py":"3696b0609c772f2222b448833608565a033123d7e92b11fbc545dd356fec590f","tool/src/inferrsi/model/validators/image.py":"79470b6b06a22d53ec6c40636a0b3d821a2e6ee2f7bb7aed3dac7241504add1a","tool/src/inferrsi/reporting.py":"1f771890ab3e69b1aa9246b548a7825596bdf904f6c395dd218208084266853d","tool/src/inferrsi/research.py":"9bfe59dd17b5476c18ce50d671711053c62800521c50d6b710e8140a2b4dca04","tool/src/inferrsi/runspec.py":"dca0952425caf42095ef0c7e2feb8ef4740cc422ecb61e777f203962d9b87709","tool/src/inferrsi/skill_catalog.py":"86d465e245b4cd6b72ca175d05124dbb88d510a78568b6bc4c44b332870c08a3","tool/src/inferrsi/summary_chart.py":"d29023d60f6be804c1b01c9eb73765d4824d6c3bb870fe22c99a24544bc1d8bc","runner/device_job.py":"b9741fb92f027ab8e117f92cccb2d533820b8f020bc63c217324bacc98fd843d","runner/evaluate.py":"b85aee68a2bf7afea087e1242d2255c8a7c39ae18043303e3c3366778ae8a9ae","runner/grade.py":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","runner/identity.py":"5f5797d08862b8002f2aa07c895beae0c46ad4074fbbe4fc9a2fbd4f1493457b","runner/process_guard.py":"ca77a5199a14823a7989eeec1bbe6c94dca3f243c31154c091ead42adfca6c4f","runner/deps/libnss_wrapper.so":"f2613646f2aff7718a981700e5c938822eba0211200b21769ff41c85503a197b","runner/deps/copyright.txt":"8ad646968023de9a62351eba9ff206961f62afb5708cfb43d9c35f8262b559af","runner/deps/provenance.json":"2f92d3e9ae28e8ef93224f4f1dae74d438c929d4726f578dd5d080791ab3397e","task/baseline.py":"ee41e4c926a7f0484848012e5bf22a31b3685daa2385c7bfe34a89d75f5568c6","task/oracle.py":"6da20f29449da57d7eac508ea0a5732aacbb49ee9d755b63438a8afb3b4208d6","task/task.json":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","task/TASK.md":"07c4f9915040399a584520e79516ae237d16327c4c8615143fa846aa31a72730","task/workload.py":"4a87ee7ee473de49467d164cb1c0b872a5515fee5a3d656c5bac1c7fe10a4463"},"bundle_lock_sha256":"4e81f8b8d8d6b0aa61470a6abd0f1185cbdca495a2ded8915e77c547209ce682","execution_adapter":"inferrsi.harness.scenarios.external.execute_trial","runtime_observed":{"device":"Ascend910B1","torch":"2.10.0+cpu","torch_npu":"2.10.0.post4","CANN":"9.1.0","python":"3.12.13","matmul_allow_hf32":false},"measurement":{"unit":"ms","scope":"complete forward, host scheduling and all communication included","rank_aggregation":"maximum elapsed for each paired call","case_aggregation":"geometric mean of baseline/candidate per-case median latency ratios","warmups":2,"samples":16,"new_input_each_call":true,"final_pairs":4,"timed_case_ids":["balanced-small","skewed-medium","balanced-large"],"samples_per_pair":8,"exploration_pairs":2,"matched_shape_warmup":{"enabled":true,"calls_per_implementation_per_timed_seed":1,"input":"distinct BF16 values from router-nullspace noise; same selected expert set for every token","noise_std_before_projection":0.3,"minimum_row_rms_input_change":0.15,"minimum_router_boundary_margin":0.02,"max_row_resampling_attempts":32,"verify_all_warm_outputs":true,"require_warm_oracle_outside_timed_output_tolerance":true,"cost":"recorded separately from scored forward latency"}},"tolerance":{"atol":0.03,"rtol":0.03,"nrmse":0.01,"frozen":true},"knowledge_sources_on":["InferRSI fixed-revision skill_library","InferRSI fixed-revision knowledge_packs","InferRSI fixed-revision repo_packs"],"common_sources":["frozen task","shared evaluator operation and necessary interfaces","omni_npu execution skill","public technical documentation"],"knowledge_access_isolation":"unverified; instruction and actual-access audit, not a hard per-session file/plugin sandbox","forbidden_cross_session_sources":["other chats","other candidates","private preflight","coordination analysis and manifest","global optimization skills/plugins/memory"],"collection_recovery":"use --collect-existing for completed device work; never rerun a candidate merely because downloading failed","token_accounting":"actual session cumulative input+output at successful evidence-ready tool return; cached input and reasoning are subsets; failures and recovery included; preparation separate","resource_accounting":"allocation process wall time includes oracle/preparation/warmup and is not NPU busy time","inference_limit":"one eight-round trajectory per condition; descriptive result, no general causal model/knowledge advantage claim","publication_type":"redacted field extract; private launch configuration omitted","allocation":{"gpus":2,"automatic":true,"inner_timeout_seconds":900,"allocation_timeout_seconds":1080,"queue_bound_seconds":1800},"extension":{"registered_at_utc":"2026-11-04T18:45:24.566840+00:00","base_plan_sha256":"fdf792c46212b1a31c7b72923847a6c3716d5b435f580688a4115b03653233b7","base_source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","expected_session_count":6,"existing_session_keys":["gpt61-kb-on","gpt61-kb-off","gpt56-kb-on","gpt56-kb-off"],"rounds_per_session":8,"final_remeasurement":"one independent final using frozen four-pair protocol","execution_order":"Append ON then OFF after the four completed sessions; no overlap.","unchanged":"Task, baseline, oracle, tolerances, source bundle, timing, knowledge access rules and token accounting unchanged.","prompt_change":"Administrative condition count changes to six; prompt notes base plan lists initial four and current model/knowledge are explicitly authorized additions. No prior results or candidate hints supplied.","isolation":"fresh projectless chats, no fork/resume, no cross-session candidates/results, xhigh, no subagents/model switches","interpretation":"Additional model conditions appended after original four; one trajectory each, descriptive comparison, no causal generalization.","additional_conditions":[{"key":"gpt6astra-kb-on","label":"GPT-6 Astra / 知识库开","model":"gpt-6-astra","thinking":"xhigh","knowledge":"on"},{"key":"gpt6astra-kb-off","label":"GPT-6 Astra / 知识库关","model":"gpt-6-astra","thinking":"xhigh","knowledge":"off"}]}} diff --git a/evidence/moe-agent-comparison-20261104/render_overview.py b/evidence/moe-agent-comparison-20261104/render_overview.py new file mode 100644 index 000000000..b6d36e4ea --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/render_overview.py @@ -0,0 +1,175 @@ +"""Replot the published values; English labels need no optional CJK font. + +Requires matplotlib. The historical Chinese PNG is preserved separately; this +portable renderer reproduces its data and three-panel view, not its exact pixels. +""" + +import argparse +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +from matplotlib.lines import Line2D +from matplotlib.ticker import FuncFormatter + +ROOT = Path(__file__).resolve().parent +MODELS = { + "gpt-6.1-sol": ("GPT-6.1 Sol", "#2164b5"), + "gpt-5.6-sol": ("GPT-5.6 Sol", "#12856a"), + "gpt-6-astra": ("GPT-6 Astra", "#d96125"), +} + + +def main(output): + source = ROOT / "results/summary.json" + original = source.read_bytes() + sessions = json.loads(original)["sessions"] + assert len(sessions) == 6 + # Outputs stay outside the frozen bundle. + output = output.resolve() + if output.is_relative_to(ROOT): + raise ValueError("Choose an output prefix outside this frozen evidence bundle") + output.parent.mkdir(parents=True, exist_ok=True) + plt.rcParams.update({"font.family": "DejaVu Sans", "font.size": 11}) + fig = plt.figure(figsize=(15, 11.3), facecolor="white") + grid = fig.add_gridspec( + 2, + 2, + left=0.075, + right=0.975, + top=0.825, + bottom=0.15, + hspace=0.36, + wspace=0.20, + height_ratios=[1, 1.30], + ) + by_round, token_growth, efficiency = ( + fig.add_subplot(grid[0, 0]), + fig.add_subplot(grid[0, 1]), + fig.add_subplot(grid[1, :]), + ) + handles = [] + for session in sessions: + rows = [r for r in session["observations"] if r["role"] == "exploratory"] + final = next(r for r in session["observations"] if r["role"] == "final") + assert [r["round_index"] for r in rows] == list(range(1, 9)) + model, color = MODELS[session["model"]] + enabled = session["knowledge"] == "on" + style, marker = ("-", "o") if enabled else ((0, (5, 3)), "s") + label = f"{model} / KB {'ON' if enabled else 'OFF'}" + rounds = [0] + [r["round_index"] for r in rows] + tokens = [0.0] + [r["cumulative_total_tokens"] / 1e6 for r in rows] + best = [1.0] + [r["best_exploration_speedup"] for r in rows] + kwargs = dict(color=color, linestyle=style, linewidth=2.1, alpha=0.92) + by_round.step(rounds, best, where="post", **kwargs) + token_growth.plot( + rounds, + tokens, + marker=marker, + markersize=3.8, + markerfacecolor=color if enabled else "white", + **kwargs, + ) + efficiency.step(tokens, best, where="post", **kwargs) + for ax, xs in ((by_round, rounds), (efficiency, tokens)): + ax.scatter( + xs[1:], + best[1:], + edgecolors=color, + marker=marker, + s=22, + facecolors=color if enabled else "white", + zorder=4, + ) + efficiency.scatter( + final["cumulative_total_tokens"] / 1e6, + final["effective_speedup"], + marker="D", + s=72, + facecolors=color if enabled else "white", + edgecolors=color, + linewidths=1.8, + zorder=6, + ) + handles.append( + Line2D( + [0], + [0], + color=color, + linestyle=style, + linewidth=2.5, + marker=marker, + markerfacecolor=color if enabled else "white", + label=label, + ) + ) + for ax in (by_round, token_growth, efficiency): + ax.spines[["top", "right"]].set_visible(False) + ax.grid(axis="y", color="#dce2e7", linewidth=0.7) + ax.set_axisbelow(True) + for ax in (by_round, efficiency): + ax.set_ylim(0.95, 2.60) + ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: f"{v:.1f}x")) + ax.set_ylabel("Best valid speedup found") + for ax in (by_round, token_growth): + ax.set_xlim(0, 8.2) + ax.set_xticks(range(9)) + ax.set_xlabel("Consumed optimization rounds") + by_round.set_title("1. Optimization progress", loc="left", fontsize=13, pad=12) + token_growth.set_title("2. Cumulative token usage", loc="left", fontsize=13, pad=12) + token_growth.set_ylabel("Tokens (millions; includes cached input)") + token_growth.set_ylim(bottom=0) + efficiency.set_title( + "3. Search results versus cumulative token usage", loc="left", fontsize=13, pad=12 + ) + efficiency.set_xlabel("Cumulative tokens through evaluation completion (millions)") + efficiency.set_xlim(left=0) + fig.suptitle( + "Six configurations: optimization progress and token usage", + x=0.075, + y=0.985, + ha="left", + fontsize=20, + fontweight="bold", + ) + fig.text( + 0.075, + 0.949, + "Two-NPU MoE / eight rounds each / solid-filled: KB ON; dashed-hollow: KB OFF", + fontsize=11, + color="#52606c", + ) + fig.legend( + handles=handles, + loc="upper left", + bbox_to_anchor=(0.07, 0.94), + ncols=3, + frameon=False, + fontsize=10.5, + columnspacing=3, + ) + notes = ( + "Steps/dots: best valid exploration result. Diamonds: independent final; " + "its cost is included, its result does not rewrite the curve.", + "Both Astra R4 failures consume rounds/tokens; best stays unchanged. " + "Report-tail tokens are excluded. Speedups use paired task baselines.", + "One trajectory per condition; tokens include cached input and are not money or wall time. " + "Astra ON queued 41.42 min; OFF ~1.28 sec.", + ) + for y, note in zip((0.091, 0.066, 0.041), notes): + fig.text(0.075, y, note, fontsize=9, color="#46505a") + fig.savefig(output.with_suffix(".png"), dpi=180, bbox_inches="tight") + plt.close(fig) + assert source.read_bytes() == original + print("Reproduced the public numerical curves; original data and historical PNG unchanged.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output", type=Path, required=True, help="Output prefix outside the evidence bundle" + ) + main(parser.parse_args().output) diff --git a/evidence/moe-agent-comparison-20261104/results/audit-extract.json b/evidence/moe-agent-comparison-20261104/results/audit-extract.json new file mode 100644 index 000000000..01d724be5 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/results/audit-extract.json @@ -0,0 +1 @@ +{"type":"selected fields from original audits","attempts":[{"session_key":"gpt61-kb-on","role":"exploratory","round_index":1,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0372922489136862,"1":1.036135375804341},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-on","role":"exploratory","round_index":2,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.635356715703455,"1":1.6615028968309808},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-on","role":"exploratory","round_index":3,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.5934444869462097,"1":1.535255992090488},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-on","role":"exploratory","round_index":4,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.273010478692503,"1":2.25271491856126},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-on","role":"exploratory","round_index":5,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.2897780628536974,"1":2.2335122508195417},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-on","role":"exploratory","round_index":6,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.1630114006030814,"1":2.2881688995086016},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-on","role":"exploratory","round_index":7,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.2015687275313636,"1":2.1305407377558776},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-on","role":"exploratory","round_index":8,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.2208267002602198,"1":2.2038369111878127},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-on","role":"final","round_index":null,"candidate_outcome":"PASS","expected_calls_per_rank":424,"timed_samples_per_label_per_case":32,"max_rank_samples_checked":192,"pair_geometric_speedups":{"0":2.1214635856241166,"1":2.2008629933193586,"2":2.1856526171313915,"3":2.2336260288902823},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":424,"timed_samples":192,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.012123823165893555},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.012123823165893555}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6297163316486991},{"rank":1,"calls":424,"timed_samples":192,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6292225951167351}]},{"session_key":"gpt61-kb-off","role":"exploratory","round_index":1,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0064051937484468,"1":1.023386133393224},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-off","role":"exploratory","round_index":2,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.415000172436954,"1":1.5505269478699448},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-off","role":"exploratory","round_index":3,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.1483147311299193,"1":2.142405826508603},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-off","role":"exploratory","round_index":4,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.1956682406408716,"1":2.3005003747043538},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-off","role":"exploratory","round_index":5,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.354132491377926,"1":2.3132493883609984},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-off","role":"exploratory","round_index":6,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.464092254701238,"1":2.477394289301688},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-off","role":"exploratory","round_index":7,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.3361649823508484,"1":2.3693848909826105},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-off","role":"exploratory","round_index":8,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.4264979895212933,"1":2.389801574017784},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt61-kb-off","role":"final","round_index":null,"candidate_outcome":"PASS","expected_calls_per_rank":424,"timed_samples_per_label_per_case":32,"max_rank_samples_checked":192,"pair_geometric_speedups":{"0":2.3652697240433453,"1":2.2772689593588606,"2":2.3376838982396446,"3":2.424677523181434},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":424,"timed_samples":192,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.012123823165893555},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.012123823165893555}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6297163316486991},{"rank":1,"calls":424,"timed_samples":192,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6292225951167351}]},{"session_key":"gpt56-kb-on","role":"exploratory","round_index":1,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0257324369140148,"1":1.0333132331145571},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-on","role":"exploratory","round_index":2,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0280413287259222,"1":1.0476993265664332},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-on","role":"exploratory","round_index":3,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0105358446574206,"1":1.015651298388978},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.00437128360309782,"max_abs_error":0.016501426696777344}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.004391511916753598,"max_abs_error":0.016718626022338867}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-on","role":"exploratory","round_index":4,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0546878762856318,"1":1.0165907491401718},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-on","role":"exploratory","round_index":5,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0287830246932697,"1":1.0462132355692522},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-on","role":"exploratory","round_index":6,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":0.9882867773140426,"1":0.9625304854829492},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.012517929077148438}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.004073423632446665,"max_abs_error":0.016718626022338867}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-on","role":"exploratory","round_index":7,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0953888898164406,"1":1.0714094998138355},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-on","role":"exploratory","round_index":8,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.1021873623180574,"1":1.0753737751100434},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-on","role":"final","round_index":null,"candidate_outcome":"PASS","expected_calls_per_rank":424,"timed_samples_per_label_per_case":32,"max_rank_samples_checked":192,"pair_geometric_speedups":{"0":1.0618925841608347,"1":1.0953619736568139,"2":1.0947018057088234,"3":1.09486988634476},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":424,"timed_samples":192,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.012123823165893555},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.012123823165893555}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6297163316486991},{"rank":1,"calls":424,"timed_samples":192,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6292225951167351}]},{"session_key":"gpt56-kb-off","role":"exploratory","round_index":1,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0572462564929241,"1":1.0249002975729893},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-off","role":"exploratory","round_index":2,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0369949329575512,"1":1.0242736481827572},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-off","role":"exploratory","round_index":3,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0058830281407491,"1":0.9870546097725549},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.00437128360309782,"max_abs_error":0.016501426696777344}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.004391511916753598,"max_abs_error":0.016718626022338867}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-off","role":"exploratory","round_index":4,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.1496850786841444,"1":1.0279361230433195},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-off","role":"exploratory","round_index":5,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.159051831491158,"1":1.1816803002659038},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-off","role":"exploratory","round_index":6,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.1977926125306166,"1":1.2179253294883694},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-off","role":"exploratory","round_index":7,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.1155208456807542,"1":1.171986089362685},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-off","role":"exploratory","round_index":8,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.214248121713004,"1":1.1685798698798904},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt56-kb-off","role":"final","round_index":null,"candidate_outcome":"PASS","expected_calls_per_rank":424,"timed_samples_per_label_per_case":32,"max_rank_samples_checked":192,"pair_geometric_speedups":{"0":1.2078107867766052,"1":1.1980181126741738,"2":1.2009419756915276,"3":1.1994723337196675},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":424,"timed_samples":192,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.012123823165893555},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.012123823165893555}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6297163316486991},{"rank":1,"calls":424,"timed_samples":192,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6292225951167351}]},{"session_key":"gpt6astra-kb-on","role":"exploratory","round_index":1,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0350802035470974,"1":1.02123322733489},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-on","role":"exploratory","round_index":2,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.643732248380617,"1":1.68483398400962},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-on","role":"exploratory","round_index":3,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.0884091128796443,"1":2.1219336381505367},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-on","role":"exploratory","round_index":4,"candidate_outcome":"FAIL","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":0,"audit_scope":"failed_candidate_prefix","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":1,"timed_samples":0,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.0038475058395686192,"max_abs_error":0.006382942199707031},"candidate":{"nrmse":null,"max_abs_error":null}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":1,"timed_samples":0,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003858346625688875,"max_abs_error":0.007839083671569824},"candidate":{"nrmse":null,"max_abs_error":null}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-on","role":"exploratory","round_index":5,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.2311998135934923,"1":2.24975558821954},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-on","role":"exploratory","round_index":6,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.1340000445589604,"1":2.13538814119884},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-on","role":"exploratory","round_index":7,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.4668217279636733,"1":2.367896851250557},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-on","role":"exploratory","round_index":8,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.3081234125182113,"1":2.411879142591842},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-on","role":"final","round_index":null,"candidate_outcome":"PASS","expected_calls_per_rank":424,"timed_samples_per_label_per_case":32,"max_rank_samples_checked":192,"pair_geometric_speedups":{"0":2.36482032630349,"1":2.3867478391008956,"2":2.299032881059286,"3":2.3793748786329156},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":424,"timed_samples":192,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.012123823165893555},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.012123823165893555}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6297163316486991},{"rank":1,"calls":424,"timed_samples":192,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6292225951167351}]},{"session_key":"gpt6astra-kb-off","role":"exploratory","round_index":1,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.0278494475288495,"1":1.0317833855062282},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-off","role":"exploratory","round_index":2,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":1.6689432461897238,"1":1.650582006954206},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-off","role":"exploratory","round_index":3,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.3444988455565587,"1":2.444790720573977},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-off","role":"exploratory","round_index":4,"candidate_outcome":"FAIL","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":0,"audit_scope":"failed_candidate_prefix","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":1,"timed_samples":0,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.0038475058395686192,"max_abs_error":0.006382942199707031},"candidate":{"nrmse":null,"max_abs_error":null}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":1,"timed_samples":0,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003858346625688875,"max_abs_error":0.007839083671569824},"candidate":{"nrmse":null,"max_abs_error":null}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-off","role":"exploratory","round_index":5,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.314729367990668,"1":2.315008771219002},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-off","role":"exploratory","round_index":6,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.290882880916215,"1":2.384674858549901},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-off","role":"exploratory","round_index":7,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.5021787419406136,"1":2.4085330944636683},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-off","role":"exploratory","round_index":8,"candidate_outcome":"PASS","expected_calls_per_rank":232,"timed_samples_per_label_per_case":16,"max_rank_samples_checked":96,"pair_geometric_speedups":{"0":2.2340014856194146,"1":2.342582485658013},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.011693239212036133}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6354638115257412},{"rank":1,"calls":232,"timed_samples":96,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6316555021522049}]},{"session_key":"gpt6astra-kb-off","role":"final","round_index":null,"candidate_outcome":"PASS","expected_calls_per_rank":424,"timed_samples_per_label_per_case":32,"max_rank_samples_checked":192,"pair_geometric_speedups":{"0":2.452320780791441,"1":2.307327094761081,"2":2.353363859618004,"3":2.3853258861028435},"audit_scope":"complete_pass","audit_status":"PASS","frozen_source_files_verified":79,"cleanup_verified":true,"ranks":[{"rank":0,"calls":424,"timed_samples":192,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.004102601501173023,"max_abs_error":0.012123823165893555},"candidate":{"nrmse":0.004102601501173023,"max_abs_error":0.012123823165893555}},"minimum_warm_input_rms_difference":0.2662879228591919,"minimum_warm_output_nrmse_difference":0.6297163316486991},{"rank":1,"calls":424,"timed_samples":192,"local_failed_calls":0,"max_errors_by_label":{"baseline":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367},"candidate":{"nrmse":0.003989958571102418,"max_abs_error":0.012614011764526367}},"minimum_warm_input_rms_difference":0.2669099271297455,"minimum_warm_output_nrmse_difference":0.6292225951167351}]}]} diff --git a/evidence/moe-agent-comparison-20261104/results/changes.md b/evidence/moe-agent-comparison-20261104/results/changes.md new file mode 100644 index 000000000..2fd33f65f --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/results/changes.md @@ -0,0 +1,89 @@ +# 逐轮尝试了什么 + +以下为实际源码变化的审查摘要，不是逐组件性能归因。精确数值、失败和成本见 +`rounds.csv`；所有快照见 `../patch/candidates.csv`。每行仍计一轮，分叉不会清除成本。 + +## GPT-6.1 Sol / ON（final 选 R5） + +| 轮次 | 实际变化 | +| --- | --- | +| R1 | 省去 logits AllGather，收集 BF16 输入后重算 FP32 router。 | +| R2 | 一次筛选/排序打包本地路由，两次 grouped GEMM、一次 SwiGLU 与 FP32 index_add。 | +| R3 | shared 并入第五组 GMM；较慢，后续撤回。 | +| R4 | 从 R2 分叉，改原生 init_routing_v2；主机读取 count 后截有效前缀，手工 FP32 combine。 | +| R5 | FP32 finalize 融合归并；非本地映射指向显式补零行。 | +| R6 | 从 R5 分叉，异步 ReduceScatter 期间执行 shared，显式 wait；未改善总分。 | +| R7 | 从 R5 分叉，安全缓存连续转置专家权重；未改善总分。 | +| R8 | 从 R5 分叉，无缓存；shared gate/up+SwiGLU 与 AllGather 交错，down 与 ReduceScatter 交错；未改善总分。 | + +## GPT-6.1 Sol / OFF（final 选 R6） + +| 轮次 | 实际变化 | +| --- | --- | +| R1 | 省去 logits AllGather，收集输入后重算 router。 | +| R2 | 一次 nonzero、专家排序和设备计数前缀，两次 grouped GEMM。 | +| R3 | 原生路由；主机读取有效 count 并切片，手工 FP32 逆映射/归并。 | +| R4 | FP32 finalize，补齐路由容量，掩蔽非本地概率。 | +| R5 | 跳过等长输入的无用 pad；shared 与异步 ReduceScatter 交错并 wait。 | +| R6 | 静态路由容量与设备端专家前缀，移除 count 主机读取/动态切片；显式清零无效尾部。 | +| R7 | 从 R6 尝试融合 gating，并对选中概率再次归一化；未改善总分。 | +| R8 | 从 R6 分叉，恢复普通 topk/softmax；shared gate/up 与 AllGather 交错，SwiGLU+down 与 ReduceScatter 交错。 | + +## GPT-5.6 Sol / ON（final 选 R8） + +| 轮次 | 实际变化 | +| --- | --- | +| R1 | 省去独立 logits AllGather，收集后重算 router。 | +| R2 | shared 与异步 AllGather 交错，保留逐专家循环。 | +| R3 | 尝试 BF16 routed 归并/通信路径。 | +| R4 | 恢复 FP32 routed 路径，改为 shared 与 ReduceScatter 交错。 | +| R5 | 拆分 shared，尝试两段通信交错。 | +| R6 | 把专家 ID、概率打包后随输入通信。 | +| R7 | 撤回打包；padding 专家 ID 设为 -1，简化逐专家 mask。 | +| R8 | 跳过无用 pad；最终 FP32 加权、累加及 ReduceScatter，shared 与 AllGather 交错。 | + +## GPT-5.6 Sol / OFF（final 选 R8） + +| 轮次 | 实际变化 | +| --- | --- | +| R1 | 省去 logits AllGather，收集后重算 router。 | +| R2 | shared 与异步 AllGather 交错。 | +| R3 | 尝试 BF16 routed 路径。 | +| R4 | 两次 torch.bmm 批量执行四个本地专家。 | +| R5 | 统一打包专家路由与回写输出。 | +| R6 | shared 与异步 ReduceScatter 交错并显式等待。 | +| R7 | 尝试 topk(sorted=False)。 | +| R8 | 恢复有序 top-k；one_hot+cumsum 替代 argsort 打包；仍有最大 count 主机读取与补零。 | + +## GPT-6 Astra / ON（final 选 R7） + +| 轮次 | 实际变化 | +| --- | --- | +| R1 | 省去 logits 通信，shared 与异步 AllGather 交错。 | +| R2 | grouped GEMM 批量专家计算。 | +| R3 | 原生路由，读取 count 并截有效前缀。 | +| R4 | 尝试 finalize，但额外 k 参数不受运行时支持；候选 FAIL，无计时分数。 | +| R5 | 删除 k 参数，FP32 finalize 正确性/测量通过。 | +| R6 | shared 第五组 GMM 与连续权重缓存尝试；未入选。 | +| R7 | 基于 R5，撤回缓存；去无用 pad，非空分支直接使用 finalize 输出。 | +| R8 | 改为 shared 与 ReduceScatter 交错；未超过 R7。 | + +## GPT-6 Astra / OFF（final 选 R3） + +| 轮次 | 实际变化 | +| --- | --- | +| R1 | 省去 logits 通信，shared 与异步 AllGather 交错。 | +| R2 | grouped GEMM 批量专家计算。 | +| R3 | 静态容量原生路由、设备端专家前缀、手工逆映射/where 清零/FP32 combine。 | +| R4 | 尝试 finalize；额外 k 参数不受运行时支持，候选 FAIL，无计时分数。 | +| R5 | 删除 k 参数修复，但分数低于 R3。 | +| R6 | 从 R3 分叉，改为 shared 与 ReduceScatter 交错。 | +| R7 | 从 R3 分叉，去无用 pad/int64 索引转换。 | +| R8 | 从 R3 分叉，缓存连续专家权重；未超过 R3。 | + +静态容量 GMM 的接受范围是本次 Ascend910B1 / CANN 9.1.0：依据有效前缀语义、 +明确处理未初始化尾部，并通过冻结用例。不能把旧 Python 包装文档与底层接口 +的差异概括成任意版本均兼容。相关公开依据是 CANN `ops-transformer` v9.1.0 +提交 `c77c0829f1f0f0b8f882db1d9b051a758b605472` 的 +[GroupedMatmulV4 文档](https://github.com/hicann/ops-transformer/blob/c77c0829f1f0f0b8f882db1d9b051a758b605472/gmm/grouped_matmul/docs/aclnnGroupedMatmulV4.md)。 +缓存尝试的参数变化用例不单独覆盖同一已缓存对象的所有失效分支；六份入选 final 均未使用缓存。 diff --git a/evidence/moe-agent-comparison-20261104/results/comparison-overview.png b/evidence/moe-agent-comparison-20261104/results/comparison-overview.png new file mode 100644 index 000000000..e39ed5032 Binary files /dev/null and b/evidence/moe-agent-comparison-20261104/results/comparison-overview.png differ diff --git a/evidence/moe-agent-comparison-20261104/results/measurements.jsonl.gz b/evidence/moe-agent-comparison-20261104/results/measurements.jsonl.gz new file mode 100644 index 000000000..18b6d5617 Binary files /dev/null and b/evidence/moe-agent-comparison-20261104/results/measurements.jsonl.gz differ diff --git a/evidence/moe-agent-comparison-20261104/results/rounds.csv b/evidence/moe-agent-comparison-20261104/results/rounds.csv new file mode 100644 index 000000000..f816adff6 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/results/rounds.csv @@ -0,0 +1,55 @@ +session_key,role,round_index,candidate_sha256,task_sha256,grader_sha256,source_revision,source_bundle_sha256,correctness,valid_measurement,aggregation,speedup,effective_speedup,best_exploration_speedup,cleanup_verified,queue_seconds,token_boundary_status,cumulative_input_tokens,cumulative_cached_input_tokens,cumulative_output_tokens,cumulative_total_tokens,cumulative_uncached_input_tokens,cumulative_reasoning_output_tokens,round_input_tokens,round_cached_input_tokens,round_output_tokens,round_total_tokens,round_uncached_input_tokens,allocation_process_seconds,original_observation_sha256,original_device_summary_sha256,attempt,round_consumed,execution_status,performance_verdict,failure_reason +gpt61-kb-on,exploratory,1,0d0f392255b28699940db896c4eafe9a859e587dd7d7bacbbe68859ebbffb383,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.033496152864355,1.033496152864355,1.033496152864355,True,0.1506795883178711,verified,1405745,1289344,8446,1414191,116401,2089,1405745,1289344,8446,1414191,116401,191.8611934185028,739e85da1ae0ca7ba4c7226bcfc77c61b35662d4288bd4f9c7cec9ad7a239b59,2debed672659e2dfae6da2cd32dc282778a2ff5f7f9f19c881e6bdee96daee5b,1,True,completed,PASS, +gpt61-kb-on,exploratory,2,2657e679b9f374d6e0aee01c889cf69cf23d7a2354ca58921ba731052cd3f452,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.659926470855875,1.659926470855875,1.659926470855875,True,0.14699196815490723,verified,2328374,2200192,11862,2340236,128182,3593,922629,910848,3416,926045,11781,194.22847890853882,18c0c93c82a2d221f21da320787c89fcf85528ecc87627991a7c4d86c47e434f,fcbf89354bad6b1ae875b48eaa23a7d1cf32202234839d37bf3493df858c4539,1,True,completed,PASS, +gpt61-kb-on,exploratory,3,393a0cd1b4affdd79ab49920a4a6724f2b53d1f7ce2d45efe585874247d04c0a,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.5610369207444954,1.5610369207444954,1.659926470855875,True,0.146392822265625,verified,3183450,3047808,15636,3199086,135642,5510,855076,847616,3774,858850,7460,188.41365265846252,99b1bd4c5d97914b77d83ec7c39d8ffd8a265eb002cea5f56bbc3ea460c11922,47026e89884a10839d1709e358917521391e0c4c1417e28f59566ba8ef631f1e,1,True,completed,PASS, +gpt61-kb-on,exploratory,4,8fc66ba373f034b851d292a7e26097dcf814307b2c922c9fb1a732b931b95d68,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.2556118202102673,2.2556118202102673,2.2556118202102673,True,0.14520955085754395,verified,4083585,3944192,19566,4103151,139393,7135,900135,896384,3930,904065,3751,192.46062564849854,59a6d3be1d248e2b51a89880fbda419ee5413520a0d15532e08fb7188abb671d,b9431a4a2cd60458c55a3624662a1e7b23a4d4d076a30005266c0ab1a79f16ac,1,True,completed,PASS, +gpt61-kb-on,exploratory,5,507d265ffde8d7b1d3348e37c46ab4f696503e0dafbfef1bbbb7cd3606e25d0e,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.256558755529666,2.256558755529666,2.256558755529666,True,0.1504838466644287,verified,5013219,4871168,22736,5035955,142051,8858,929634,926976,3170,932804,2658,195.5376443862915,4bc6d6ad521dfcd4d66228d6779ec408280161fcb7f60053c578728fb740644e,ff0459525727a3fb9a68ddb45f472480a4326d06d0545463c140cd6af012f2d7,1,True,completed,PASS, +gpt61-kb-on,exploratory,6,81a9b68ef4108e90279e99e9a323f7dba8ecdd211c909a67b115225adceb26d7,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.2407286057764324,2.2407286057764324,2.256558755529666,True,0.14667773246765137,verified,5819085,5674496,30079,5849164,144589,10313,805866,803328,7343,813209,2538,205.60196185112,a1fd2af2d94a2d6f28131fdd439617caece1ad3f9cd411b478c97566f4d8db75,e4889458cb398ea385e9128ce8b8efc76a01927274841e2cf07720180aaf652f,1,True,completed,PASS, +gpt61-kb-on,exploratory,7,0cf406d0472202fa9ac35390ab28993194e512c2414696a1840e19a4efc304fa,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.1809720767812473,2.1809720767812473,2.256558755529666,True,0.14430689811706543,verified,7021441,6870144,33863,7055304,151297,12188,1202356,1195648,3784,1206140,6708,206.1087031364441,773faf5fddc5753a5dd0609fbb191448b56c5cb3eb2d16f639abc9b14b0c63e3,6bcda80b83c0078c84074c4586ebd924727a10e93f43b0085b75f0cf4b06524b,1,True,completed,PASS, +gpt61-kb-on,exploratory,8,c896e3e99aa4be3789526c41f74ca5bf34fa310a5532407638e12df13de4e832,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.2291567851269463,2.2291567851269463,2.256558755529666,True,0.14380669593811035,verified,8093276,7939456,38542,8131818,153820,13736,1071835,1069312,4679,1076514,2523,205.95839715003967,955b09390f31bfb3cfc776f80d9dc81e3da30e068e7cd9d632438b8d93d14ffe,78b2e02b9ed3abd4700d737aae22e567edcf1d53366d6985e5b3544cc50cb3fa,1,True,completed,PASS, +gpt61-kb-on,final,,507d265ffde8d7b1d3348e37c46ab4f696503e0dafbfef1bbbb7cd3606e25d0e,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.2163455443832887,2.2163455443832887,2.256558755529666,True,0.13953661918640137,verified,9761936,9604096,43495,9805431,157840,16720,1668660,1664640,4953,1673613,4020,357.989972114563,54d6271b0b286ff028d6264c039d70b8bc8c4ca5d661ee4264777667b32b52e2,b677794e3267a76b4a215d051d5e46b11807a11880bd106e3ebeaec4f6ce62af,1,True,completed,PASS, +gpt61-kb-off,exploratory,1,74c9cbb8ac2be1b9aac1f45625d2bb77fa951c5e65cc0d3e0e7900db87ca2a42,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.0182446674027414,1.0182446674027414,1.0182446674027414,True,0.14409542083740234,verified,1028458,955264,7669,1036127,73194,2314,1028458,955264,7669,1036127,73194,201.93721413612366,2876e1a4b080b8cdf73080528b1becc7cdef77cbc8a0769187285030d0e55def,36bde9b63e044dde30ad10782884ead189654dff5b09d26eb541277eb26dd5c5,1,True,completed,PASS, +gpt61-kb-off,exploratory,2,f163d015875b80f90d67b8f50dd246f59fb937299ea2a43f0279036f574e1437,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.5011271720808808,1.5011271720808808,1.5011271720808808,True,0.14340543746948242,verified,1930173,1844224,11946,1942119,85949,4209,901715,888960,4277,905992,12755,205.39305973052979,bcc18426b029e631aa0803444f8e840805c2b86d2715554cefed2c200e6489ad,f6a2a5ad8d174df2e566cef5a7fc4b6cf8bf1344dcdcab339deff6fdf07ac512,1,True,completed,PASS, +gpt61-kb-off,exploratory,3,9cdee85e0b487703a92a85a3d4d78ad3bd11749b434dbae00ece0f03fc92db08,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.1573718369619095,2.1573718369619095,2.1573718369619095,True,0.14728975296020508,verified,3035590,2943232,16010,3051600,92358,6084,1105417,1099008,4064,1109481,6409,199.14277029037476,f316eadd7d1a68d637a721e33be315feeca8b814f610d2de91dd387825908a0c,c9007cbea914654b6aca79c1b96b4f29217c8e57f94ebaa33c18643d5cd6ff0f,1,True,completed,PASS, +gpt61-kb-off,exploratory,4,10d7bd7d74c147f476e050bd5e23e347dd51096d83575e802d13d150df923f4c,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.239197543098337,2.239197543098337,2.239197543098337,True,0.14458942413330078,verified,4022724,3920512,21880,4044604,102212,7349,987134,977280,5870,993004,9854,201.5980749130249,48839c99a5f325382e2512d54a628f63b6446d09b6aee4c4ef7c0341ebc207a4,a51911a69e4b83fb3336ef357753ea392dd5bf5f9b7b722a8a1712a2b31cd9e0,1,True,completed,PASS, +gpt61-kb-off,exploratory,5,69efab972a84228f7e31cf827e615b23777316a0586cc0624214edea8e64e2eb,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.329806968277373,2.329806968277373,2.329806968277373,True,0.15340185165405273,verified,4946814,4840448,26126,4972940,106366,10543,924090,919936,4246,928336,4154,201.4565303325653,238743d9d09c7dc19fc72b6b91a8a301f07b45618dc7006f72bcdefeab7e555b,57586c82e3ad83be02dd8839195cb0eb1f2341556a617e131d9454df886a0338,1,True,completed,PASS, +gpt61-kb-off,exploratory,6,ac521b38f2555639bcde827f35b9b205767136b91ccf170e3a3f72c1c06de4f2,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.4630234063840075,2.4630234063840075,2.4630234063840075,True,0.13998985290527344,verified,6208420,6097152,30212,6238632,111268,12160,1261606,1256704,4086,1265692,4902,202.2952857017517,93cf7c3b2b0a7a27982b7a4e57152195cfb9457ccd8e7cf92b3a070da4bc4fd9,6707e81ae878abdc28cfef8bbe0517fc11635ae154d9a24af4b5dc24fd9d707a,1,True,completed,PASS, +gpt61-kb-off,exploratory,7,67bb00dd912136041066cbcb1949ff2f9b5397be54973dffca81dc55ad4b0b17,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.3718703757080677,2.3718703757080677,2.4630234063840075,True,0.14416289329528809,verified,7236698,7121152,33972,7270670,115546,14653,1028278,1024000,3760,1032038,4278,203.76398253440857,bcf0ccdfa1cff73fac6045404a45f584f8a652952575f1f60df6252f0d58f092,d1402d982f47bcdec8f9055a65c80656ca9ff3182c01325ee18d65b5458f16e1,1,True,completed,PASS, +gpt61-kb-off,exploratory,8,36a9ac623ee1c3c2cf0717d69bd48d5915edbd4a828832a1693b83bc2632ccc7,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.4228360401116964,2.4228360401116964,2.4630234063840075,True,0.14646649360656738,verified,8162262,8042368,39950,8202212,119894,17771,925564,921216,5978,931542,4348,199.70272850990295,8f76eccfae220cdeaacffe5cbdcaa07319c4ed4d99cc4a6c6024a0fea4aae57e,cf1d62a8a8cc68d64ab4aed7e6b393eeda515f8b324252efe74d47b24f59f842,1,True,completed,PASS, +gpt61-kb-off,final,,ac521b38f2555639bcde827f35b9b205767136b91ccf170e3a3f72c1c06de4f2,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.3806962845718207,2.3806962845718207,2.4630234063840075,True,0.14612269401550293,verified,9819788,9693056,48207,9867995,126732,23456,1657526,1650688,8257,1665783,6838,358.9751510620117,d5d049f08d189e2108545ce3e483214e3e7df82a295aeec7ee14e915d8a73101,7371e777b9e9e4f905d2e27a246e7c44761a66c14a2be5378f4958c6c8f2dd8b,1,True,completed,PASS, +gpt56-kb-on,exploratory,1,aedc1c7d99e2a5615f18443acdccdd1285befbcfd341a7aedb25586718e9e354,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.033127728274557,1.033127728274557,1.033127728274557,True,0.14676356315612793,verified,2616718,2519552,18360,2635078,97166,6208,2616718,2519552,18360,2635078,97166,195.71978950500488,47b94dbbe64d854b8004ca89b27865f829eea05896dc9abbeaf8ef12585fa613,91c89b577df42e8f8cecd24aba9a18cc9b3a3fd4ee60bb68dfe8df9648d318e3,1,True,completed,PASS, +gpt56-kb-on,exploratory,2,e9324fb9b5def09b0313528a516b39cf5bdcec1a5cb8f77c62a07800bc18955f,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.043939426200217,1.043939426200217,1.043939426200217,True,0.14348196983337402,verified,3757287,3656064,21779,3779066,101223,6646,1140569,1136512,3419,1143988,4057,201.93050742149353,8351a4b80b07b9c9989878ad3039bdc75019d4a5c61c40551dd81d6dec043bc9,82d6fd682b26b15afe4336b6b8868149dffd796d581e209cce0b4826b8d65b48,1,True,completed,PASS, +gpt56-kb-on,exploratory,3,464916a52eacbed262279c369da3ecdf04c4c862a0ab8f44bac1ca6368a4d862,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.0103050146988586,1.0103050146988586,1.043939426200217,True,0.14808988571166992,verified,4953713,4848128,24995,4978708,105585,6873,1196426,1192064,3216,1199642,4362,197.9350609779358,9be3800c4e258ec492e532235be10a1d8f9765c5d3cdebd2f415c1cb336c52f5,0d2478981077c1359872906b5b64d30262d2481f0c3a4eae37f682e64c0a975e,1,True,completed,PASS, +gpt56-kb-on,exploratory,4,3f14efc39ace2ba129a048214d66eec2f624f522bc4b9baa76b509bad188f3ca,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.0367692061223202,1.0367692061223202,1.043939426200217,True,0.1413097381591797,verified,6717756,6606848,28783,6746539,110908,7573,1764043,1758720,3788,1767831,5323,204.3094515800476,5233740a6750673eb5e03e85d94ea70538affcb0e54946285343f0dd6c27d1e3,ae786be0177c9c3a268a8881f127ece09d85b6b81fa4f1667f9c2e075ac80be3,1,True,completed,PASS, +gpt56-kb-on,exploratory,5,8d52e2f95ae28b6beeaba607868a492a5a2ab2b647d806d19e8155ff1b94c9cf,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.0310284033167278,1.0310284033167278,1.043939426200217,True,0.14885640144348145,verified,8170548,8055424,31938,8202486,115124,8424,1452792,1448576,3155,1455947,4216,203.10643601417542,238d7a4d23f2312e2c74edb82c8d5f1d66fe38741b2f807dc3f1b99c7a1aec82,b2b21cb73409fd95efd736e96db992b11a727edc590d899748f3ce6bbcbdeb74,1,True,completed,PASS, +gpt56-kb-on,exploratory,6,a3762e1d73ef5b4f1e672b727e708b4d12fa6d2be53a1c8755db3c8829c80348,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,0.9764517576218519,0.9764517576218519,1.043939426200217,True,0.1410059928894043,verified,9844322,9723264,35964,9880286,121058,9345,1673774,1667840,4026,1677800,5934,200.8367359638214,4a624e9dc12f47e5793689b91288c257783e2cdb49a93aa2f339dd2978867a8c,3768ecb3a40da487e0d90fe9165297131230c8819b4b4d28f6212c70d8a1b8a7,1,True,completed,PASS, +gpt56-kb-on,exploratory,7,7a3080fb599fbe18f58bc2c86f558740d4843e95e72b189d8b790f7460128483,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.0825436212364279,1.0825436212364279,1.0825436212364279,True,0.14468598365783691,verified,11454616,11328128,39156,11493772,126488,9933,1610294,1604864,3192,1613486,5430,196.1109836101532,923bd1ba2055ce7b4231daec0e443f815650cb8500fa57f81bf5d2ef9b4911cd,5de5918b12bd37be5576cb77f10366beb9db20f1c575008b80b463ca54937774,1,True,completed,PASS, +gpt56-kb-on,exploratory,8,4815b5f5fbae5bcf2f357ab51196301a57330e8ee391ef530fc9324202564f4e,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.0913441544227527,1.0913441544227527,1.0913441544227527,True,0.1469707489013672,verified,12974249,12843648,42115,13016364,130601,10983,1519633,1515520,2959,1522592,4113,198.73881268501282,c8acc9bb4259569ae055ba893c8fcfd7b58fadd95e185c162d899035d36fbfb5,57aab0e0c179999cb81fcfd0c9caddc4aa75662f955289bae9d7afadcf6917ee,1,True,completed,PASS, +gpt56-kb-on,final,,4815b5f5fbae5bcf2f357ab51196301a57330e8ee391ef530fc9324202564f4e,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.0923856414422102,1.0923856414422102,1.0913441544227527,True,0.14021706581115723,verified,14689589,14554880,43331,14732920,134709,11085,1715340,1711232,1216,1716556,4108,351.495703458786,f84bf94cbff5b94e117172e842ba8b0392bcb1d3963b8bb9f63ff4d64da86bfd,83ec9b4de116fa5394fde6d46e6471b74928da287ebcbaac0f3f9ec33891ef79,1,True,completed,PASS, +gpt56-kb-off,exploratory,1,b868abaac4b60f655b9360c2af5276ba53986c2c5da9e13b029685431244edb6,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.041752689517352,1.041752689517352,1.041752689517352,True,0.13953709602355957,verified,494182,459520,5765,499947,34662,2360,494182,459520,5765,499947,34662,205.7549819946289,b85b14e2f70add52e6e9150c1375242c16c9cd8a0ae8597cf850690fbaca4206,6c77c48c607209af5372cf2f4b2ef472c214db61332e3d5ec86abe3e1e305fc9,1,True,completed,PASS, +gpt56-kb-off,exploratory,2,c323e4c03a3129c3817eed5c4ece012337c6c0336d28585eb2a37653bffb3c81,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.0310698379156245,1.0310698379156245,1.041752689517352,True,0.14560580253601074,verified,1021705,976768,8186,1029891,44937,2755,527523,517248,2421,529944,10275,198.61666584014893,277a922962e1645b25943b1318444339cb8487ae8ccab1a91c4bd982591c5980,606318a24695230bd482c9ae242830371af118860b4a6d71833ac653c938fc09,1,True,completed,PASS, +gpt56-kb-off,exploratory,3,88fe22030b7344cf1207adc5939a3954c5b1935c95aa83f25454c19ddf6ce579,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,0.9969993268819651,0.9969993268819651,1.041752689517352,True,0.17164921760559082,verified,1542884,1493888,10280,1553164,48996,3071,521179,517120,2094,523273,4059,206.43270206451416,263f289ca34319c42a56f304b8e770da79a7e785fe7db49dbc289c1073145845,2da1e5101bd14352568aff7995691d17785c5b8a1e83b5e275b12e582227ffe5,1,True,completed,PASS, +gpt56-kb-off,exploratory,4,f2518462b20b01cd36b206948e28613041357aae2c58386b3fa902b36d5a3290,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.0804578633518793,1.0804578633518793,1.0804578633518793,True,0.1430978775024414,verified,2117639,2064640,14138,2131777,52999,4689,574755,570752,3858,578613,4003,211.60647702217102,7f61073e3419554a7855685260c7984b64493f71e4e75296743083c622024101,ff3c39c2609c1553fc1b33fdea22dec52d2f1a6fcc8fa14dbff43bb4b82f7a29,1,True,completed,PASS, +gpt56-kb-off,exploratory,5,dc4de6caa9c81aa388255f2ec886f804bd15ced5495feadb72400d60a559f3cd,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.173947151000938,1.173947151000938,1.173947151000938,True,0.14515924453735352,verified,2742734,2685952,17371,2760105,56782,5684,625095,621312,3233,628328,3783,198.3997802734375,b4f53219cc4706869a0905924bc2173e500b9e3795973a73b392cf3d384bd385,135ddb85444fb2211595a8ede688311005a66be3013ace8375140cd114e281a8,1,True,completed,PASS, +gpt56-kb-off,exploratory,6,3f61231245363d3e5029148c1761fee4fe3bf8e64573f7bd50bd31fccccc6d52,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.1985941942776748,1.1985941942776748,1.1985941942776748,True,0.14246344566345215,verified,3413908,3353088,20126,3434034,60820,6932,671174,667136,2755,673929,4038,198.47230434417725,409ab6976f5f80a73c962faece6612aae5a91d3d5425a6dc80148bd8ad711ae3,430975e5d609cf0aa5ef42e542c6a83ccf4b63c30bdb0ed207790027e1061842,1,True,completed,PASS, +gpt56-kb-off,exploratory,7,7090b96e2442cffa124c58fb1b25749dce228b7907ce8c968d827344fca8e594,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.1759629915970067,1.1759629915970067,1.1985941942776748,True,0.14681792259216309,verified,4127342,4062592,22488,4149830,64750,7833,713434,709504,2362,715796,3930,201.95138812065125,9b7cc40a6af291133872a535c1d19b05cb1deb2eec0b8c1fd74f9b1f8f39368d,8391ddeb2ba304a615591eefa2909d614822adab576f135026a7fe1b038c963e,1,True,completed,PASS, +gpt56-kb-off,exploratory,8,07d0a19761a91c5e3b3a080c14c4ddcaf262f9e0499de8ad3d898d94bd98046b,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.2175874138276492,1.2175874138276492,1.2175874138276492,True,0.14858031272888184,verified,4879881,4812928,24478,4904359,66953,8056,752539,750336,1990,754529,2203,206.4881181716919,718a8dae7785f0c6f32547e5343c243a10c9d180ada67a1a9c52e5c26d251503,443d6b96f705cd412a485346a3b1d77af9305d5597a3c1aa820131d55ca687e0,1,True,completed,PASS, +gpt56-kb-off,final,,07d0a19761a91c5e3b3a080c14c4ddcaf262f9e0499de8ad3d898d94bd98046b,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.207114117091939,1.207114117091939,1.2175874138276492,True,0.14788508415222168,verified,6028959,5957120,26507,6055466,71839,8314,1149078,1144192,2029,1151107,4886,373.905065536499,1b2d05ab5a8240a77ed6597cdfe9bda62c98100d810a6b6695b50b0e4abfd84f,7cd256965f7aee20b1ed725dd4481b86284bf3ac469b95ae31fcdb6fd4347817,1,True,completed,PASS, +gpt6astra-kb-on,exploratory,1,68c5f4a4cef7d800aab923af778431bdf0f35dde432717aac53b524b3c3e41d7,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.0362349563628106,1.0362349563628106,1.0362349563628106,True,189.54539728164673,verified,2326190,2235776,11953,2338143,90414,3322,2326190,2235776,11953,2338143,90414,202.2415487766266,5bee7c5d9c7c4e15989adb638cdc5fe9f88bd49b9b6de927ca74af856607bea0,830ba82f18d15691a25e1c59d453a79e63c90c0bd832ad15fdda4e7484090ad7,1,True,completed,PASS, +gpt6astra-kb-on,exploratory,2,854773d84fdbd027ffbe2368dbda499c0e3e76b148958f36f64c54964b4d8b87,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.6738736592100159,1.6738736592100159,1.6738736592100159,True,28.270265340805054,verified,3373553,3274112,16717,3390270,99441,6041,1047363,1038336,4764,1052127,9027,201.46938753128052,1280ef75c2cc047a993b87a450c565a323a15de93ce09de049766c68e5b5fd3b,5f04177c05df732f79047521360402f065eece801d02db13f580a1c82a76d0c3,1,True,completed,PASS, +gpt6astra-kb-on,exploratory,3,993a923efd56399775efc9596dfe3344ff68478a135de42fadf953bf21e5b884,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.123848810256551,2.123848810256551,2.123848810256551,True,265.54253339767456,verified,5591803,5477376,27183,5618986,114427,10249,2218250,2203264,10466,2228716,14986,214.5753481388092,270ed06158c99b002705b92846d0e9aecc7673ddcdf2804de18e0b961d30a722,f507e98e77386726f92832956b8d3a349d9bfb90b20ee509a96b5b191e15a8c8,1,True,completed,PASS, +gpt6astra-kb-on,exploratory,4,0016258c26a8ac4b9ababeed1b8fcf13871c0578f52965df3b05eec4a55746ac,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,FAIL,False,geometric_mean_case_speedups,,,2.123848810256551,True,320.7573881149292,verified,7706550,7583360,33864,7740414,123190,14539,2114747,2105984,6681,2121428,8763,30.503424167633057,8d7f6232250cd9523a6f5700f7d3442ef71c3915c4ef7cc67072b76ba5045e8a,,1,True,candidate_failure,INCONCLUSIVE,Candidate rejected on both ranks before timing: installed npu_moe_finalize_routing does not accept keyword argument 'k'. +gpt6astra-kb-on,exploratory,5,73a72a0c29de1a0961e874a66173324002d3c80e0702cfc11f907258090713b1,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.241795215896914,2.241795215896914,2.241795215896914,True,0.14263057708740234,verified,8988213,8859136,38795,9027008,129077,18100,1281663,1275776,4931,1286594,5887,203.39757227897644,3cf70528bb00d2ad298d2bf8b4b154d709bd1e43237d0e938220a4fd1ee8619b,236fd4a0e02602c5d323cf55934c2b662e66f8b064d0bc224f8783aa1fbccd71,1,True,completed,PASS, +gpt6astra-kb-on,exploratory,6,b0bede9598ec1737ae9932d636dda6067f8a1073c7711bb523b67015d3eb4539,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.152762300865101,2.152762300865101,2.241795215896914,True,437.6984884738922,verified,12473969,12337920,52291,12526260,136049,27636,3485756,3478784,13496,3499252,6972,202.65567922592163,9a06b0bd99202c2e7be4886a19ee6920c07127816c8858f362f914fd35ca0cf2,3290997ad14baf18b0244bbf0b03d4f55a0f7b326a8ddb4aa9ce78845426f379,1,True,completed,PASS, +gpt6astra-kb-on,exploratory,7,1561c1fb8f1bfdcd625802e69146b24d1b69958b22dee3d855921fe4eceb657c,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.447966325812682,2.447966325812682,2.447966325812682,True,14.20435643196106,verified,13971152,13830784,59664,14030816,140368,32127,1497183,1492864,7373,1504556,4319,204.89156699180603,4c5e60c54b42447cb8539f83e4b5b7523bc04b49d4e76936378d6d49ebf0a4c0,23049f408ccf531dfa9065950f36646333693257b0ccb3586a6369163a6c621b,1,True,completed,PASS, +gpt6astra-kb-on,exploratory,8,aea3a29f087b3ff6459b5e799f67c0aba829aed03da3da2f03af93cecc022f12,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.3960949518750145,2.3960949518750145,2.447966325812682,True,809.058810710907,verified,19474012,19325056,77753,19551765,148956,45293,5502860,5494272,18089,5520949,8588,204.87169575691223,7e5c4b1cafc3380ac5d0e6c4de84b0ed93fcdc1fae60127d57661dd0dbb94bc9,6b6587878baac3abb2b175cc780c11735d1a716c2dfbc682e9138f2d0144ad2c,1,True,completed,PASS, +gpt6astra-kb-on,final,,1561c1fb8f1bfdcd625802e69146b24d1b69958b22dee3d855921fe4eceb657c,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.3670674096602466,2.3670674096602466,2.447966325812682,True,419.87462067604065,verified,22139428,21984896,90813,22230241,154532,55046,2665416,2659840,13060,2678476,5576,346.90401124954224,53a9ddfe7bfc0d10ac485dfb480307a516748783144c902d34f5be1a7b69b10a,41586d088e9c164aba002ec94de0d0bb4d9e117419ba15e29d84f86e119f9974,1,True,completed,PASS, +gpt6astra-kb-off,exploratory,1,c7b20dfbc23f611feb5022f08e0d40e428e2b365062b74229d6a64ca7946958b,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.0331759575264898,1.0331759575264898,1.0331759575264898,True,0.14717555046081543,verified,932457,870144,6215,938672,62313,1502,932457,870144,6215,938672,62313,197.50759291648865,4e777a35eb6a66276ae2a25b3dd71e1917e3dbf35257771511e59bbcd11524d7,0bff218e2a368d7f0cf4e7f03b29d59b16c9c36b9fb528aa9e5b526d94e53416,1,True,completed,PASS, +gpt6astra-kb-off,exploratory,2,b0f9d88f3fa94668744c188a7f7964160aced4bd1f3e1f9ada924b397292e536,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,1.6526363575584917,1.6526363575584917,1.6526363575584917,True,0.14017701148986816,verified,2188818,2117504,10848,2199666,71314,3615,1256361,1247360,4633,1260994,9001,196.23589491844177,d46ef9042d8b045bc08cee16f266859d2b1bb1ab1fbae42e726cc8452e1abe07,9bbc4a9c594d92c39e45a7bac227e2f1ca78521db3f46c857191f32b40eceb20,1,True,completed,PASS, +gpt6astra-kb-off,exploratory,3,ef5c55d92cd74bd904f3130eba476a682c3d17d4113e4163a41c0663a89735e9,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.43652966789247,2.43652966789247,2.43652966789247,True,0.14071369171142578,verified,3493062,3414528,17130,3510192,78534,5314,1304244,1297024,6282,1310526,7220,202.84937405586243,fc659a40ccf616b44e7347727511a54a75983287d07216f47ec0392bbc355071,eb038834d3a8ba83264d94c8e9bb563b2f2afa7fe6f095c8703c38b002aa8897,1,True,completed,PASS, +gpt6astra-kb-off,exploratory,4,252af26dddc8856a78f6ad112e939c640cf229028820b453d66d46cc838d696e,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,FAIL,False,geometric_mean_case_speedups,,,2.43652966789247,True,0.13994932174682617,verified,3796004,3714688,19403,3815407,81316,6829,302942,300160,2273,305215,2782,29.44064688682556,6dd72f28d934c853fc104dc2a903f3e8bde2f35273146f0fd4ed759901793800,,1,True,candidate_failure,INCONCLUSIVE,Candidate rejected on both ranks before timing: installed npu_moe_finalize_routing does not accept keyword argument 'k'. +gpt6astra-kb-off,exploratory,5,38ac84ca35b444caf4f71854bbe62211f4c359b0bd07bc95192539a1d3d0cb0e,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.3047518247039847,2.3047518247039847,2.43652966789247,True,0.14074420928955078,verified,4868843,4782080,22263,4891106,86763,8530,1072839,1067392,2860,1075699,5447,203.85016012191772,cec9ac465a66ef9424d86d65692f8fa2552abb26cea9300eb003e8f12bdc3908,25bc92b63164b9847cb9f49069ac5f3d5979af59799ad0fcb6bf83d743f91646,1,True,completed,PASS, +gpt6astra-kb-off,exploratory,6,84ec3d844910f2ac40079165f5ab0bbe4263a7b53315fbe1040fc645c63170d2,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.339274874728824,2.339274874728824,2.43652966789247,True,0.1399071216583252,verified,5663395,5572992,26064,5689459,90403,11283,794552,790912,3801,798353,3640,199.04994177818298,81b0f30da84387c24805247c289fbb390b651389f2f02e51ab658f0ba58f576d,1314f6b6aa455e680c24b9352834c4176e92b479d7919d0afa3bbc88e1b57928,1,True,completed,PASS, +gpt6astra-kb-off,exploratory,7,b84a624eb193224e3fd1f8c1bd087d080b2d2ac46c713c5cd808543ae64da0c3,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.4042781514496108,2.4042781514496108,2.43652966789247,True,0.14899277687072754,verified,6734054,6639488,28749,6762803,94566,12859,1070659,1066496,2685,1073344,4163,202.39580416679382,90e64f6944b52d03906dfd93d8db19169f3fab3a1b32fa6ee3365266a8b86f39,b045b71e03a07a3322b9c1003ac3b1f3cf9092281d5904d82284caed0583a4f5,1,True,completed,PASS, +gpt6astra-kb-off,exploratory,8,0d784712015edc1ad94393e15ee19152e660b2288ad66213d886b4ea7972c4f0,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.2913687750805463,2.2913687750805463,2.43652966789247,True,0.14332795143127441,verified,7740861,7642368,33855,7774716,98493,15414,1006807,1002880,5106,1011913,3927,202.17194604873657,994f964dc2b66cc39d97e3fa0868afb816d52dc0a544f7a28684967151cdae90,7f329857b8dc8a3b5b949f1b95587b66431cdc4d7bdb4c08505a585e5a468285,1,True,completed,PASS, +gpt6astra-kb-off,final,,ef5c55d92cd74bd904f3130eba476a682c3d17d4113e4163a41c0663a89735e9,36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae,34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e,6ed93bb41f7db9dafb420dcb6f8705f1864bc45c,174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7,PASS,True,geometric_mean_case_speedups,2.339809467070293,2.339809467070293,2.43652966789247,True,0.13927984237670898,verified,9772433,9667328,41965,9814398,105105,19233,2031572,2024960,8110,2039682,6612,357.21352314949036,b14eb5765736a8e5ed2eaf1cc1284149d9c160ed09c5996ea681d9d14ef6ae44,2d874b3a7d18d7f49af2e54e465669ad48b9b14cb1eb46e43c7aa5c46ec68db3,1,True,completed,PASS, diff --git a/evidence/moe-agent-comparison-20261104/results/session-usage.csv b/evidence/moe-agent-comparison-20261104/results/session-usage.csv new file mode 100644 index 000000000..d1505bd69 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/results/session-usage.csv @@ -0,0 +1,7 @@ +session_key,model,knowledge,exploration_tokens,final_stage_tokens,report_tail_tokens,request_records,input_tokens,cached_input_tokens,output_tokens,total_tokens,uncached_input_tokens,reasoning_output_tokens,monetary_cost,queue_seconds,allocation_process_seconds +gpt61-kb-on,gpt-6.1-sol,on,8131818,1673613,577259,75,10337498,10175488,45192,10382690,162010,17666,,1.3140857219696045,1938.160629272461 +gpt61-kb-off,gpt-6.1-sol,off,8202212,1665783,533633,85,10350761,10215680,50867,10401628,135081,25209,,1.3095238208770752,1974.2647972106934 +gpt56-kb-on,gpt-5.6-sol,on,13016364,1716556,3155714,140,17828530,17654272,60104,17888634,174258,16080,,1.3013813495635986,1950.1834812164307 +gpt56-kb-off,gpt-5.6-sol,off,4904359,1151107,1238744,102,7253603,7169664,40607,7294210,83939,11706,,1.3307960033416748,2001.62748336792 +gpt6astra-kb-on,gpt-6-astra,on,19551765,2678476,950283,152,23086198,22921344,94326,23180524,164854,57260,,2485.094491004944,1811.510234117508 +gpt6astra-kb-off,gpt-6-astra,off,7774716,2039682,600255,104,10365965,10253184,48688,10414653,112781,22922,,1.2802674770355225,1790.7148840427399 diff --git a/evidence/moe-agent-comparison-20261104/results/summary.json b/evidence/moe-agent-comparison-20261104/results/summary.json new file mode 100644 index 000000000..56516ffc1 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/results/summary.json @@ -0,0 +1 @@ +{"schema_version":1,"campaign_id":"moe-serial-20261104","expected_session_count":6,"campaign_complete":true,"condition_comparison_verified":true,"publication_type":"redacted field extract; not the original summary","sessions":[{"key":"gpt61-kb-on","label":"GPT-6.1 / 知识库开","model":"gpt-6.1-sol","knowledge":"on","status":"completed","rounds_completed":8,"rounds_expected":8,"validation_status":"COMPLETE","best_exploration_speedup":2.256558755529666,"final_speedup":2.2163455443832887,"post_measurement_reporting_tokens":577259,"reasoning_effort":["xhigh"],"usage":{"request_records":75,"input_tokens":10337498,"cached_input_tokens":10175488,"output_tokens":45192,"total_tokens":10382690,"uncached_input_tokens":162010,"reasoning_output_tokens":17666,"monetary_cost":null},"queue_seconds":1.3140857219696045,"allocation_process_seconds":1938.160629272461,"observations":[{"role":"exploratory","round_index":1,"candidate_sha256":"0d0f392255b28699940db896c4eafe9a859e587dd7d7bacbbe68859ebbffb383","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.033496152864355,"effective_speedup":1.033496152864355,"best_exploration_speedup":1.033496152864355,"cleanup_verified":true,"queue_seconds":0.1506795883178711,"token_boundary_status":"verified","cumulative_input_tokens":1405745,"cumulative_cached_input_tokens":1289344,"cumulative_output_tokens":8446,"cumulative_total_tokens":1414191,"cumulative_uncached_input_tokens":116401,"cumulative_reasoning_output_tokens":2089,"round_input_tokens":1405745,"round_cached_input_tokens":1289344,"round_output_tokens":8446,"round_total_tokens":1414191,"round_uncached_input_tokens":116401,"allocation_process_seconds":191.8611934185028,"original_observation_sha256":"739e85da1ae0ca7ba4c7226bcfc77c61b35662d4288bd4f9c7cec9ad7a239b59","original_device_summary_sha256":"2debed672659e2dfae6da2cd32dc282778a2ff5f7f9f19c881e6bdee96daee5b","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":2,"candidate_sha256":"2657e679b9f374d6e0aee01c889cf69cf23d7a2354ca58921ba731052cd3f452","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.659926470855875,"effective_speedup":1.659926470855875,"best_exploration_speedup":1.659926470855875,"cleanup_verified":true,"queue_seconds":0.14699196815490723,"token_boundary_status":"verified","cumulative_input_tokens":2328374,"cumulative_cached_input_tokens":2200192,"cumulative_output_tokens":11862,"cumulative_total_tokens":2340236,"cumulative_uncached_input_tokens":128182,"cumulative_reasoning_output_tokens":3593,"round_input_tokens":922629,"round_cached_input_tokens":910848,"round_output_tokens":3416,"round_total_tokens":926045,"round_uncached_input_tokens":11781,"allocation_process_seconds":194.22847890853882,"original_observation_sha256":"18c0c93c82a2d221f21da320787c89fcf85528ecc87627991a7c4d86c47e434f","original_device_summary_sha256":"fcbf89354bad6b1ae875b48eaa23a7d1cf32202234839d37bf3493df858c4539","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":3,"candidate_sha256":"393a0cd1b4affdd79ab49920a4a6724f2b53d1f7ce2d45efe585874247d04c0a","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.5610369207444954,"effective_speedup":1.5610369207444954,"best_exploration_speedup":1.659926470855875,"cleanup_verified":true,"queue_seconds":0.146392822265625,"token_boundary_status":"verified","cumulative_input_tokens":3183450,"cumulative_cached_input_tokens":3047808,"cumulative_output_tokens":15636,"cumulative_total_tokens":3199086,"cumulative_uncached_input_tokens":135642,"cumulative_reasoning_output_tokens":5510,"round_input_tokens":855076,"round_cached_input_tokens":847616,"round_output_tokens":3774,"round_total_tokens":858850,"round_uncached_input_tokens":7460,"allocation_process_seconds":188.41365265846252,"original_observation_sha256":"99b1bd4c5d97914b77d83ec7c39d8ffd8a265eb002cea5f56bbc3ea460c11922","original_device_summary_sha256":"47026e89884a10839d1709e358917521391e0c4c1417e28f59566ba8ef631f1e","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":4,"candidate_sha256":"8fc66ba373f034b851d292a7e26097dcf814307b2c922c9fb1a732b931b95d68","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.2556118202102673,"effective_speedup":2.2556118202102673,"best_exploration_speedup":2.2556118202102673,"cleanup_verified":true,"queue_seconds":0.14520955085754395,"token_boundary_status":"verified","cumulative_input_tokens":4083585,"cumulative_cached_input_tokens":3944192,"cumulative_output_tokens":19566,"cumulative_total_tokens":4103151,"cumulative_uncached_input_tokens":139393,"cumulative_reasoning_output_tokens":7135,"round_input_tokens":900135,"round_cached_input_tokens":896384,"round_output_tokens":3930,"round_total_tokens":904065,"round_uncached_input_tokens":3751,"allocation_process_seconds":192.46062564849854,"original_observation_sha256":"59a6d3be1d248e2b51a89880fbda419ee5413520a0d15532e08fb7188abb671d","original_device_summary_sha256":"b9431a4a2cd60458c55a3624662a1e7b23a4d4d076a30005266c0ab1a79f16ac","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":5,"candidate_sha256":"507d265ffde8d7b1d3348e37c46ab4f696503e0dafbfef1bbbb7cd3606e25d0e","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.256558755529666,"effective_speedup":2.256558755529666,"best_exploration_speedup":2.256558755529666,"cleanup_verified":true,"queue_seconds":0.1504838466644287,"token_boundary_status":"verified","cumulative_input_tokens":5013219,"cumulative_cached_input_tokens":4871168,"cumulative_output_tokens":22736,"cumulative_total_tokens":5035955,"cumulative_uncached_input_tokens":142051,"cumulative_reasoning_output_tokens":8858,"round_input_tokens":929634,"round_cached_input_tokens":926976,"round_output_tokens":3170,"round_total_tokens":932804,"round_uncached_input_tokens":2658,"allocation_process_seconds":195.5376443862915,"original_observation_sha256":"4bc6d6ad521dfcd4d66228d6779ec408280161fcb7f60053c578728fb740644e","original_device_summary_sha256":"ff0459525727a3fb9a68ddb45f472480a4326d06d0545463c140cd6af012f2d7","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":6,"candidate_sha256":"81a9b68ef4108e90279e99e9a323f7dba8ecdd211c909a67b115225adceb26d7","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.2407286057764324,"effective_speedup":2.2407286057764324,"best_exploration_speedup":2.256558755529666,"cleanup_verified":true,"queue_seconds":0.14667773246765137,"token_boundary_status":"verified","cumulative_input_tokens":5819085,"cumulative_cached_input_tokens":5674496,"cumulative_output_tokens":30079,"cumulative_total_tokens":5849164,"cumulative_uncached_input_tokens":144589,"cumulative_reasoning_output_tokens":10313,"round_input_tokens":805866,"round_cached_input_tokens":803328,"round_output_tokens":7343,"round_total_tokens":813209,"round_uncached_input_tokens":2538,"allocation_process_seconds":205.60196185112,"original_observation_sha256":"a1fd2af2d94a2d6f28131fdd439617caece1ad3f9cd411b478c97566f4d8db75","original_device_summary_sha256":"e4889458cb398ea385e9128ce8b8efc76a01927274841e2cf07720180aaf652f","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":7,"candidate_sha256":"0cf406d0472202fa9ac35390ab28993194e512c2414696a1840e19a4efc304fa","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.1809720767812473,"effective_speedup":2.1809720767812473,"best_exploration_speedup":2.256558755529666,"cleanup_verified":true,"queue_seconds":0.14430689811706543,"token_boundary_status":"verified","cumulative_input_tokens":7021441,"cumulative_cached_input_tokens":6870144,"cumulative_output_tokens":33863,"cumulative_total_tokens":7055304,"cumulative_uncached_input_tokens":151297,"cumulative_reasoning_output_tokens":12188,"round_input_tokens":1202356,"round_cached_input_tokens":1195648,"round_output_tokens":3784,"round_total_tokens":1206140,"round_uncached_input_tokens":6708,"allocation_process_seconds":206.1087031364441,"original_observation_sha256":"773faf5fddc5753a5dd0609fbb191448b56c5cb3eb2d16f639abc9b14b0c63e3","original_device_summary_sha256":"6bcda80b83c0078c84074c4586ebd924727a10e93f43b0085b75f0cf4b06524b","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":8,"candidate_sha256":"c896e3e99aa4be3789526c41f74ca5bf34fa310a5532407638e12df13de4e832","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.2291567851269463,"effective_speedup":2.2291567851269463,"best_exploration_speedup":2.256558755529666,"cleanup_verified":true,"queue_seconds":0.14380669593811035,"token_boundary_status":"verified","cumulative_input_tokens":8093276,"cumulative_cached_input_tokens":7939456,"cumulative_output_tokens":38542,"cumulative_total_tokens":8131818,"cumulative_uncached_input_tokens":153820,"cumulative_reasoning_output_tokens":13736,"round_input_tokens":1071835,"round_cached_input_tokens":1069312,"round_output_tokens":4679,"round_total_tokens":1076514,"round_uncached_input_tokens":2523,"allocation_process_seconds":205.95839715003967,"original_observation_sha256":"955b09390f31bfb3cfc776f80d9dc81e3da30e068e7cd9d632438b8d93d14ffe","original_device_summary_sha256":"78b2e02b9ed3abd4700d737aae22e567edcf1d53366d6985e5b3544cc50cb3fa","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"final","round_index":null,"candidate_sha256":"507d265ffde8d7b1d3348e37c46ab4f696503e0dafbfef1bbbb7cd3606e25d0e","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.2163455443832887,"effective_speedup":2.2163455443832887,"best_exploration_speedup":2.256558755529666,"cleanup_verified":true,"queue_seconds":0.13953661918640137,"token_boundary_status":"verified","cumulative_input_tokens":9761936,"cumulative_cached_input_tokens":9604096,"cumulative_output_tokens":43495,"cumulative_total_tokens":9805431,"cumulative_uncached_input_tokens":157840,"cumulative_reasoning_output_tokens":16720,"round_input_tokens":1668660,"round_cached_input_tokens":1664640,"round_output_tokens":4953,"round_total_tokens":1673613,"round_uncached_input_tokens":4020,"allocation_process_seconds":357.989972114563,"original_observation_sha256":"54d6271b0b286ff028d6264c039d70b8bc8c4ca5d661ee4264777667b32b52e2","original_device_summary_sha256":"b677794e3267a76b4a215d051d5e46b11807a11880bd106e3ebeaec4f6ce62af","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null}],"selected_round":5,"final_candidate_sha256":"507d265ffde8d7b1d3348e37c46ab4f696503e0dafbfef1bbbb7cd3606e25d0e"},{"key":"gpt61-kb-off","label":"GPT-6.1 / 知识库关","model":"gpt-6.1-sol","knowledge":"off","status":"completed","rounds_completed":8,"rounds_expected":8,"validation_status":"COMPLETE","best_exploration_speedup":2.4630234063840075,"final_speedup":2.3806962845718207,"post_measurement_reporting_tokens":533633,"reasoning_effort":["xhigh"],"usage":{"request_records":85,"input_tokens":10350761,"cached_input_tokens":10215680,"output_tokens":50867,"total_tokens":10401628,"uncached_input_tokens":135081,"reasoning_output_tokens":25209,"monetary_cost":null},"queue_seconds":1.3095238208770752,"allocation_process_seconds":1974.2647972106934,"observations":[{"role":"exploratory","round_index":1,"candidate_sha256":"74c9cbb8ac2be1b9aac1f45625d2bb77fa951c5e65cc0d3e0e7900db87ca2a42","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.0182446674027414,"effective_speedup":1.0182446674027414,"best_exploration_speedup":1.0182446674027414,"cleanup_verified":true,"queue_seconds":0.14409542083740234,"token_boundary_status":"verified","cumulative_input_tokens":1028458,"cumulative_cached_input_tokens":955264,"cumulative_output_tokens":7669,"cumulative_total_tokens":1036127,"cumulative_uncached_input_tokens":73194,"cumulative_reasoning_output_tokens":2314,"round_input_tokens":1028458,"round_cached_input_tokens":955264,"round_output_tokens":7669,"round_total_tokens":1036127,"round_uncached_input_tokens":73194,"allocation_process_seconds":201.93721413612366,"original_observation_sha256":"2876e1a4b080b8cdf73080528b1becc7cdef77cbc8a0769187285030d0e55def","original_device_summary_sha256":"36bde9b63e044dde30ad10782884ead189654dff5b09d26eb541277eb26dd5c5","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":2,"candidate_sha256":"f163d015875b80f90d67b8f50dd246f59fb937299ea2a43f0279036f574e1437","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.5011271720808808,"effective_speedup":1.5011271720808808,"best_exploration_speedup":1.5011271720808808,"cleanup_verified":true,"queue_seconds":0.14340543746948242,"token_boundary_status":"verified","cumulative_input_tokens":1930173,"cumulative_cached_input_tokens":1844224,"cumulative_output_tokens":11946,"cumulative_total_tokens":1942119,"cumulative_uncached_input_tokens":85949,"cumulative_reasoning_output_tokens":4209,"round_input_tokens":901715,"round_cached_input_tokens":888960,"round_output_tokens":4277,"round_total_tokens":905992,"round_uncached_input_tokens":12755,"allocation_process_seconds":205.39305973052979,"original_observation_sha256":"bcc18426b029e631aa0803444f8e840805c2b86d2715554cefed2c200e6489ad","original_device_summary_sha256":"f6a2a5ad8d174df2e566cef5a7fc4b6cf8bf1344dcdcab339deff6fdf07ac512","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":3,"candidate_sha256":"9cdee85e0b487703a92a85a3d4d78ad3bd11749b434dbae00ece0f03fc92db08","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.1573718369619095,"effective_speedup":2.1573718369619095,"best_exploration_speedup":2.1573718369619095,"cleanup_verified":true,"queue_seconds":0.14728975296020508,"token_boundary_status":"verified","cumulative_input_tokens":3035590,"cumulative_cached_input_tokens":2943232,"cumulative_output_tokens":16010,"cumulative_total_tokens":3051600,"cumulative_uncached_input_tokens":92358,"cumulative_reasoning_output_tokens":6084,"round_input_tokens":1105417,"round_cached_input_tokens":1099008,"round_output_tokens":4064,"round_total_tokens":1109481,"round_uncached_input_tokens":6409,"allocation_process_seconds":199.14277029037476,"original_observation_sha256":"f316eadd7d1a68d637a721e33be315feeca8b814f610d2de91dd387825908a0c","original_device_summary_sha256":"c9007cbea914654b6aca79c1b96b4f29217c8e57f94ebaa33c18643d5cd6ff0f","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":4,"candidate_sha256":"10d7bd7d74c147f476e050bd5e23e347dd51096d83575e802d13d150df923f4c","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.239197543098337,"effective_speedup":2.239197543098337,"best_exploration_speedup":2.239197543098337,"cleanup_verified":true,"queue_seconds":0.14458942413330078,"token_boundary_status":"verified","cumulative_input_tokens":4022724,"cumulative_cached_input_tokens":3920512,"cumulative_output_tokens":21880,"cumulative_total_tokens":4044604,"cumulative_uncached_input_tokens":102212,"cumulative_reasoning_output_tokens":7349,"round_input_tokens":987134,"round_cached_input_tokens":977280,"round_output_tokens":5870,"round_total_tokens":993004,"round_uncached_input_tokens":9854,"allocation_process_seconds":201.5980749130249,"original_observation_sha256":"48839c99a5f325382e2512d54a628f63b6446d09b6aee4c4ef7c0341ebc207a4","original_device_summary_sha256":"a51911a69e4b83fb3336ef357753ea392dd5bf5f9b7b722a8a1712a2b31cd9e0","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":5,"candidate_sha256":"69efab972a84228f7e31cf827e615b23777316a0586cc0624214edea8e64e2eb","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.329806968277373,"effective_speedup":2.329806968277373,"best_exploration_speedup":2.329806968277373,"cleanup_verified":true,"queue_seconds":0.15340185165405273,"token_boundary_status":"verified","cumulative_input_tokens":4946814,"cumulative_cached_input_tokens":4840448,"cumulative_output_tokens":26126,"cumulative_total_tokens":4972940,"cumulative_uncached_input_tokens":106366,"cumulative_reasoning_output_tokens":10543,"round_input_tokens":924090,"round_cached_input_tokens":919936,"round_output_tokens":4246,"round_total_tokens":928336,"round_uncached_input_tokens":4154,"allocation_process_seconds":201.4565303325653,"original_observation_sha256":"238743d9d09c7dc19fc72b6b91a8a301f07b45618dc7006f72bcdefeab7e555b","original_device_summary_sha256":"57586c82e3ad83be02dd8839195cb0eb1f2341556a617e131d9454df886a0338","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":6,"candidate_sha256":"ac521b38f2555639bcde827f35b9b205767136b91ccf170e3a3f72c1c06de4f2","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.4630234063840075,"effective_speedup":2.4630234063840075,"best_exploration_speedup":2.4630234063840075,"cleanup_verified":true,"queue_seconds":0.13998985290527344,"token_boundary_status":"verified","cumulative_input_tokens":6208420,"cumulative_cached_input_tokens":6097152,"cumulative_output_tokens":30212,"cumulative_total_tokens":6238632,"cumulative_uncached_input_tokens":111268,"cumulative_reasoning_output_tokens":12160,"round_input_tokens":1261606,"round_cached_input_tokens":1256704,"round_output_tokens":4086,"round_total_tokens":1265692,"round_uncached_input_tokens":4902,"allocation_process_seconds":202.2952857017517,"original_observation_sha256":"93cf7c3b2b0a7a27982b7a4e57152195cfb9457ccd8e7cf92b3a070da4bc4fd9","original_device_summary_sha256":"6707e81ae878abdc28cfef8bbe0517fc11635ae154d9a24af4b5dc24fd9d707a","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":7,"candidate_sha256":"67bb00dd912136041066cbcb1949ff2f9b5397be54973dffca81dc55ad4b0b17","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.3718703757080677,"effective_speedup":2.3718703757080677,"best_exploration_speedup":2.4630234063840075,"cleanup_verified":true,"queue_seconds":0.14416289329528809,"token_boundary_status":"verified","cumulative_input_tokens":7236698,"cumulative_cached_input_tokens":7121152,"cumulative_output_tokens":33972,"cumulative_total_tokens":7270670,"cumulative_uncached_input_tokens":115546,"cumulative_reasoning_output_tokens":14653,"round_input_tokens":1028278,"round_cached_input_tokens":1024000,"round_output_tokens":3760,"round_total_tokens":1032038,"round_uncached_input_tokens":4278,"allocation_process_seconds":203.76398253440857,"original_observation_sha256":"bcf0ccdfa1cff73fac6045404a45f584f8a652952575f1f60df6252f0d58f092","original_device_summary_sha256":"d1402d982f47bcdec8f9055a65c80656ca9ff3182c01325ee18d65b5458f16e1","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":8,"candidate_sha256":"36a9ac623ee1c3c2cf0717d69bd48d5915edbd4a828832a1693b83bc2632ccc7","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.4228360401116964,"effective_speedup":2.4228360401116964,"best_exploration_speedup":2.4630234063840075,"cleanup_verified":true,"queue_seconds":0.14646649360656738,"token_boundary_status":"verified","cumulative_input_tokens":8162262,"cumulative_cached_input_tokens":8042368,"cumulative_output_tokens":39950,"cumulative_total_tokens":8202212,"cumulative_uncached_input_tokens":119894,"cumulative_reasoning_output_tokens":17771,"round_input_tokens":925564,"round_cached_input_tokens":921216,"round_output_tokens":5978,"round_total_tokens":931542,"round_uncached_input_tokens":4348,"allocation_process_seconds":199.70272850990295,"original_observation_sha256":"8f76eccfae220cdeaacffe5cbdcaa07319c4ed4d99cc4a6c6024a0fea4aae57e","original_device_summary_sha256":"cf1d62a8a8cc68d64ab4aed7e6b393eeda515f8b324252efe74d47b24f59f842","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"final","round_index":null,"candidate_sha256":"ac521b38f2555639bcde827f35b9b205767136b91ccf170e3a3f72c1c06de4f2","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.3806962845718207,"effective_speedup":2.3806962845718207,"best_exploration_speedup":2.4630234063840075,"cleanup_verified":true,"queue_seconds":0.14612269401550293,"token_boundary_status":"verified","cumulative_input_tokens":9819788,"cumulative_cached_input_tokens":9693056,"cumulative_output_tokens":48207,"cumulative_total_tokens":9867995,"cumulative_uncached_input_tokens":126732,"cumulative_reasoning_output_tokens":23456,"round_input_tokens":1657526,"round_cached_input_tokens":1650688,"round_output_tokens":8257,"round_total_tokens":1665783,"round_uncached_input_tokens":6838,"allocation_process_seconds":358.9751510620117,"original_observation_sha256":"d5d049f08d189e2108545ce3e483214e3e7df82a295aeec7ee14e915d8a73101","original_device_summary_sha256":"7371e777b9e9e4f905d2e27a246e7c44761a66c14a2be5378f4958c6c8f2dd8b","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null}],"selected_round":6,"final_candidate_sha256":"ac521b38f2555639bcde827f35b9b205767136b91ccf170e3a3f72c1c06de4f2"},{"key":"gpt56-kb-on","label":"GPT-5.6 / 知识库开","model":"gpt-5.6-sol","knowledge":"on","status":"completed","rounds_completed":8,"rounds_expected":8,"validation_status":"COMPLETE","best_exploration_speedup":1.0913441544227527,"final_speedup":1.0923856414422102,"post_measurement_reporting_tokens":3155714,"reasoning_effort":["xhigh"],"usage":{"request_records":140,"input_tokens":17828530,"cached_input_tokens":17654272,"output_tokens":60104,"total_tokens":17888634,"uncached_input_tokens":174258,"reasoning_output_tokens":16080,"monetary_cost":null},"queue_seconds":1.3013813495635986,"allocation_process_seconds":1950.1834812164307,"observations":[{"role":"exploratory","round_index":1,"candidate_sha256":"aedc1c7d99e2a5615f18443acdccdd1285befbcfd341a7aedb25586718e9e354","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.033127728274557,"effective_speedup":1.033127728274557,"best_exploration_speedup":1.033127728274557,"cleanup_verified":true,"queue_seconds":0.14676356315612793,"token_boundary_status":"verified","cumulative_input_tokens":2616718,"cumulative_cached_input_tokens":2519552,"cumulative_output_tokens":18360,"cumulative_total_tokens":2635078,"cumulative_uncached_input_tokens":97166,"cumulative_reasoning_output_tokens":6208,"round_input_tokens":2616718,"round_cached_input_tokens":2519552,"round_output_tokens":18360,"round_total_tokens":2635078,"round_uncached_input_tokens":97166,"allocation_process_seconds":195.71978950500488,"original_observation_sha256":"47b94dbbe64d854b8004ca89b27865f829eea05896dc9abbeaf8ef12585fa613","original_device_summary_sha256":"91c89b577df42e8f8cecd24aba9a18cc9b3a3fd4ee60bb68dfe8df9648d318e3","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":2,"candidate_sha256":"e9324fb9b5def09b0313528a516b39cf5bdcec1a5cb8f77c62a07800bc18955f","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.043939426200217,"effective_speedup":1.043939426200217,"best_exploration_speedup":1.043939426200217,"cleanup_verified":true,"queue_seconds":0.14348196983337402,"token_boundary_status":"verified","cumulative_input_tokens":3757287,"cumulative_cached_input_tokens":3656064,"cumulative_output_tokens":21779,"cumulative_total_tokens":3779066,"cumulative_uncached_input_tokens":101223,"cumulative_reasoning_output_tokens":6646,"round_input_tokens":1140569,"round_cached_input_tokens":1136512,"round_output_tokens":3419,"round_total_tokens":1143988,"round_uncached_input_tokens":4057,"allocation_process_seconds":201.93050742149353,"original_observation_sha256":"8351a4b80b07b9c9989878ad3039bdc75019d4a5c61c40551dd81d6dec043bc9","original_device_summary_sha256":"82d6fd682b26b15afe4336b6b8868149dffd796d581e209cce0b4826b8d65b48","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":3,"candidate_sha256":"464916a52eacbed262279c369da3ecdf04c4c862a0ab8f44bac1ca6368a4d862","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.0103050146988586,"effective_speedup":1.0103050146988586,"best_exploration_speedup":1.043939426200217,"cleanup_verified":true,"queue_seconds":0.14808988571166992,"token_boundary_status":"verified","cumulative_input_tokens":4953713,"cumulative_cached_input_tokens":4848128,"cumulative_output_tokens":24995,"cumulative_total_tokens":4978708,"cumulative_uncached_input_tokens":105585,"cumulative_reasoning_output_tokens":6873,"round_input_tokens":1196426,"round_cached_input_tokens":1192064,"round_output_tokens":3216,"round_total_tokens":1199642,"round_uncached_input_tokens":4362,"allocation_process_seconds":197.9350609779358,"original_observation_sha256":"9be3800c4e258ec492e532235be10a1d8f9765c5d3cdebd2f415c1cb336c52f5","original_device_summary_sha256":"0d2478981077c1359872906b5b64d30262d2481f0c3a4eae37f682e64c0a975e","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":4,"candidate_sha256":"3f14efc39ace2ba129a048214d66eec2f624f522bc4b9baa76b509bad188f3ca","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.0367692061223202,"effective_speedup":1.0367692061223202,"best_exploration_speedup":1.043939426200217,"cleanup_verified":true,"queue_seconds":0.1413097381591797,"token_boundary_status":"verified","cumulative_input_tokens":6717756,"cumulative_cached_input_tokens":6606848,"cumulative_output_tokens":28783,"cumulative_total_tokens":6746539,"cumulative_uncached_input_tokens":110908,"cumulative_reasoning_output_tokens":7573,"round_input_tokens":1764043,"round_cached_input_tokens":1758720,"round_output_tokens":3788,"round_total_tokens":1767831,"round_uncached_input_tokens":5323,"allocation_process_seconds":204.3094515800476,"original_observation_sha256":"5233740a6750673eb5e03e85d94ea70538affcb0e54946285343f0dd6c27d1e3","original_device_summary_sha256":"ae786be0177c9c3a268a8881f127ece09d85b6b81fa4f1667f9c2e075ac80be3","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":5,"candidate_sha256":"8d52e2f95ae28b6beeaba607868a492a5a2ab2b647d806d19e8155ff1b94c9cf","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.0310284033167278,"effective_speedup":1.0310284033167278,"best_exploration_speedup":1.043939426200217,"cleanup_verified":true,"queue_seconds":0.14885640144348145,"token_boundary_status":"verified","cumulative_input_tokens":8170548,"cumulative_cached_input_tokens":8055424,"cumulative_output_tokens":31938,"cumulative_total_tokens":8202486,"cumulative_uncached_input_tokens":115124,"cumulative_reasoning_output_tokens":8424,"round_input_tokens":1452792,"round_cached_input_tokens":1448576,"round_output_tokens":3155,"round_total_tokens":1455947,"round_uncached_input_tokens":4216,"allocation_process_seconds":203.10643601417542,"original_observation_sha256":"238d7a4d23f2312e2c74edb82c8d5f1d66fe38741b2f807dc3f1b99c7a1aec82","original_device_summary_sha256":"b2b21cb73409fd95efd736e96db992b11a727edc590d899748f3ce6bbcbdeb74","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":6,"candidate_sha256":"a3762e1d73ef5b4f1e672b727e708b4d12fa6d2be53a1c8755db3c8829c80348","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":0.9764517576218519,"effective_speedup":0.9764517576218519,"best_exploration_speedup":1.043939426200217,"cleanup_verified":true,"queue_seconds":0.1410059928894043,"token_boundary_status":"verified","cumulative_input_tokens":9844322,"cumulative_cached_input_tokens":9723264,"cumulative_output_tokens":35964,"cumulative_total_tokens":9880286,"cumulative_uncached_input_tokens":121058,"cumulative_reasoning_output_tokens":9345,"round_input_tokens":1673774,"round_cached_input_tokens":1667840,"round_output_tokens":4026,"round_total_tokens":1677800,"round_uncached_input_tokens":5934,"allocation_process_seconds":200.8367359638214,"original_observation_sha256":"4a624e9dc12f47e5793689b91288c257783e2cdb49a93aa2f339dd2978867a8c","original_device_summary_sha256":"3768ecb3a40da487e0d90fe9165297131230c8819b4b4d28f6212c70d8a1b8a7","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":7,"candidate_sha256":"7a3080fb599fbe18f58bc2c86f558740d4843e95e72b189d8b790f7460128483","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.0825436212364279,"effective_speedup":1.0825436212364279,"best_exploration_speedup":1.0825436212364279,"cleanup_verified":true,"queue_seconds":0.14468598365783691,"token_boundary_status":"verified","cumulative_input_tokens":11454616,"cumulative_cached_input_tokens":11328128,"cumulative_output_tokens":39156,"cumulative_total_tokens":11493772,"cumulative_uncached_input_tokens":126488,"cumulative_reasoning_output_tokens":9933,"round_input_tokens":1610294,"round_cached_input_tokens":1604864,"round_output_tokens":3192,"round_total_tokens":1613486,"round_uncached_input_tokens":5430,"allocation_process_seconds":196.1109836101532,"original_observation_sha256":"923bd1ba2055ce7b4231daec0e443f815650cb8500fa57f81bf5d2ef9b4911cd","original_device_summary_sha256":"5de5918b12bd37be5576cb77f10366beb9db20f1c575008b80b463ca54937774","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":8,"candidate_sha256":"4815b5f5fbae5bcf2f357ab51196301a57330e8ee391ef530fc9324202564f4e","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.0913441544227527,"effective_speedup":1.0913441544227527,"best_exploration_speedup":1.0913441544227527,"cleanup_verified":true,"queue_seconds":0.1469707489013672,"token_boundary_status":"verified","cumulative_input_tokens":12974249,"cumulative_cached_input_tokens":12843648,"cumulative_output_tokens":42115,"cumulative_total_tokens":13016364,"cumulative_uncached_input_tokens":130601,"cumulative_reasoning_output_tokens":10983,"round_input_tokens":1519633,"round_cached_input_tokens":1515520,"round_output_tokens":2959,"round_total_tokens":1522592,"round_uncached_input_tokens":4113,"allocation_process_seconds":198.73881268501282,"original_observation_sha256":"c8acc9bb4259569ae055ba893c8fcfd7b58fadd95e185c162d899035d36fbfb5","original_device_summary_sha256":"57aab0e0c179999cb81fcfd0c9caddc4aa75662f955289bae9d7afadcf6917ee","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"final","round_index":null,"candidate_sha256":"4815b5f5fbae5bcf2f357ab51196301a57330e8ee391ef530fc9324202564f4e","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.0923856414422102,"effective_speedup":1.0923856414422102,"best_exploration_speedup":1.0913441544227527,"cleanup_verified":true,"queue_seconds":0.14021706581115723,"token_boundary_status":"verified","cumulative_input_tokens":14689589,"cumulative_cached_input_tokens":14554880,"cumulative_output_tokens":43331,"cumulative_total_tokens":14732920,"cumulative_uncached_input_tokens":134709,"cumulative_reasoning_output_tokens":11085,"round_input_tokens":1715340,"round_cached_input_tokens":1711232,"round_output_tokens":1216,"round_total_tokens":1716556,"round_uncached_input_tokens":4108,"allocation_process_seconds":351.495703458786,"original_observation_sha256":"f84bf94cbff5b94e117172e842ba8b0392bcb1d3963b8bb9f63ff4d64da86bfd","original_device_summary_sha256":"83ec9b4de116fa5394fde6d46e6471b74928da287ebcbaac0f3f9ec33891ef79","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null}],"selected_round":8,"final_candidate_sha256":"4815b5f5fbae5bcf2f357ab51196301a57330e8ee391ef530fc9324202564f4e"},{"key":"gpt56-kb-off","label":"GPT-5.6 / 知识库关","model":"gpt-5.6-sol","knowledge":"off","status":"completed","rounds_completed":8,"rounds_expected":8,"validation_status":"COMPLETE","best_exploration_speedup":1.2175874138276492,"final_speedup":1.207114117091939,"post_measurement_reporting_tokens":1238744,"reasoning_effort":["xhigh"],"usage":{"request_records":102,"input_tokens":7253603,"cached_input_tokens":7169664,"output_tokens":40607,"total_tokens":7294210,"uncached_input_tokens":83939,"reasoning_output_tokens":11706,"monetary_cost":null},"queue_seconds":1.3307960033416748,"allocation_process_seconds":2001.62748336792,"observations":[{"role":"exploratory","round_index":1,"candidate_sha256":"b868abaac4b60f655b9360c2af5276ba53986c2c5da9e13b029685431244edb6","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.041752689517352,"effective_speedup":1.041752689517352,"best_exploration_speedup":1.041752689517352,"cleanup_verified":true,"queue_seconds":0.13953709602355957,"token_boundary_status":"verified","cumulative_input_tokens":494182,"cumulative_cached_input_tokens":459520,"cumulative_output_tokens":5765,"cumulative_total_tokens":499947,"cumulative_uncached_input_tokens":34662,"cumulative_reasoning_output_tokens":2360,"round_input_tokens":494182,"round_cached_input_tokens":459520,"round_output_tokens":5765,"round_total_tokens":499947,"round_uncached_input_tokens":34662,"allocation_process_seconds":205.7549819946289,"original_observation_sha256":"b85b14e2f70add52e6e9150c1375242c16c9cd8a0ae8597cf850690fbaca4206","original_device_summary_sha256":"6c77c48c607209af5372cf2f4b2ef472c214db61332e3d5ec86abe3e1e305fc9","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":2,"candidate_sha256":"c323e4c03a3129c3817eed5c4ece012337c6c0336d28585eb2a37653bffb3c81","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.0310698379156245,"effective_speedup":1.0310698379156245,"best_exploration_speedup":1.041752689517352,"cleanup_verified":true,"queue_seconds":0.14560580253601074,"token_boundary_status":"verified","cumulative_input_tokens":1021705,"cumulative_cached_input_tokens":976768,"cumulative_output_tokens":8186,"cumulative_total_tokens":1029891,"cumulative_uncached_input_tokens":44937,"cumulative_reasoning_output_tokens":2755,"round_input_tokens":527523,"round_cached_input_tokens":517248,"round_output_tokens":2421,"round_total_tokens":529944,"round_uncached_input_tokens":10275,"allocation_process_seconds":198.61666584014893,"original_observation_sha256":"277a922962e1645b25943b1318444339cb8487ae8ccab1a91c4bd982591c5980","original_device_summary_sha256":"606318a24695230bd482c9ae242830371af118860b4a6d71833ac653c938fc09","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":3,"candidate_sha256":"88fe22030b7344cf1207adc5939a3954c5b1935c95aa83f25454c19ddf6ce579","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":0.9969993268819651,"effective_speedup":0.9969993268819651,"best_exploration_speedup":1.041752689517352,"cleanup_verified":true,"queue_seconds":0.17164921760559082,"token_boundary_status":"verified","cumulative_input_tokens":1542884,"cumulative_cached_input_tokens":1493888,"cumulative_output_tokens":10280,"cumulative_total_tokens":1553164,"cumulative_uncached_input_tokens":48996,"cumulative_reasoning_output_tokens":3071,"round_input_tokens":521179,"round_cached_input_tokens":517120,"round_output_tokens":2094,"round_total_tokens":523273,"round_uncached_input_tokens":4059,"allocation_process_seconds":206.43270206451416,"original_observation_sha256":"263f289ca34319c42a56f304b8e770da79a7e785fe7db49dbc289c1073145845","original_device_summary_sha256":"2da1e5101bd14352568aff7995691d17785c5b8a1e83b5e275b12e582227ffe5","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":4,"candidate_sha256":"f2518462b20b01cd36b206948e28613041357aae2c58386b3fa902b36d5a3290","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.0804578633518793,"effective_speedup":1.0804578633518793,"best_exploration_speedup":1.0804578633518793,"cleanup_verified":true,"queue_seconds":0.1430978775024414,"token_boundary_status":"verified","cumulative_input_tokens":2117639,"cumulative_cached_input_tokens":2064640,"cumulative_output_tokens":14138,"cumulative_total_tokens":2131777,"cumulative_uncached_input_tokens":52999,"cumulative_reasoning_output_tokens":4689,"round_input_tokens":574755,"round_cached_input_tokens":570752,"round_output_tokens":3858,"round_total_tokens":578613,"round_uncached_input_tokens":4003,"allocation_process_seconds":211.60647702217102,"original_observation_sha256":"7f61073e3419554a7855685260c7984b64493f71e4e75296743083c622024101","original_device_summary_sha256":"ff3c39c2609c1553fc1b33fdea22dec52d2f1a6fcc8fa14dbff43bb4b82f7a29","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":5,"candidate_sha256":"dc4de6caa9c81aa388255f2ec886f804bd15ced5495feadb72400d60a559f3cd","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.173947151000938,"effective_speedup":1.173947151000938,"best_exploration_speedup":1.173947151000938,"cleanup_verified":true,"queue_seconds":0.14515924453735352,"token_boundary_status":"verified","cumulative_input_tokens":2742734,"cumulative_cached_input_tokens":2685952,"cumulative_output_tokens":17371,"cumulative_total_tokens":2760105,"cumulative_uncached_input_tokens":56782,"cumulative_reasoning_output_tokens":5684,"round_input_tokens":625095,"round_cached_input_tokens":621312,"round_output_tokens":3233,"round_total_tokens":628328,"round_uncached_input_tokens":3783,"allocation_process_seconds":198.3997802734375,"original_observation_sha256":"b4f53219cc4706869a0905924bc2173e500b9e3795973a73b392cf3d384bd385","original_device_summary_sha256":"135ddb85444fb2211595a8ede688311005a66be3013ace8375140cd114e281a8","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":6,"candidate_sha256":"3f61231245363d3e5029148c1761fee4fe3bf8e64573f7bd50bd31fccccc6d52","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.1985941942776748,"effective_speedup":1.1985941942776748,"best_exploration_speedup":1.1985941942776748,"cleanup_verified":true,"queue_seconds":0.14246344566345215,"token_boundary_status":"verified","cumulative_input_tokens":3413908,"cumulative_cached_input_tokens":3353088,"cumulative_output_tokens":20126,"cumulative_total_tokens":3434034,"cumulative_uncached_input_tokens":60820,"cumulative_reasoning_output_tokens":6932,"round_input_tokens":671174,"round_cached_input_tokens":667136,"round_output_tokens":2755,"round_total_tokens":673929,"round_uncached_input_tokens":4038,"allocation_process_seconds":198.47230434417725,"original_observation_sha256":"409ab6976f5f80a73c962faece6612aae5a91d3d5425a6dc80148bd8ad711ae3","original_device_summary_sha256":"430975e5d609cf0aa5ef42e542c6a83ccf4b63c30bdb0ed207790027e1061842","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":7,"candidate_sha256":"7090b96e2442cffa124c58fb1b25749dce228b7907ce8c968d827344fca8e594","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.1759629915970067,"effective_speedup":1.1759629915970067,"best_exploration_speedup":1.1985941942776748,"cleanup_verified":true,"queue_seconds":0.14681792259216309,"token_boundary_status":"verified","cumulative_input_tokens":4127342,"cumulative_cached_input_tokens":4062592,"cumulative_output_tokens":22488,"cumulative_total_tokens":4149830,"cumulative_uncached_input_tokens":64750,"cumulative_reasoning_output_tokens":7833,"round_input_tokens":713434,"round_cached_input_tokens":709504,"round_output_tokens":2362,"round_total_tokens":715796,"round_uncached_input_tokens":3930,"allocation_process_seconds":201.95138812065125,"original_observation_sha256":"9b7cc40a6af291133872a535c1d19b05cb1deb2eec0b8c1fd74f9b1f8f39368d","original_device_summary_sha256":"8391ddeb2ba304a615591eefa2909d614822adab576f135026a7fe1b038c963e","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":8,"candidate_sha256":"07d0a19761a91c5e3b3a080c14c4ddcaf262f9e0499de8ad3d898d94bd98046b","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.2175874138276492,"effective_speedup":1.2175874138276492,"best_exploration_speedup":1.2175874138276492,"cleanup_verified":true,"queue_seconds":0.14858031272888184,"token_boundary_status":"verified","cumulative_input_tokens":4879881,"cumulative_cached_input_tokens":4812928,"cumulative_output_tokens":24478,"cumulative_total_tokens":4904359,"cumulative_uncached_input_tokens":66953,"cumulative_reasoning_output_tokens":8056,"round_input_tokens":752539,"round_cached_input_tokens":750336,"round_output_tokens":1990,"round_total_tokens":754529,"round_uncached_input_tokens":2203,"allocation_process_seconds":206.4881181716919,"original_observation_sha256":"718a8dae7785f0c6f32547e5343c243a10c9d180ada67a1a9c52e5c26d251503","original_device_summary_sha256":"443d6b96f705cd412a485346a3b1d77af9305d5597a3c1aa820131d55ca687e0","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"final","round_index":null,"candidate_sha256":"07d0a19761a91c5e3b3a080c14c4ddcaf262f9e0499de8ad3d898d94bd98046b","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.207114117091939,"effective_speedup":1.207114117091939,"best_exploration_speedup":1.2175874138276492,"cleanup_verified":true,"queue_seconds":0.14788508415222168,"token_boundary_status":"verified","cumulative_input_tokens":6028959,"cumulative_cached_input_tokens":5957120,"cumulative_output_tokens":26507,"cumulative_total_tokens":6055466,"cumulative_uncached_input_tokens":71839,"cumulative_reasoning_output_tokens":8314,"round_input_tokens":1149078,"round_cached_input_tokens":1144192,"round_output_tokens":2029,"round_total_tokens":1151107,"round_uncached_input_tokens":4886,"allocation_process_seconds":373.905065536499,"original_observation_sha256":"1b2d05ab5a8240a77ed6597cdfe9bda62c98100d810a6b6695b50b0e4abfd84f","original_device_summary_sha256":"7cd256965f7aee20b1ed725dd4481b86284bf3ac469b95ae31fcdb6fd4347817","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null}],"selected_round":8,"final_candidate_sha256":"07d0a19761a91c5e3b3a080c14c4ddcaf262f9e0499de8ad3d898d94bd98046b"},{"key":"gpt6astra-kb-on","label":"GPT-6 Astra / 知识库开","model":"gpt-6-astra","knowledge":"on","status":"completed","rounds_completed":8,"rounds_expected":8,"validation_status":"COMPLETE","best_exploration_speedup":2.447966325812682,"final_speedup":2.3670674096602466,"post_measurement_reporting_tokens":950283,"reasoning_effort":["xhigh"],"usage":{"request_records":152,"input_tokens":23086198,"cached_input_tokens":22921344,"output_tokens":94326,"total_tokens":23180524,"uncached_input_tokens":164854,"reasoning_output_tokens":57260,"monetary_cost":null},"queue_seconds":2485.094491004944,"allocation_process_seconds":1811.510234117508,"observations":[{"role":"exploratory","round_index":1,"candidate_sha256":"68c5f4a4cef7d800aab923af778431bdf0f35dde432717aac53b524b3c3e41d7","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.0362349563628106,"effective_speedup":1.0362349563628106,"best_exploration_speedup":1.0362349563628106,"cleanup_verified":true,"queue_seconds":189.54539728164673,"token_boundary_status":"verified","cumulative_input_tokens":2326190,"cumulative_cached_input_tokens":2235776,"cumulative_output_tokens":11953,"cumulative_total_tokens":2338143,"cumulative_uncached_input_tokens":90414,"cumulative_reasoning_output_tokens":3322,"round_input_tokens":2326190,"round_cached_input_tokens":2235776,"round_output_tokens":11953,"round_total_tokens":2338143,"round_uncached_input_tokens":90414,"allocation_process_seconds":202.2415487766266,"original_observation_sha256":"5bee7c5d9c7c4e15989adb638cdc5fe9f88bd49b9b6de927ca74af856607bea0","original_device_summary_sha256":"830ba82f18d15691a25e1c59d453a79e63c90c0bd832ad15fdda4e7484090ad7","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":2,"candidate_sha256":"854773d84fdbd027ffbe2368dbda499c0e3e76b148958f36f64c54964b4d8b87","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.6738736592100159,"effective_speedup":1.6738736592100159,"best_exploration_speedup":1.6738736592100159,"cleanup_verified":true,"queue_seconds":28.270265340805054,"token_boundary_status":"verified","cumulative_input_tokens":3373553,"cumulative_cached_input_tokens":3274112,"cumulative_output_tokens":16717,"cumulative_total_tokens":3390270,"cumulative_uncached_input_tokens":99441,"cumulative_reasoning_output_tokens":6041,"round_input_tokens":1047363,"round_cached_input_tokens":1038336,"round_output_tokens":4764,"round_total_tokens":1052127,"round_uncached_input_tokens":9027,"allocation_process_seconds":201.46938753128052,"original_observation_sha256":"1280ef75c2cc047a993b87a450c565a323a15de93ce09de049766c68e5b5fd3b","original_device_summary_sha256":"5f04177c05df732f79047521360402f065eece801d02db13f580a1c82a76d0c3","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":3,"candidate_sha256":"993a923efd56399775efc9596dfe3344ff68478a135de42fadf953bf21e5b884","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.123848810256551,"effective_speedup":2.123848810256551,"best_exploration_speedup":2.123848810256551,"cleanup_verified":true,"queue_seconds":265.54253339767456,"token_boundary_status":"verified","cumulative_input_tokens":5591803,"cumulative_cached_input_tokens":5477376,"cumulative_output_tokens":27183,"cumulative_total_tokens":5618986,"cumulative_uncached_input_tokens":114427,"cumulative_reasoning_output_tokens":10249,"round_input_tokens":2218250,"round_cached_input_tokens":2203264,"round_output_tokens":10466,"round_total_tokens":2228716,"round_uncached_input_tokens":14986,"allocation_process_seconds":214.5753481388092,"original_observation_sha256":"270ed06158c99b002705b92846d0e9aecc7673ddcdf2804de18e0b961d30a722","original_device_summary_sha256":"f507e98e77386726f92832956b8d3a349d9bfb90b20ee509a96b5b191e15a8c8","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":4,"candidate_sha256":"0016258c26a8ac4b9ababeed1b8fcf13871c0578f52965df3b05eec4a55746ac","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"FAIL","valid_measurement":false,"aggregation":"geometric_mean_case_speedups","speedup":null,"effective_speedup":null,"best_exploration_speedup":2.123848810256551,"cleanup_verified":true,"queue_seconds":320.7573881149292,"token_boundary_status":"verified","cumulative_input_tokens":7706550,"cumulative_cached_input_tokens":7583360,"cumulative_output_tokens":33864,"cumulative_total_tokens":7740414,"cumulative_uncached_input_tokens":123190,"cumulative_reasoning_output_tokens":14539,"round_input_tokens":2114747,"round_cached_input_tokens":2105984,"round_output_tokens":6681,"round_total_tokens":2121428,"round_uncached_input_tokens":8763,"allocation_process_seconds":30.503424167633057,"original_observation_sha256":"8d7f6232250cd9523a6f5700f7d3442ef71c3915c4ef7cc67072b76ba5045e8a","original_device_summary_sha256":null,"attempt":1,"round_consumed":true,"execution_status":"candidate_failure","performance_verdict":"INCONCLUSIVE","failure_reason":"Candidate rejected on both ranks before timing: installed npu_moe_finalize_routing does not accept keyword argument 'k'."},{"role":"exploratory","round_index":5,"candidate_sha256":"73a72a0c29de1a0961e874a66173324002d3c80e0702cfc11f907258090713b1","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.241795215896914,"effective_speedup":2.241795215896914,"best_exploration_speedup":2.241795215896914,"cleanup_verified":true,"queue_seconds":0.14263057708740234,"token_boundary_status":"verified","cumulative_input_tokens":8988213,"cumulative_cached_input_tokens":8859136,"cumulative_output_tokens":38795,"cumulative_total_tokens":9027008,"cumulative_uncached_input_tokens":129077,"cumulative_reasoning_output_tokens":18100,"round_input_tokens":1281663,"round_cached_input_tokens":1275776,"round_output_tokens":4931,"round_total_tokens":1286594,"round_uncached_input_tokens":5887,"allocation_process_seconds":203.39757227897644,"original_observation_sha256":"3cf70528bb00d2ad298d2bf8b4b154d709bd1e43237d0e938220a4fd1ee8619b","original_device_summary_sha256":"236fd4a0e02602c5d323cf55934c2b662e66f8b064d0bc224f8783aa1fbccd71","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":6,"candidate_sha256":"b0bede9598ec1737ae9932d636dda6067f8a1073c7711bb523b67015d3eb4539","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.152762300865101,"effective_speedup":2.152762300865101,"best_exploration_speedup":2.241795215896914,"cleanup_verified":true,"queue_seconds":437.6984884738922,"token_boundary_status":"verified","cumulative_input_tokens":12473969,"cumulative_cached_input_tokens":12337920,"cumulative_output_tokens":52291,"cumulative_total_tokens":12526260,"cumulative_uncached_input_tokens":136049,"cumulative_reasoning_output_tokens":27636,"round_input_tokens":3485756,"round_cached_input_tokens":3478784,"round_output_tokens":13496,"round_total_tokens":3499252,"round_uncached_input_tokens":6972,"allocation_process_seconds":202.65567922592163,"original_observation_sha256":"9a06b0bd99202c2e7be4886a19ee6920c07127816c8858f362f914fd35ca0cf2","original_device_summary_sha256":"3290997ad14baf18b0244bbf0b03d4f55a0f7b326a8ddb4aa9ce78845426f379","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":7,"candidate_sha256":"1561c1fb8f1bfdcd625802e69146b24d1b69958b22dee3d855921fe4eceb657c","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.447966325812682,"effective_speedup":2.447966325812682,"best_exploration_speedup":2.447966325812682,"cleanup_verified":true,"queue_seconds":14.20435643196106,"token_boundary_status":"verified","cumulative_input_tokens":13971152,"cumulative_cached_input_tokens":13830784,"cumulative_output_tokens":59664,"cumulative_total_tokens":14030816,"cumulative_uncached_input_tokens":140368,"cumulative_reasoning_output_tokens":32127,"round_input_tokens":1497183,"round_cached_input_tokens":1492864,"round_output_tokens":7373,"round_total_tokens":1504556,"round_uncached_input_tokens":4319,"allocation_process_seconds":204.89156699180603,"original_observation_sha256":"4c5e60c54b42447cb8539f83e4b5b7523bc04b49d4e76936378d6d49ebf0a4c0","original_device_summary_sha256":"23049f408ccf531dfa9065950f36646333693257b0ccb3586a6369163a6c621b","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":8,"candidate_sha256":"aea3a29f087b3ff6459b5e799f67c0aba829aed03da3da2f03af93cecc022f12","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.3960949518750145,"effective_speedup":2.3960949518750145,"best_exploration_speedup":2.447966325812682,"cleanup_verified":true,"queue_seconds":809.058810710907,"token_boundary_status":"verified","cumulative_input_tokens":19474012,"cumulative_cached_input_tokens":19325056,"cumulative_output_tokens":77753,"cumulative_total_tokens":19551765,"cumulative_uncached_input_tokens":148956,"cumulative_reasoning_output_tokens":45293,"round_input_tokens":5502860,"round_cached_input_tokens":5494272,"round_output_tokens":18089,"round_total_tokens":5520949,"round_uncached_input_tokens":8588,"allocation_process_seconds":204.87169575691223,"original_observation_sha256":"7e5c4b1cafc3380ac5d0e6c4de84b0ed93fcdc1fae60127d57661dd0dbb94bc9","original_device_summary_sha256":"6b6587878baac3abb2b175cc780c11735d1a716c2dfbc682e9138f2d0144ad2c","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"final","round_index":null,"candidate_sha256":"1561c1fb8f1bfdcd625802e69146b24d1b69958b22dee3d855921fe4eceb657c","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.3670674096602466,"effective_speedup":2.3670674096602466,"best_exploration_speedup":2.447966325812682,"cleanup_verified":true,"queue_seconds":419.87462067604065,"token_boundary_status":"verified","cumulative_input_tokens":22139428,"cumulative_cached_input_tokens":21984896,"cumulative_output_tokens":90813,"cumulative_total_tokens":22230241,"cumulative_uncached_input_tokens":154532,"cumulative_reasoning_output_tokens":55046,"round_input_tokens":2665416,"round_cached_input_tokens":2659840,"round_output_tokens":13060,"round_total_tokens":2678476,"round_uncached_input_tokens":5576,"allocation_process_seconds":346.90401124954224,"original_observation_sha256":"53a9ddfe7bfc0d10ac485dfb480307a516748783144c902d34f5be1a7b69b10a","original_device_summary_sha256":"41586d088e9c164aba002ec94de0d0bb4d9e117419ba15e29d84f86e119f9974","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null}],"selected_round":7,"final_candidate_sha256":"1561c1fb8f1bfdcd625802e69146b24d1b69958b22dee3d855921fe4eceb657c"},{"key":"gpt6astra-kb-off","label":"GPT-6 Astra / 知识库关","model":"gpt-6-astra","knowledge":"off","status":"completed","rounds_completed":8,"rounds_expected":8,"validation_status":"COMPLETE","best_exploration_speedup":2.43652966789247,"final_speedup":2.339809467070293,"post_measurement_reporting_tokens":600255,"reasoning_effort":["xhigh"],"usage":{"request_records":104,"input_tokens":10365965,"cached_input_tokens":10253184,"output_tokens":48688,"total_tokens":10414653,"uncached_input_tokens":112781,"reasoning_output_tokens":22922,"monetary_cost":null},"queue_seconds":1.2802674770355225,"allocation_process_seconds":1790.7148840427399,"observations":[{"role":"exploratory","round_index":1,"candidate_sha256":"c7b20dfbc23f611feb5022f08e0d40e428e2b365062b74229d6a64ca7946958b","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.0331759575264898,"effective_speedup":1.0331759575264898,"best_exploration_speedup":1.0331759575264898,"cleanup_verified":true,"queue_seconds":0.14717555046081543,"token_boundary_status":"verified","cumulative_input_tokens":932457,"cumulative_cached_input_tokens":870144,"cumulative_output_tokens":6215,"cumulative_total_tokens":938672,"cumulative_uncached_input_tokens":62313,"cumulative_reasoning_output_tokens":1502,"round_input_tokens":932457,"round_cached_input_tokens":870144,"round_output_tokens":6215,"round_total_tokens":938672,"round_uncached_input_tokens":62313,"allocation_process_seconds":197.50759291648865,"original_observation_sha256":"4e777a35eb6a66276ae2a25b3dd71e1917e3dbf35257771511e59bbcd11524d7","original_device_summary_sha256":"0bff218e2a368d7f0cf4e7f03b29d59b16c9c36b9fb528aa9e5b526d94e53416","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":2,"candidate_sha256":"b0f9d88f3fa94668744c188a7f7964160aced4bd1f3e1f9ada924b397292e536","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":1.6526363575584917,"effective_speedup":1.6526363575584917,"best_exploration_speedup":1.6526363575584917,"cleanup_verified":true,"queue_seconds":0.14017701148986816,"token_boundary_status":"verified","cumulative_input_tokens":2188818,"cumulative_cached_input_tokens":2117504,"cumulative_output_tokens":10848,"cumulative_total_tokens":2199666,"cumulative_uncached_input_tokens":71314,"cumulative_reasoning_output_tokens":3615,"round_input_tokens":1256361,"round_cached_input_tokens":1247360,"round_output_tokens":4633,"round_total_tokens":1260994,"round_uncached_input_tokens":9001,"allocation_process_seconds":196.23589491844177,"original_observation_sha256":"d46ef9042d8b045bc08cee16f266859d2b1bb1ab1fbae42e726cc8452e1abe07","original_device_summary_sha256":"9bbc4a9c594d92c39e45a7bac227e2f1ca78521db3f46c857191f32b40eceb20","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":3,"candidate_sha256":"ef5c55d92cd74bd904f3130eba476a682c3d17d4113e4163a41c0663a89735e9","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.43652966789247,"effective_speedup":2.43652966789247,"best_exploration_speedup":2.43652966789247,"cleanup_verified":true,"queue_seconds":0.14071369171142578,"token_boundary_status":"verified","cumulative_input_tokens":3493062,"cumulative_cached_input_tokens":3414528,"cumulative_output_tokens":17130,"cumulative_total_tokens":3510192,"cumulative_uncached_input_tokens":78534,"cumulative_reasoning_output_tokens":5314,"round_input_tokens":1304244,"round_cached_input_tokens":1297024,"round_output_tokens":6282,"round_total_tokens":1310526,"round_uncached_input_tokens":7220,"allocation_process_seconds":202.84937405586243,"original_observation_sha256":"fc659a40ccf616b44e7347727511a54a75983287d07216f47ec0392bbc355071","original_device_summary_sha256":"eb038834d3a8ba83264d94c8e9bb563b2f2afa7fe6f095c8703c38b002aa8897","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":4,"candidate_sha256":"252af26dddc8856a78f6ad112e939c640cf229028820b453d66d46cc838d696e","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"FAIL","valid_measurement":false,"aggregation":"geometric_mean_case_speedups","speedup":null,"effective_speedup":null,"best_exploration_speedup":2.43652966789247,"cleanup_verified":true,"queue_seconds":0.13994932174682617,"token_boundary_status":"verified","cumulative_input_tokens":3796004,"cumulative_cached_input_tokens":3714688,"cumulative_output_tokens":19403,"cumulative_total_tokens":3815407,"cumulative_uncached_input_tokens":81316,"cumulative_reasoning_output_tokens":6829,"round_input_tokens":302942,"round_cached_input_tokens":300160,"round_output_tokens":2273,"round_total_tokens":305215,"round_uncached_input_tokens":2782,"allocation_process_seconds":29.44064688682556,"original_observation_sha256":"6dd72f28d934c853fc104dc2a903f3e8bde2f35273146f0fd4ed759901793800","original_device_summary_sha256":null,"attempt":1,"round_consumed":true,"execution_status":"candidate_failure","performance_verdict":"INCONCLUSIVE","failure_reason":"Candidate rejected on both ranks before timing: installed npu_moe_finalize_routing does not accept keyword argument 'k'."},{"role":"exploratory","round_index":5,"candidate_sha256":"38ac84ca35b444caf4f71854bbe62211f4c359b0bd07bc95192539a1d3d0cb0e","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.3047518247039847,"effective_speedup":2.3047518247039847,"best_exploration_speedup":2.43652966789247,"cleanup_verified":true,"queue_seconds":0.14074420928955078,"token_boundary_status":"verified","cumulative_input_tokens":4868843,"cumulative_cached_input_tokens":4782080,"cumulative_output_tokens":22263,"cumulative_total_tokens":4891106,"cumulative_uncached_input_tokens":86763,"cumulative_reasoning_output_tokens":8530,"round_input_tokens":1072839,"round_cached_input_tokens":1067392,"round_output_tokens":2860,"round_total_tokens":1075699,"round_uncached_input_tokens":5447,"allocation_process_seconds":203.85016012191772,"original_observation_sha256":"cec9ac465a66ef9424d86d65692f8fa2552abb26cea9300eb003e8f12bdc3908","original_device_summary_sha256":"25bc92b63164b9847cb9f49069ac5f3d5979af59799ad0fcb6bf83d743f91646","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":6,"candidate_sha256":"84ec3d844910f2ac40079165f5ab0bbe4263a7b53315fbe1040fc645c63170d2","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.339274874728824,"effective_speedup":2.339274874728824,"best_exploration_speedup":2.43652966789247,"cleanup_verified":true,"queue_seconds":0.1399071216583252,"token_boundary_status":"verified","cumulative_input_tokens":5663395,"cumulative_cached_input_tokens":5572992,"cumulative_output_tokens":26064,"cumulative_total_tokens":5689459,"cumulative_uncached_input_tokens":90403,"cumulative_reasoning_output_tokens":11283,"round_input_tokens":794552,"round_cached_input_tokens":790912,"round_output_tokens":3801,"round_total_tokens":798353,"round_uncached_input_tokens":3640,"allocation_process_seconds":199.04994177818298,"original_observation_sha256":"81b0f30da84387c24805247c289fbb390b651389f2f02e51ab658f0ba58f576d","original_device_summary_sha256":"1314f6b6aa455e680c24b9352834c4176e92b479d7919d0afa3bbc88e1b57928","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":7,"candidate_sha256":"b84a624eb193224e3fd1f8c1bd087d080b2d2ac46c713c5cd808543ae64da0c3","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.4042781514496108,"effective_speedup":2.4042781514496108,"best_exploration_speedup":2.43652966789247,"cleanup_verified":true,"queue_seconds":0.14899277687072754,"token_boundary_status":"verified","cumulative_input_tokens":6734054,"cumulative_cached_input_tokens":6639488,"cumulative_output_tokens":28749,"cumulative_total_tokens":6762803,"cumulative_uncached_input_tokens":94566,"cumulative_reasoning_output_tokens":12859,"round_input_tokens":1070659,"round_cached_input_tokens":1066496,"round_output_tokens":2685,"round_total_tokens":1073344,"round_uncached_input_tokens":4163,"allocation_process_seconds":202.39580416679382,"original_observation_sha256":"90e64f6944b52d03906dfd93d8db19169f3fab3a1b32fa6ee3365266a8b86f39","original_device_summary_sha256":"b045b71e03a07a3322b9c1003ac3b1f3cf9092281d5904d82284caed0583a4f5","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"exploratory","round_index":8,"candidate_sha256":"0d784712015edc1ad94393e15ee19152e660b2288ad66213d886b4ea7972c4f0","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.2913687750805463,"effective_speedup":2.2913687750805463,"best_exploration_speedup":2.43652966789247,"cleanup_verified":true,"queue_seconds":0.14332795143127441,"token_boundary_status":"verified","cumulative_input_tokens":7740861,"cumulative_cached_input_tokens":7642368,"cumulative_output_tokens":33855,"cumulative_total_tokens":7774716,"cumulative_uncached_input_tokens":98493,"cumulative_reasoning_output_tokens":15414,"round_input_tokens":1006807,"round_cached_input_tokens":1002880,"round_output_tokens":5106,"round_total_tokens":1011913,"round_uncached_input_tokens":3927,"allocation_process_seconds":202.17194604873657,"original_observation_sha256":"994f964dc2b66cc39d97e3fa0868afb816d52dc0a544f7a28684967151cdae90","original_device_summary_sha256":"7f329857b8dc8a3b5b949f1b95587b66431cdc4d7bdb4c08505a585e5a468285","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null},{"role":"final","round_index":null,"candidate_sha256":"ef5c55d92cd74bd904f3130eba476a682c3d17d4113e4163a41c0663a89735e9","task_sha256":"36d6004d0ac15125748279fdb47975ffc89f9d014145bb96bfb4406d7e986bae","grader_sha256":"34f8068e6a9d1fd1a09c8b55107ac42d2f8e6dbc86e16ae4232f3ecc6a24520e","source_revision":"6ed93bb41f7db9dafb420dcb6f8705f1864bc45c","source_bundle_sha256":"174adad7cc826f53da6d0b633768807e7bf531437d3c3464352d35abae7caae7","correctness":"PASS","valid_measurement":true,"aggregation":"geometric_mean_case_speedups","speedup":2.339809467070293,"effective_speedup":2.339809467070293,"best_exploration_speedup":2.43652966789247,"cleanup_verified":true,"queue_seconds":0.13927984237670898,"token_boundary_status":"verified","cumulative_input_tokens":9772433,"cumulative_cached_input_tokens":9667328,"cumulative_output_tokens":41965,"cumulative_total_tokens":9814398,"cumulative_uncached_input_tokens":105105,"cumulative_reasoning_output_tokens":19233,"round_input_tokens":2031572,"round_cached_input_tokens":2024960,"round_output_tokens":8110,"round_total_tokens":2039682,"round_uncached_input_tokens":6612,"allocation_process_seconds":357.21352314949036,"original_observation_sha256":"b14eb5765736a8e5ed2eaf1cc1284149d9c160ed09c5996ea681d9d14ef6ae44","original_device_summary_sha256":"2d874b3a7d18d7f49af2e54e465669ad48b9b14cb1eb46e43c7aa5c46ec68db3","attempt":1,"round_consumed":true,"execution_status":"completed","performance_verdict":"PASS","failure_reason":null}],"selected_round":3,"final_candidate_sha256":"ef5c55d92cd74bd904f3130eba476a682c3d17d4113e4163a41c0663a89735e9"}]} diff --git a/evidence/moe-agent-comparison-20261104/validator/README.md b/evidence/moe-agent-comparison-20261104/validator/README.md new file mode 100644 index 000000000..cb4ac4694 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/validator/README.md @@ -0,0 +1,15 @@ +# Frozen historical task and grader + +`task/` and `runner/grade.py` preserve the original bytes used by the experiment. +Their hashes are checked against `../plan/publication.json` and the public bundle +integrity map. Source revision and omitted source identities are also retained. +These files explain and permit review of the mathematical, input and timing +protocol; they are not a portable launch command or a test of the PR's new +TaskPackage. Host-specific launch configuration is deliberately not published. + +`../verify.py` is a separate, CPU-only publication checker. It recomputes all +case medians, speedups, best-so-far states and token sums; verifies source and +bundle bytes; and checks counts/selection against the public audit extracts. +It never imports candidates, runs the frozen grader, starts a device job, or +changes a historical verdict. Full raw rank calls, original condition audits +and session logs remain external and are explicitly unavailable in this checkout. diff --git a/evidence/moe-agent-comparison-20261104/validator/runner/grade.py b/evidence/moe-agent-comparison-20261104/validator/runner/grade.py new file mode 100644 index 000000000..f0500f174 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/validator/runner/grade.py @@ -0,0 +1,329 @@ +"""Trusted two-rank correctness and full-forward timer, inside torchrun only.""" +import copy +import datetime as dt +import hashlib +import importlib.util +import json +import math +import os +from pathlib import Path +import statistics +import sys +import time +import traceback + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "results" +sys.path.insert(0, str(ROOT / "task")) + + +def stamp(): + return dt.datetime.now(dt.timezone.utc).isoformat() + + +def write(path, value): + temp = path.with_suffix(path.suffix + ".tmp") + temp.write_text(json.dumps(value, indent=2, ensure_ascii=False), encoding="utf-8") + temp.replace(path) + + +def load(path, name): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if not callable(getattr(module, "compute", None)): + raise ValueError("Candidate must define compute(x, params, *, config, group)") + return module.compute + + +def tensor_snapshot(tensor): + # BF16 is represented as bytes without asking numpy to support its dtype. + cpu = tensor.detach().cpu().contiguous() + return {"shape": list(tensor.shape), "dtype": str(tensor.dtype), "device": str(tensor.device), + "stride": list(tensor.stride()), "storage_offset": tensor.storage_offset(), + "sha256": hashlib.sha256(cpu.view(torch.uint8).numpy().tobytes()).hexdigest()} + + +def snapshots(item): + return {"x": tensor_snapshot(item["x"]), + "params": {k: tensor_snapshot(v) for k, v in item["params"].items()}, + "config": copy.deepcopy(item["config"])} + + +def check_output(output, item, expected, before, tolerance): + spec = before["x"] + shape = (isinstance(output, torch.Tensor) and list(output.shape) == spec["shape"] + and tuple(output.shape) == tuple(expected.shape)) + identity = shape and str(output.dtype) == spec["dtype"] and str(output.device) == spec["device"] + result = {"shape_dtype_device": bool(identity), "input_parameters_preserved": snapshots(item) == before} + if not identity: + result.update(finite=False, values=False, nrmse=None, max_abs_error=None) + return result + actual = output.detach().float().cpu() + finite = bool(torch.isfinite(actual).all()) + diff = actual - expected + denom = max(float(expected.square().mean().sqrt()), 1e-12) if expected.numel() else 1.0 + nrmse = float(diff.square().mean().sqrt()) / denom if expected.numel() else 0.0 + result.update(finite=finite, values=finite and bool(torch.allclose( + actual, expected, atol=tolerance["atol"], rtol=tolerance["rtol"])), + nrmse=nrmse, nrmse_pass=math.isfinite(nrmse) and nrmse <= tolerance["nrmse"], + max_abs_error=float(diff.abs().max()) if diff.numel() else 0.0) + return result + + +def valid(checks): + return all(checks.get(k) is True for k in ( + "shape_dtype_device", "input_parameters_preserved", "finite", "values", "nrmse_pass")) + + +rank = int(os.environ["RANK"]) +local_rank = int(os.environ["LOCAL_RANK"]) +record = {"rank": rank, "started_at_utc": stamp(), "completed": False, + "correctness": "INCONCLUSIVE", "calls": [], "samples": [], "phase": "initializing"} +path = OUT / f"rank-{rank}.json" +write(path, record) +initialized = False +try: + import torch + import torch_npu + import torch.distributed as dist + from torch_npu.npu.utils import get_cann_version + from workload import load_task, load_cases, build_case, cpu_input, warm_input + from oracle import expected as oracle_expected + from oracle import expected_from_input + + torch.set_num_threads(8) + assert int(os.environ["WORLD_SIZE"]) == 2 + assert torch.npu.device_count() == 2 + torch.npu.set_device(local_rank) + torch.npu.matmul.allow_hf32 = False + record["phase"] = "hccl_initialization" + write(path, record) + dist.init_process_group("hccl", timeout=dt.timedelta(seconds=150)) + initialized = True + dist.barrier() + torch.npu.synchronize() + request = json.loads((ROOT / "request.json").read_text()) + task = load_task() + cases = load_cases() + tolerance = task["tolerance"] + measure = task["measurement"] + record["runtime"] = {"torch": torch.__version__, "torch_npu": torch_npu.__version__, + "cann": str(get_cann_version()), "python": sys.version.split()[0], + "device": torch.npu.get_device_name(local_rank), "logical_device": local_rank, + "allocated_physical": os.environ.get("ASCEND_RT_VISIBLE_DEVICES"), + "matmul_allow_hf32": bool(torch.npu.matmul.allow_hf32)} + record["phase"] = "baseline_import" + write(path, record) + functions = {"baseline": load(ROOT / "task" / "baseline.py", "trusted_baseline")} + record["phase"] = "candidate_import" + write(path, record) + functions["candidate"] = load(ROOT / "candidate" / "implementation.py", "evaluated_candidate") + + parameter_bundles = {} + configs = {} + expected_cache = {} + prepared_inputs = {} + + def prepare_case(case, case_index): + """Prepare both sides before timing, including all expensive CPU work. + + Computing the oracle just before the first implementation of a pair + creates an asymmetric idle gap. Neither side may pay that gap alone. + """ + started = time.monotonic() + record["phase"] = "trusted_case_preparation" + record["active_case"] = case["id"] + write(path, record) + base_seed = 100000 + case_index * 10000 + seeds = [base_seed, base_seed + 1] + timed_seeds = [] + if case["measured"]: + seeds.extend(base_seed + 10 + i for i in range(measure["warmups"])) + timed_seeds = [base_seed + 100 + pair * 100 + sample + for pair in range(request["repetitions"]) + for sample in range(measure["samples_per_pair"])] + seeds.extend(timed_seeds) + # All margin checks, FP32 oracle GEMMs and host tensors are prepared + # before either label's correctness/warmup/measurement starts. + host_inputs = {(seed, "standard"): cpu_input(case, rank, seed) for seed in seeds} + expected_cache.clear() + expected_cache.update({(seed, "standard"): oracle_expected(case, seed, rank) for seed in seeds}) + warm_started = time.monotonic() + warm_pairs = [] + for seed in timed_seeds: + warmed = warm_input(case, rank, seed) + warm_expected = expected_from_input(case, warmed) + original, original_expected = host_inputs[(seed, "standard")], expected_cache[(seed, "standard")] + if original.numel(): + assert not torch.equal(original, warmed), "Shape warmup must use different input values" + assert not torch.allclose(original_expected, warm_expected, + atol=tolerance["atol"], rtol=tolerance["rtol"]), "Shape warmup must have a different output" + input_delta = (warmed.float() - original.float()).square().mean(dim=1).sqrt() + output_delta = (warm_expected - original_expected).square().mean().sqrt() + output_scale = max(float(original_expected.square().mean().sqrt()), 1e-12) + difference_nrmse = float(output_delta) / output_scale + assert math.isfinite(difference_nrmse) and difference_nrmse > 0.1, "Shape warmup output must prevent replay" + warm_pairs.append({"seed": seed, "minimum_row_input_rms_difference": float(input_delta.min()), + "output_nrmse_difference": difference_nrmse, + "output_outside_tolerance": True}) + host_inputs[(seed, "shape_warmup")] = warmed + expected_cache[(seed, "shape_warmup")] = warm_expected + warm_preparation_seconds = time.monotonic() - warm_started + prepared_inputs.clear() + for label in ("baseline", "candidate"): + bundle_key = (case["id"], label) + item = build_case(case, rank, torch.device(f"npu:{local_rank}"), base_seed) + if case["id"] == "weight-update": + previous = parameter_bundles[("unequal-top1", label)] + for name, value in item["params"].items(): + previous[name].copy_(value) + item["params"] = previous + parameter_bundles[bundle_key] = item["params"] + configs[bundle_key] = copy.deepcopy(item["config"]) + for (seed, variant), host_input in host_inputs.items(): + prepared_inputs[(seed, label, variant)] = host_input.to(f"npu:{local_rank}", copy=True) + torch.npu.synchronize() + record.setdefault("case_preparation", []).append({"case_id": case["id"], + "distinct_seeds": len(seeds), "shape_warmup_inputs": len(timed_seeds), + "shape_warmup_preparation_seconds": warm_preparation_seconds, + "shape_warmup_pairs": warm_pairs, + "elapsed_seconds": time.monotonic() - started}) + write(path, record) + + def call(case, seed, label, phase, *, timed=False, pair=None, sample=None, variant="standard"): + record["phase"] = "trusted_pre_call_validation" + record["active_case"] = case["id"] + write(path, record) + # Production model parameters stay stable across forward calls. The + # two implementations own separate, identical parameter tensors. + bundle_key = (case["id"], label) + item = {"x": prepared_inputs[(seed, label, variant)], "params": parameter_bundles[bundle_key], + "config": copy.deepcopy(configs[bundle_key])} + before = snapshots(item) + expected = expected_cache[(seed, variant)] + assert expected.device.type == "cpu" and expected.dtype == torch.float32 + # This barrier is outside the measured interval. synchronize completes + # HCCL/device work before starting the full host-to-device call timer. + record["phase"] = label + "_" + phase + write(path, record) + try: + dist.barrier() + torch.npu.synchronize() + except BaseException: + record["phase"] = "trusted_rank_alignment_failure" + write(path, record) + raise + started = time.perf_counter_ns() + with torch.inference_mode(): + output = functions[label](item["x"], item["params"], config=item["config"], group=dist.group.WORLD) + torch.npu.synchronize() + local_ms = (time.perf_counter_ns() - started) / 1e6 + record["phase"] = "trusted_output_validation" + write(path, record) + checks = check_output(output, item, expected, before, tolerance) + good = torch.tensor([int(valid(checks))], device=item["x"].device, dtype=torch.int32) + dist.all_reduce(good, op=dist.ReduceOp.MIN) + torch.npu.synchronize() + call_record = {"case_id": case["id"], "seed": seed, "label": label, + "phase": phase, "variant": variant, "pair": pair, "sample": sample, "checks": checks, + "input_sha256": before["x"]["sha256"], "all_ranks_correct": bool(good.item()), + "local_elapsed_ms": local_ms, "included_in_score": timed} + record["calls"].append(call_record) + if not call_record["all_ranks_correct"]: + record["correctness"] = "FAIL" if label == "candidate" else "INCONCLUSIVE" + record["failure_kind"] = "candidate_invalid_output" if label == "candidate" else "baseline_invalid_output" + write(path, record) + raise ValueError(f"{label} {case['id']} failed independent correctness") + if timed: + elapsed = torch.tensor([local_ms], device=item["x"].device, dtype=torch.float32) + dist.all_reduce(elapsed, op=dist.ReduceOp.MAX) + torch.npu.synchronize() + slowest = float(elapsed.item()) + if not math.isfinite(slowest) or slowest <= 0: + raise ValueError("Invalid completed-forward time") + record["samples"].append({**{k: call_record[k] for k in ( + "case_id", "seed", "label", "pair", "sample", "input_sha256")}, + "local_elapsed_ms": local_ms, "slowest_rank_elapsed_ms": slowest}) + # Explicitly overwrite returned storage after verification. Retaining + # an output object for replay cannot satisfy a later changed input. + record["phase"] = "trusted_output_reset" + write(path, record) + with torch.inference_mode(): + output.zero_() + torch.npu.synchronize() + write(path, record) + + for case_index, case in enumerate(cases): + prepare_case(case, case_index) + # Each verification and timed input changes. Both implementations use + # exactly the same inputs/weights, constructed independently. + base_seed = 100000 + case_index * 10000 + for variant in range(2): + for label in ("baseline", "candidate"): + call(case, base_seed + variant, label, "correctness") + if not case["measured"]: + continue + for warmup in range(measure["warmups"]): + for label in ("baseline", "candidate"): + call(case, base_seed + 10 + warmup, label, "warmup") + for pair in range(request["repetitions"]): + order = ("baseline", "candidate") if pair % 2 == 0 else ("candidate", "baseline") + for sample in range(measure["samples_per_pair"]): + seed = base_seed + 100 + pair * 100 + sample + # Both implementations see their own shape-matched but + # numerically different input before either timed call. This + # pays dynamic operator preparation independently of whether + # the other implementation happens to use the same operators. + for label in order: + call(case, seed, label, "shape_warmup", pair=pair, sample=sample, + variant="shape_warmup") + for label in order: + call(case, seed, label, "measurement", + timed=True, pair=pair, sample=sample) + + record["phase"] = "trusted_summary" + write(path, record) + result_cases = [] + for case in cases: + if not case["measured"]: + continue + grouped = {label: [s["slowest_rank_elapsed_ms"] for s in record["samples"] + if s["case_id"] == case["id"] and s["label"] == label] + for label in ("baseline", "candidate")} + expected_count = request["repetitions"] * measure["samples_per_pair"] + assert all(len(values) == expected_count for values in grouped.values()) + b, c = (statistics.median(grouped[label]) for label in ("baseline", "candidate")) + result_cases.append({"id": case["id"], "baseline_median_ms": b, + "candidate_median_ms": c, "speedup": b / c, + "samples_ms": grouped}) + speedup = math.exp(statistics.mean(math.log(c["speedup"]) for c in result_cases)) + record.update(completed=True, correctness="PASS", phase="completed", cases=result_cases, + speedup=speedup, aggregation="geometric_mean_case_speedups", + measurement_protocol="shape-matched-different-values-warmup-both-labels-before-paired-timing", + shape_warmup_cost={label: { + "calls": sum(c["phase"] == "shape_warmup" and c["label"] == label for c in record["calls"]), + "local_forward_ms": sum(c["local_elapsed_ms"] for c in record["calls"] + if c["phase"] == "shape_warmup" and c["label"] == label)} + for label in ("baseline", "candidate")}, + peak_allocated_bytes=int(torch.npu.max_memory_allocated(local_rank))) + if rank == 0: + summary = {k: record[k] for k in ( + "completed", "correctness", "cases", "speedup", "aggregation", "measurement_protocol")} + summary["metrics"] = {"suite_latency_ms": sum(c["candidate_median_ms"] for c in result_cases)} + write(OUT / "summary.json", summary) + write(Path(os.environ["INFERRSI_RESULT_PATH"]), summary) +except BaseException as error: + record["error"] = repr(error) + record["traceback"] = traceback.format_exc() +finally: + record["completed_at_utc"] = stamp() + write(path, record) + if initialized: + # Fatal candidate exceptions can leave the peer in a collective. + # torchrun plus the outer bounded driver owns full-job termination. + try: + dist.destroy_process_group() + except BaseException: + pass +sys.exit(0 if record["completed"] else 2) diff --git a/evidence/moe-agent-comparison-20261104/validator/task/TASK.md b/evidence/moe-agent-comparison-20261104/validator/task/TASK.md new file mode 100644 index 000000000..2d0989c77 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/validator/task/TASK.md @@ -0,0 +1,47 @@ +# 双 NPU：带共享专家的 MoE FFN 完整 forward + +目标是在保持结果正确的前提下，缩短两张 Ascend NPU 上完整 MoE FFN forward 的完成时间。输入是每张卡自己的 token，输出仍保留在输入所属卡。这是一层模型计算，不涉及模型下载、训练或 serving。 + +## 输入与数学语义 + +实现 `compute(x, params, *, config, group)`，返回与 `x` 相同形状、dtype、设备的输出张量。`group` 是已建立的两 rank HCCL process group；设备和物理卡号由评测器调度，候选不创建或销毁 process group、不修改设备可见性。 + +- `x`：本 rank 的 `[N_rank, H]` BF16 输入。两个 rank 的内容不同，token 数可能不同，也可能有一个 rank 没有输入。 +- `config`：`world_size=2`、`rank`、`num_experts=8`、`experts_per_rank=4`、`hidden`、`intermediate`、`top_k`、`rank_tokens`。只包含结构信息。 +- `params.router_weight`：FP32 `[8,H]`；`router_bias`：FP32 `[8]`。 +- `params.expert_gate_up`：BF16 `[4,2I,H]`；`expert_down`：BF16 `[4,H,I]`。rank 0 只有 experts 0–3，rank 1 只有 experts 4–7。 +- `params.shared_gate_up`：BF16 `[2I,H]`；`shared_down`：BF16 `[H,I]`。共享专家和 router 参数在两卡相同。 + +对每个 token，以 FP32 计算 `logits = x @ router_weight.T + router_bias`，选择最大的 `top_k` 项，再对选中的 logits 做 softmax 得到权重。每个专家执行 `W_down(silu(W_gate x) * (W_up x))`，其中 `W_gate`、`W_up` 是 `gate_up` 的前后两半。结果是选中专家输出的加权和，再加共享专家的输出。最终输出 BF16，保持本 rank 原始 token 顺序。 + +不丢 token；没有 dropout、capacity 截断、量化、residual 或额外 norm。生成输入排除了 top-k 边界近似相等的情况，避免以浮点舍入决定不同路由；路由仍必须根据本次输入计算。 + +## 候选边界 + +可以更改本实现中的计算、内存布局和通信组织，可以调用现有 PyTorch/torch_npu/HCCL 或自定义 NPU kernel。使用的工具必须来自当前已配置环境，不能更改共享软件安装。 + +输入和所有参数只读。不得使用评测 seed、输入生成器、oracle、其他 session 的代码或结果；不得根据评测阶段、文件名或计时器选择不同语义；不得省略任何 rank 的工作。不能获取或重建另一 rank 的专家权重并在未计时区内复制，以改变 EP 所有权。不能缓存输入相关的路由、专家计算或最终输出。 + +参数会变化，张量地址也可能复用。若实现中缓存权重排布或编译结果，必须正确失效；首次建立过程在首次调用中记录，其额外内存也要报告。输入相关工作全部在 `compute` 内。候选不允许外部后台计算、写结果文件、修改评测器或创建脱离评测器生命周期的进程。不得 monkeypatch PyTorch、通信库、oracle 或其他全局运行时状态以影响基线和评测器；本执行器接收经审查的可信源码，不是恶意代码隔离沙箱。 + +## 验收与计分 + +评测器使用独立 CPU FP32 数学实现，检查两个 rank 的全部输出、有限性、输入/参数保持和误差。用不同输入、权重变化、top-k=1/2、不等 token 数、空专家以及零输入 rank 检查正确性。数值容差以冻结后的 `task.json` 为准。 + +计时覆盖 host 调度、数据重排、路由、通信、专家计算、共享专家和输出完成。每次以最慢 rank 的完成时间计分；只提交通信或 kernel 而未完成输出不能获得较短时间。baseline/candidate 在同一组设备交错测量，每次计时输入都变化。各性能 case 的加速比用几何平均汇总，同时保留逐 case 数据；编译、首次调用、搜索墙钟和 token 消耗另报。 + +性能集合包含三种规模与路由分布，均为 top-k=2： + +| Case | 每 rank token 数 | Hidden H | 专家 intermediate I | 路由 | +| --- | --- | --- | --- | --- | +| balanced-small | 64 | 512 | 1024 | 均衡 | +| skewed-medium | 256 | 1024 | 2048 | 偏斜 | +| balanced-large | 1024 | 2048 | 4096 | 均衡 | + +每 case 的所有输入和 FP32 oracle 都在开始评测前共同生成，两种实现获得独立但相同的输入张量；避免只有先运行的一方在计时前经历昂贵的 CPU 参考计算。每次计时使用不同输入。探索使用两组 AB/BA 配对、每组 8 个样本，即每个实现每 case 16 个样本；最终复测使用四组、每个实现每 case 32 个样本。额外的小形状边界用例仅检查正确性，不计入速度分数。准备时间和首次调用时间另存；同进程首次调用受共享底层算子缓存影响，不能解释为两个实现各自在全新进程中的冷启动对比。 + +每个计时输入还配有一个**数值不同、专家数量相同的预热输入**。评测器在 router 权重的零空间加入标准差约 0.3 的噪声，量化为 BF16 后逐 token 检查选中专家集合不变、top-k 边界间隔至少 0.02、输入 RMS 差至少 0.15；不满足时重采样，不退化成原输入、token 排列或微小扰动。这样两种实现可以预热同样的动态专家矩阵形状，避免先执行的一方单独承担新形状的底层算子准备开销。每个预热输入仍需独立数学验收、只读检查和输出清零；预热与正式 oracle 的输出必须相差超过正式容差，防止直接复用预热输出。预热成本单列，不计入 forward 速度分数；输入相关计算和输出缓存仍然禁止。 + +本题的 baseline 是可读、完整的 AllGather/本地专家/ReduceScatter 实现，已经合并 gate/up 投影，并使用 NPU SwiGLU。它是正常算法基线，不代表 vLLM-Ascend 最新生产实现的性能；对它的加速比不得解释为对 vLLM 服务或整模型的同等加速。 + +任务结构参考 vLLM-Ascend 的 [AllGather preparation](https://github.com/vllm-project/vllm-ascend/blob/0f85ed44bbbbdc0f0447f83ce62a82a8ad55afb3/vllm_ascend/ops/fused_moe/prepare_finalize.py) 和 [MoE forward](https://github.com/vllm-project/vllm-ascend/blob/0f85ed44bbbbdc0f0447f83ce62a82a8ad55afb3/vllm_ascend/ops/fused_moe/moe_comm_method.py)。本任务的代码是独立编写的最小实现，不依赖整个 vLLM 服务栈。 diff --git a/evidence/moe-agent-comparison-20261104/validator/task/baseline.py b/evidence/moe-agent-comparison-20261104/validator/task/baseline.py new file mode 100644 index 000000000..f7e54be2b --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/validator/task/baseline.py @@ -0,0 +1,67 @@ +"""EP=2 MoE + shared expert: AllGather/local experts/ReduceScatter baseline. + +This is a self-contained, newly written implementation of the common +AllGather-based expert-parallel MoE workload architecture, not a copy of the +vLLM serving stack. It uses a fused gate/up projection and NPU SwiGLU. +""" + +import torch +import torch.distributed as dist +import torch.nn.functional as F + + +def _mlp(x, gate_up, down): + if x.shape[0] == 0: + return x.new_empty((0, down.shape[0])) + projected = F.linear(x, gate_up) + if x.device.type == "npu": + import torch_npu + activated = torch_npu.npu_swiglu(projected, dim=-1) + else: + gate, up = projected.chunk(2, dim=-1) + activated = F.silu(gate) * up + return F.linear(activated, down) + + +def compute(x, params, *, config, group): + world_size, rank = dist.get_world_size(group), dist.get_rank(group) + if world_size != 2 or rank != config["rank"]: + raise ValueError("Expected the configured two-rank process group") + max_tokens = max(config["rank_tokens"]) + hidden = config["hidden"] + logits = ( + F.linear(x.float(), params["router_weight"], params["router_bias"]) + if x.shape[0] else torch.empty((0, 8), dtype=torch.float32, device=x.device) + ) + padded_x = F.pad(x, (0, 0, 0, max_tokens - x.shape[0])) + padded_logits = F.pad(logits, (0, 0, 0, max_tokens - x.shape[0])) + all_x = torch.empty((2 * max_tokens, hidden), dtype=x.dtype, device=x.device) + all_logits = torch.empty((2 * max_tokens, 8), dtype=torch.float32, device=x.device) + dist.all_gather_into_tensor(all_x, padded_x.contiguous(), group=group) + dist.all_gather_into_tensor(all_logits, padded_logits.contiguous(), group=group) + selected_logits, selected_experts = all_logits.topk(config["top_k"], dim=-1, sorted=True) + probabilities = selected_logits.softmax(dim=-1) + # Padded slots have zero weight and never perform an expert MLP. + for source_rank, count in enumerate(config["rank_tokens"]): + if count < max_tokens: + probabilities[source_rank * max_tokens + count:(source_rank + 1) * max_tokens] = 0 + local_contribution = torch.zeros((2 * max_tokens, hidden), dtype=torch.float32, device=x.device) + for local_expert in range(4): + expert_id = rank * 4 + local_expert + positions = torch.nonzero( + (selected_experts == expert_id) & (probabilities > 0), as_tuple=False + ) + if positions.shape[0] == 0: + continue + token_ids, slot_ids = positions[:, 0], positions[:, 1] + values = _mlp( + all_x.index_select(0, token_ids), + params["expert_gate_up"][local_expert], + params["expert_down"][local_expert], + ) + weighted = values.float() * probabilities[token_ids, slot_ids].unsqueeze(-1) + local_contribution.index_add_(0, token_ids, weighted) + local_routed = torch.empty((max_tokens, hidden), dtype=torch.float32, device=x.device) + dist.reduce_scatter_tensor(local_routed, local_contribution, group=group) + shared = _mlp(x, params["shared_gate_up"], params["shared_down"]) + return (local_routed[:x.shape[0]] + shared.float()).to(dtype=x.dtype) diff --git a/evidence/moe-agent-comparison-20261104/validator/task/oracle.py b/evidence/moe-agent-comparison-20261104/validator/task/oracle.py new file mode 100644 index 000000000..9752a6a98 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/validator/task/oracle.py @@ -0,0 +1,90 @@ +"""Independent, unsharded CPU FP32 oracle for the complete mathematical MoE. + +No baseline/candidate, distributed collective, NPU fused operator, permutation +or combine implementation is reused. Input and weight generators are shared +only to define the problem data. +""" + +from functools import lru_cache + +import torch + +from workload import _weight_key, cpu_input, cpu_parameters + + +@lru_cache(maxsize=12) +def _float_parameters(key): + hidden, intermediate, weight_seed, variant, routing = key + case = { + "hidden": hidden, "intermediate": intermediate, + "weight_seed": weight_seed, "weight_variant": variant, + "routing": routing, + } + return {name: value.float() for name, value in cpu_parameters(case, range(8)).items()} + + +def _expert(x, w_gate_up, w_down): + intermediate = w_down.shape[1] + # Separate matmuls deliberately differ from the implementation's fused + # projection, while representing exactly the same mathematical function. + gate = x @ w_gate_up[:intermediate].T + up = x @ w_gate_up[intermediate:].T + activated = (gate / (1.0 + torch.exp(-gate))) * up + return activated @ w_down.T + + +@torch.inference_mode() +def expected(case, seed, rank): + return expected_from_input(case, cpu_input(case, rank, seed)) + + +@torch.inference_mode() +def expected_from_input(case, x_cpu): + """Evaluate supplied problem data without using any candidate routing.""" + if x_cpu.device.type != "cpu" or x_cpu.dtype != torch.bfloat16: + raise ValueError("Oracle input must be CPU BF16 problem data") + if x_cpu.ndim != 2 or x_cpu.shape[1] != int(case["hidden"]): + raise ValueError("Oracle input shape does not match the case") + x = x_cpu.float() + if x.shape[0] == 0: + return x + params = _float_parameters(_weight_key(case)) + logits = x @ params["router_weight"].T + params["router_bias"] + # Sorting is an independent selection implementation. Generated input + # margins exclude top-k boundary ties. + order = torch.argsort(logits, dim=1, descending=True) + chosen = order[:, :int(case["top_k"])] + chosen_logits = logits.gather(1, chosen) + exponents = torch.exp(chosen_logits - chosen_logits.max(dim=1, keepdim=True).values) + probabilities = exponents / exponents.sum(dim=1, keepdim=True) + result = _expert(x, params["shared_gate_up"], params["shared_down"]) + for expert_id in range(8): + # Weighted output is accumulated directly at the original token row; + # no distributed packing/permutation logic is involved. + rows, slots = torch.where(chosen == expert_id) + if rows.numel() == 0: + continue + expert_values = _expert( + x[rows], params["expert_gate_up"][expert_id], params["expert_down"][expert_id] + ) + result[rows] += expert_values * probabilities[rows, slots, None] + return result.contiguous() + + +def metrics(actual, expected_output): + """Useful calibration diagnostics; verdict thresholds belong to task.json.""" + values = actual.detach().cpu().float() + reference = expected_output.detach().cpu().float() + if values.shape != reference.shape: + return {"shape_match": False, "finite": False} + if values.numel() == 0: + return {"shape_match": True, "finite": True, "max_abs": 0.0, "nrmse": 0.0} + error = values - reference + reference_rms = torch.sqrt(torch.mean(reference.square())).item() + return { + "shape_match": True, + "finite": bool(torch.isfinite(values).all()), + "max_abs": float(error.abs().max()), + "nrmse": float(torch.sqrt(torch.mean(error.square()))) / max(reference_rms, 1e-12), + "reference_rms": reference_rms, + } diff --git a/evidence/moe-agent-comparison-20261104/validator/task/task.json b/evidence/moe-agent-comparison-20261104/validator/task/task.json new file mode 100644 index 000000000..d2d3a82b8 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/validator/task/task.json @@ -0,0 +1,181 @@ +{ + "schema_version": 1, + "id": "moe-shared-ep2-npu-v1", + "title": "Expert-parallel MoE FFN with a shared expert", + "status": "frozen", + "backend": "ascend", + "world_size": 2, + "dtype": "bfloat16", + "entrypoint": "compute(x, params, *, config, group)", + "num_experts": 8, + "experts_per_rank": 4, + "semantics": { + "router": "FP32 linear with bias; sorted top-k logits; softmax over selected logits", + "expert": "linear_gate_up -> silu(gate) * up -> linear_down", + "shared": "one replicated SwiGLU expert evaluated on this rank's input", + "output": "routed weighted sum plus shared output, BF16, original local token order", + "dropless": true, + "residual": false, + "weight_ownership": "contiguous EP partition; rank r owns experts 4*r through 4*r+3" + }, + "correctness": { + "oracle": "independent CPU FP32 calculation from BF16-quantized inputs/weights", + "rtol": 0.03, + "atol": 0.03, + "max_nrmse": 0.01, + "tolerances_frozen": true, + "preserve_inputs_and_parameters": true, + "check_all_ranks": true, + "check_all_outputs": true, + "finite_required": true + }, + "measurement": { + "unit": "ms", + "scope": "complete forward, host scheduling and all communication included", + "rank_aggregation": "maximum elapsed for each paired call", + "case_aggregation": "geometric mean of baseline/candidate per-case median latency ratios", + "warmups": 2, + "samples": 16, + "new_input_each_call": true, + "final_pairs": 4, + "timed_case_ids": [ + "balanced-small", + "skewed-medium", + "balanced-large" + ], + "samples_per_pair": 8, + "exploration_pairs": 2, + "matched_shape_warmup": { + "enabled": true, + "calls_per_implementation_per_timed_seed": 1, + "input": "distinct BF16 values from router-nullspace noise; same selected expert set for every token", + "noise_std_before_projection": 0.3, + "minimum_row_rms_input_change": 0.15, + "minimum_router_boundary_margin": 0.02, + "max_row_resampling_attempts": 32, + "verify_all_warm_outputs": true, + "require_warm_oracle_outside_timed_output_tolerance": true, + "cost": "recorded separately from scored forward latency" + } + }, + "cases": [ + { + "id": "balanced-small", + "rank_tokens": [ + 64, + 64 + ], + "hidden": 512, + "intermediate": 1024, + "top_k": 2, + "routing": "balanced", + "weight_seed": 1729, + "weight_variant": 0, + "measured": true + }, + { + "id": "skewed-medium", + "rank_tokens": [ + 256, + 256 + ], + "hidden": 1024, + "intermediate": 2048, + "top_k": 2, + "routing": "skewed", + "weight_seed": 2718, + "weight_variant": 0, + "measured": true + }, + { + "id": "balanced-large", + "rank_tokens": [ + 1024, + 1024 + ], + "hidden": 2048, + "intermediate": 4096, + "top_k": 2, + "routing": "balanced", + "weight_seed": 3141, + "weight_variant": 0, + "measured": true + }, + { + "id": "unequal-top1", + "rank_tokens": [ + 31, + 7 + ], + "hidden": 128, + "intermediate": 256, + "top_k": 1, + "routing": "balanced", + "weight_seed": 1618, + "weight_variant": 0, + "measured": false + }, + { + "id": "empty-rank-experts", + "rank_tokens": [ + 23, + 17 + ], + "hidden": 128, + "intermediate": 256, + "top_k": 2, + "routing": "empty-rank-experts", + "weight_seed": 1618, + "weight_variant": 0, + "measured": false + }, + { + "id": "zero-input-rank", + "rank_tokens": [ + 0, + 19 + ], + "hidden": 128, + "intermediate": 256, + "top_k": 2, + "routing": "balanced", + "weight_seed": 1618, + "weight_variant": 0, + "measured": false + }, + { + "id": "weight-update", + "rank_tokens": [ + 31, + 7 + ], + "hidden": 128, + "intermediate": 256, + "top_k": 1, + "routing": "balanced", + "weight_seed": 1618, + "weight_variant": 1, + "measured": false + } + ], + "source_provenance": [ + { + "project": "vLLM-Ascend", + "revision": "0f85ed44bbbbdc0f0447f83ce62a82a8ad55afb3", + "url": "https://github.com/vllm-project/vllm-ascend/blob/0f85ed44bbbbdc0f0447f83ce62a82a8ad55afb3/vllm_ascend/ops/fused_moe/prepare_finalize.py", + "role": "AllGather preparation and local expert / combine workload architecture; implementation here is newly written" + }, + { + "project": "vLLM-Ascend", + "revision": "0f85ed44bbbbdc0f0447f83ce62a82a8ad55afb3", + "url": "https://github.com/vllm-project/vllm-ascend/blob/0f85ed44bbbbdc0f0447f83ce62a82a8ad55afb3/vllm_ascend/ops/fused_moe/moe_comm_method.py", + "role": "complete dispatch / expert MLP / combine forward boundary" + } + ], + "tolerance": { + "atol": 0.03, + "rtol": 0.03, + "nrmse": 0.01, + "frozen": true + } +} diff --git a/evidence/moe-agent-comparison-20261104/validator/task/workload.py b/evidence/moe-agent-comparison-20261104/validator/task/workload.py new file mode 100644 index 000000000..b061d42c1 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/validator/task/workload.py @@ -0,0 +1,193 @@ +"""Trusted, CPU-created inputs and deterministic EP-local model parameters. + +No device import/initialization occurs until build_case receives a device. +Candidate implementations may use only the tensors/config passed to compute; +regenerating other ranks' weights or using these generator seeds is prohibited. +""" + +import json +import math +from functools import lru_cache +from pathlib import Path + +import torch + + +TASK_FILE = Path(__file__).with_name("task.json") + + +def load_task(): + return json.loads(TASK_FILE.read_text(encoding="utf-8")) + + +def load_cases(): + return load_task()["cases"] + + +def _generator(seed): + return torch.Generator(device="cpu").manual_seed(int(seed) % (2**63 - 1)) + + +def _weight_key(case): + return ( + int(case["hidden"]), int(case["intermediate"]), + int(case["weight_seed"]), int(case.get("weight_variant", 0)), + str(case["routing"]), + ) + + +@lru_cache(maxsize=12) +def _common_cpu_weights(key): + hidden, intermediate, seed, variant, routing = key + gen = _generator(seed + 1_000_003 * variant) + router_weight = torch.randn(8, hidden, generator=gen) / math.sqrt(hidden) + router_bias = torch.zeros(8, dtype=torch.float32) + if routing == "skewed": + router_bias[0], router_bias[1] = 3.0, 1.5 + elif routing == "empty-rank-experts": + router_bias.fill_(-16.0) + router_bias[0], router_bias[1] = 16.0, 15.0 + elif routing != "balanced": + raise ValueError(f"Unknown routing distribution: {routing}") + shared_gate_up = ( + torch.randn(2 * intermediate, hidden, generator=gen) / math.sqrt(hidden) + ).to(torch.bfloat16) + shared_down = ( + torch.randn(hidden, intermediate, generator=gen) / math.sqrt(intermediate) + ).to(torch.bfloat16) + return router_weight, router_bias, shared_gate_up, shared_down + + +@lru_cache(maxsize=96) +def _expert_cpu_weights(key, expert_id): + hidden, intermediate, seed, variant, _ = key + gen = _generator(seed + 1_000_003 * variant + 10_007 * (int(expert_id) + 1)) + gate_up = ( + torch.randn(2 * intermediate, hidden, generator=gen) / math.sqrt(hidden) + ).to(torch.bfloat16) + down = ( + torch.randn(hidden, intermediate, generator=gen) / math.sqrt(intermediate) + ).to(torch.bfloat16) + return gate_up, down + + +def cpu_parameters(case, expert_ids): + """Fresh parameter containers; caller chooses only its owned expert IDs. + + Oracle may request all eight experts. The candidate never receives this + function, the case record, seeds, or the global expert tensors as arguments. + """ + key = _weight_key(case) + router_weight, router_bias, shared_gate_up, shared_down = _common_cpu_weights(key) + experts = [_expert_cpu_weights(key, expert_id) for expert_id in expert_ids] + return { + "router_weight": router_weight, + "router_bias": router_bias, + "expert_gate_up": torch.stack([weights[0] for weights in experts]), + "expert_down": torch.stack([weights[1] for weights in experts]), + "shared_gate_up": shared_gate_up, + "shared_down": shared_down, + } + + +def cpu_input(case, rank, seed): + """Different rank and call seeds produce different, BF16 input tensors. + + Random test inputs near a router top-k boundary are regenerated outside + measurement. This avoids making an essentially tied routing decision a + numerical-precision contest between CPU and NPU FP32 matmul implementations. + It does not precompute or provide candidate routing decisions. + """ + count = int(case["rank_tokens"][rank]) + hidden = int(case["hidden"]) + gen = _generator(int(seed) + 97_409 * (rank + 1)) + x = (0.7 * torch.randn(count, hidden, generator=gen)).to(torch.bfloat16) + if count == 0: + return x + router, bias, _, _ = _common_cpu_weights(_weight_key(case)) + for _ in range(16): + logits = torch.nn.functional.linear(x.float(), router, bias) + ordered = logits.sort(dim=-1, descending=True).values + k = int(case["top_k"]) + bad = ordered[:, k - 1] - ordered[:, k] < 0.025 + n_bad = int(bad.sum()) + if not n_bad: + return x.contiguous() + x[bad] = (0.7 * torch.randn(n_bad, hidden, generator=gen)).to(torch.bfloat16) + raise RuntimeError("Could not create unambiguous routing inputs") + + +@lru_cache(maxsize=12) +def _router_nullspace_projection(key): + router, _, _, _ = _common_cpu_weights(key) + # Solve rather than explicitly invert W W^T. The eight random router rows + # are full rank; failure is an invalid workload, never a silent fallback. + return torch.linalg.solve(router @ router.T, router) + + +def warm_input(case, rank, seed): + """Different token values with the same per-token selected expert sets. + + The timed input is never reused as its warmup, nor merely permuted. A + substantial independent perturbation in the router's nullspace preserves + expert GEMM row counts while changing the expert/shared-expert computation. + BF16 quantization slightly perturbs router logits, so every row is checked + again after quantization. Rejected rows are resampled; there is no fallback + to the original input or to smaller noise. + """ + original = cpu_input(case, rank, seed) + if original.shape[0] == 0: + return original.clone() + original_fp32 = original.float() + router, bias, _, _ = _common_cpu_weights(_weight_key(case)) + projection = _router_nullspace_projection(_weight_key(case)) + top_k = int(case["top_k"]) + original_logits = torch.nn.functional.linear(original_fp32, router, bias) + original_experts = original_logits.topk(top_k, dim=-1).indices.sort(dim=-1).values + gen = _generator(int(seed) + 30_000_019 + 1_009_117 * (rank + 1)) + pending = torch.arange(original.shape[0]) + warmed = torch.empty_like(original) + for _ in range(32): + noise = 0.3 * torch.randn(pending.numel(), original.shape[1], generator=gen) + noise = noise - (noise @ router.T) @ projection + trial = (original_fp32[pending] + noise).to(torch.bfloat16) + trial_fp32 = trial.float() + logits = torch.nn.functional.linear(trial_fp32, router, bias) + selected = logits.topk(top_k, dim=-1).indices.sort(dim=-1).values + ordered = logits.sort(dim=-1, descending=True).values + margin = ordered[:, top_k - 1] - ordered[:, top_k] + row_rms_change = (trial_fp32 - original_fp32[pending]).square().mean(dim=-1).sqrt() + accepted = ( + (selected == original_experts[pending]).all(dim=-1) + & (margin >= 0.02) + & (row_rms_change >= 0.15) + & torch.isfinite(trial_fp32).all(dim=-1) + ) + warmed[pending[accepted]] = trial[accepted] + pending = pending[~accepted] + if pending.numel() == 0: + return warmed.contiguous() + raise RuntimeError("Could not create distinct routing-shape-matched warm input") + + +def build_case(case, rank, device, seed): + """Return local input, EP-local parameters and public structural config.""" + if rank not in (0, 1): + raise ValueError("This task requires exactly two ranks") + cpu_params = cpu_parameters(case, range(rank * 4, rank * 4 + 4)) + params = {name: tensor.to(device=device, copy=True) for name, tensor in cpu_params.items()} + config = { + "world_size": 2, + "rank": rank, + "num_experts": 8, + "experts_per_rank": 4, + "hidden": int(case["hidden"]), + "intermediate": int(case["intermediate"]), + "top_k": int(case["top_k"]), + "rank_tokens": list(case["rank_tokens"]), + } + return { + "x": cpu_input(case, rank, seed).to(device=device, copy=True), + "params": params, + "config": config, + } diff --git a/evidence/moe-agent-comparison-20261104/verify.py b/evidence/moe-agent-comparison-20261104/verify.py new file mode 100644 index 000000000..fe65775b8 --- /dev/null +++ b/evidence/moe-agent-comparison-20261104/verify.py @@ -0,0 +1,260 @@ +"""CPU-only integrity/statistics verification; does not execute candidates.""" + +import argparse +import csv +import gzip +import hashlib +import json +import math +import re +import statistics +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent + + +def read(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def close(actual, expected): + assert math.isclose(actual, expected, rel_tol=1e-12, abs_tol=1e-12), (actual, expected) + + +def csv_rows(path): + with path.open(encoding="utf-8", newline="") as stream: + return list(csv.DictReader(stream)) + + +def scan_text(name, data): + text = data.decode("utf-8") + patterns = { + "personal absolute path": r"(?i)(?:(? 0 for v in values + ) + samples += len(values) + close(statistics.median(values), case[f"{label}_median_ms"]) + ratio = case["baseline_median_ms"] / case["candidate_median_ms"] + close(ratio, case["speedup"]) + case_speedups.append(ratio) + close(statistics.geometric_mean(case_speedups), o["speedup"]) + assert all(r["local_failed_calls"] == 0 for r in audit["ranks"]) + assert all( + r["calls"] == (424 if o["role"] == "final" else 232) for r in audit["ranks"] + ) + if o["role"] == "exploratory": + best = max(best, o["speedup"]) + else: + final_count += 1 + else: + failed += 1 + assert key.startswith("gpt6astra-") and o["round_index"] == 4 + assert o["correctness"] == "FAIL" and o["performance_verdict"] == "INCONCLUSIVE" + assert o["speedup"] is None and row["cases"] is None and o["failure_reason"] + close(o["best_exploration_speedup"], best) + selected = obs[session["selected_round"] - 1] + assert ( + selected["candidate_sha256"] + == obs[-1]["candidate_sha256"] + == session["final_candidate_sha256"] + ) + close(session["final_speedup"], obs[-1]["speedup"]) + u = session["usage"] + assert ( + u["total_tokens"] + == obs[-1]["cumulative_total_tokens"] + session["post_measurement_reporting_tokens"] + ) + assert u["total_tokens"] == u["input_tokens"] + u["output_tokens"] + assert u["input_tokens"] == u["cached_input_tokens"] + u["uncached_input_tokens"] + assert u["reasoning_output_tokens"] <= u["output_tokens"] + assert int(usage["total_tokens"]) == u["total_tokens"] + assert int(usage["exploration_tokens"]) == obs[7]["cumulative_total_tokens"] + assert int(usage["final_stage_tokens"]) == obs[8]["round_total_tokens"] + assert int(usage["report_tail_tokens"]) == session["post_measurement_reporting_tokens"] + close(sum(o["queue_seconds"] for o in obs), session["queue_seconds"]) + close( + sum(o["allocation_process_seconds"] for o in obs), session["allocation_process_seconds"] + ) + assert (valid, failed, final_count, samples) == (52, 2, 6, 5568) + assert sum(s["usage"]["total_tokens"] for s in summary["sessions"]) == 79562339 + for csv_row, row in zip(csv_rounds, measurements): + for field, value in row.items(): + if field not in csv_row: + continue + assert csv_row[field] == ("" if value is None else str(value)), field + plan = read(root / "plan/publication.json") + for prefix, files in ( + ("task", (root / "validator/task").iterdir()), + ("runner", (root / "validator/runner").iterdir()), + ): + for path in files: + assert sha(path.read_bytes()) == plan["source_hashes"][f"{prefix}/{path.name}"] + if source is not None: + original = read(source / "results/summary.json") + for public_session, original_session in zip(summary["sessions"], original["sessions"]): + assert public_session["key"] == original_session["key"] + shared_fields = public_session.keys() & original_session.keys() + for field in shared_fields - {"observations", "usage"}: + assert public_session[field] == original_session[field], field + for field, value in public_session["usage"].items(): + assert value == original_session["usage"][field], field + assert public_session["reasoning_effort"] == original_session["usage"]["efforts"] + assert public_session["queue_seconds"] == original_session["recorded_queue_seconds"] + assert public_session["allocation_process_seconds"] == original_session[ + "recorded_device_active_seconds" + ] + for public_row, original_row in zip( + public_session["observations"], original_session["observations"] + ): + for field in public_row.keys() & original_row.keys(): + assert public_row[field] == original_row[field], field + m = next( + r + for r in measurements + if r["session_key"] == public_session["key"] + and r["role"] == public_row["role"] + and r["round_index"] == public_row["round_index"] + ) + assert m["cases"] == original_row["cases"] + p = Path(original_row["observation_path"]) + assert sha(p.read_bytes()) == m["original_observation_sha256"] + source_key = ( + public_session["key"], + m["role"], + "" if m["round_index"] is None else str(m["round_index"]), + ) + assert source_bytes[source_key] == (p.parent / "implementation.py").read_bytes() + for name, entry in read(root / "plan/provenance.json")["original_artifacts"].items(): + assert sha((source / name).read_bytes()) == entry["sha256"], name + print( + json.dumps( + { + "status": "PASS", + "attempts": 54, + "exploration_pass": 46, + "exploration_fail": 2, + "final_pass": final_count, + "timing_samples": samples, + "candidate_sources": 54, + "source_archive_compared": source is not None, + "scope": "public extraction integrity/statistics; no device rerun", + } + ) + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--source", type=Path, help="Optional authorized original archive for byte comparison" + ) + args = parser.parse_args() + verify(ROOT, args.source) diff --git a/pyproject.toml b/pyproject.toml index e44a7a806..8573ac769 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ where = ["src"] namespaces = false [tool.setuptools.package-data] -inferrsi = ["data/**/*.json", "data/**/*.md", "data/**/LICENSE", "data/**/THIRD_PARTY_NOTICES", "data/**/scripts/*.py", "data/**/references/licenses/*.txt", "data/**/references/upstream-NOTICE"] +inferrsi = ["data/**/*.json", "data/**/*.md", "data/**/LICENSE", "data/**/THIRD_PARTY_NOTICES", "data/**/scripts/*.py", "data/engineering_benchmarks/**/*.py", "data/engineering_scenarios/**/*.py", "data/**/references/licenses/*.txt", "data/**/references/upstream-NOTICE"] [tool.setuptools_scm] version_file = "src/inferrsi/_version.py" @@ -35,8 +35,9 @@ prerelease = true [project.optional-dependencies] skills = ["PyYAML>=6.0"] -dev = ["pytest>=7.4", "ruff>=0.6", "PyYAML>=6.0"] +dev = ["pytest>=7.4", "ruff>=0.6", "PyYAML>=6.0", "numpy>=1.24"] image = ["numpy>=1.24", "Pillow>=10", "scikit-image>=0.21"] +benchmarks = ["numpy>=1.24"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py index 46b5e9336..6d7aece10 100644 --- a/src/inferrsi/arguments.py +++ b/src/inferrsi/arguments.py @@ -39,6 +39,9 @@ def parser() -> argparse.ArgumentParser: ) root.add_argument("--version", action="version", version=__version__) commands = root.add_subparsers(dest="command", required=True) + from .harness.benchmarks.cli import add_parser + + add_parser(commands) doctor = commands.add_parser( "doctor", help="Inspect a prepared local target without installing dependencies" ) diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py index 349e3df34..87c369c90 100644 --- a/src/inferrsi/cli.py +++ b/src/inferrsi/cli.py @@ -116,6 +116,20 @@ def main(argv: list[str] | None = None) -> int: argument_parser = parser() args = argument_parser.parse_args(argv) try: + if args.command == "benchmarks": + from .harness.benchmarks.cli import dispatch + + result = dispatch(args) + _print(result) + if args.benchmark_action == "compare": + verdict = result["status"].get("verdict") + return 0 if verdict == "PASS" else (1 if verdict == "FAIL" else 2) + if isinstance(result, dict): + if "verdict" in result: + return 0 if result["verdict"] == "PASS" else 1 + if "status" in result: + return 0 if result["status"] == "completed" else 1 + return 0 if args.command == "skills": # Optional dependencies load only for the skills command; the core # CLI stays independent of them. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/LICENSE b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/LICENSE new file mode 100644 index 000000000..f74af3c84 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/LICENSE @@ -0,0 +1,41 @@ +rom PyTorch: + +Copyright (c) 2020, Huawei Technologies Co., Ltd +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America + and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md new file mode 100644 index 000000000..6fb0d2a68 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md @@ -0,0 +1,50 @@ +# Affine RMSNorm on Ascend NPU + +Implement `compute(x, weight, *, eps) -> y` for contiguous float32 NPU tensors +`x[rows, features]` and `weight[features]`: +`y = x / sqrt(mean(x*x, axis=-1, keepdims=True) + eps) * weight`. +Preserve input and weight bytes, shape, dtype and device. The public, seeded +suite includes nonuniform signed weights, ordinary, zero, large and tiny inputs. +Only the decode/prefill shapes are timed. The independent CPU oracle uses scalar +double arithmetic and `math.fsum`; CPU reference computation is never a timed +candidate or a device fallback. + +The baseline composes eager PyTorch NPU operations. The example candidate uses +the existing fused `torch_npu.npu_rms_norm` implementation; it is a reference +control, not a newly written kernel or evidence of Agent efficacy. + +## Execution boundary + +Run reviewed source only, inside a caller-owned single-device allocation. The +worker requires exactly one `ASCEND_RT_VISIBLE_DEVICES` entry and uses logical +device 0 without changing the scheduler's mapping. Discovery needs no PyTorch +import; execution requires an installed compatible torch/torch_npu/CANN runtime. +The existing formal SkillEval CPU sandbox explicitly rejects this adapter. + +Input/weight generation, transfer, oracle preparation and output validation are +outside timing. Each timed call has distinct retained input objects and bytes. +The clock surrounds the eager call and its completion synchronization, following +an untimed pre-synchronization. It includes Python dispatch and synchronization +cost, so **synchronized eager-call latency is not pure device kernel time**. +Output copies and checks happen after the clock stops. Every measured output is +checked; incorrect output has no usable performance metric. + +`profile` uses the Ascend PyTorch CPU/NPU profiler after warmup, separately from +measurement. Its raw local trace includes verification copies and must be +reviewed before publication; it is diagnostic, not a speed score. An empty +device-kernel trace is an error rather than successful profiling. + +## Sources and limits + +API/shape semantics follow the [official op-plugin API](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/docs/en/custom_APIs/torch_npu/%28beta%29torch_npu-npu_rms_norm.md), +[implementation](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/op_plugin/ops/opapi/RmsNormKernelOpApi.cpp) +and [tests](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/test/test_custom_ops/test_npu_rms_norm.py). +The source BSD-3-Clause license is retained in `LICENSE`. Workloads, scalar +oracle and adapters here are independently authored; upstream performance +numbers are not imported. + +First qualification is limited to the recorded device and installed runtime, +float32, contiguous 2D forward execution and this public suite. It establishes +neither BF16/FP16, gradients, graph replay, hostile-producer isolation, arbitrary +NPU qualification nor service/skill/search benefit. A/A and reference controls +and issue-scoped predeclared gates are needed before choosing by speed. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/baseline.py b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/baseline.py new file mode 100644 index 000000000..af36f4051 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/baseline.py @@ -0,0 +1,7 @@ +"""Eager affine RMSNorm baseline; all arithmetic executes on the input NPU.""" + +import torch + + +def compute(x, weight, *, eps): + return x * torch.rsqrt(torch.mean(x * x, dim=-1, keepdim=True) + eps) * weight diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/candidate.py b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/candidate.py new file mode 100644 index 000000000..7b492f940 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/candidate.py @@ -0,0 +1,7 @@ +"""Authored reference optimization, not an Agent/skill efficacy result.""" + +import torch_npu + + +def compute(x, weight, *, eps): + return torch_npu.npu_rms_norm(x, weight, epsilon=eps)[0] diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/task.json b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/task.json new file mode 100644 index 000000000..3ea4889d5 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/task.json @@ -0,0 +1,103 @@ +{ + "schema_version": 1, + "id": "rmsnorm-npu", + "adapter": "torch-npu-rmsnorm-v1", + "description": "Optimize affine RMSNorm on contiguous 2D float32 NPU tensors; preserve input and nonuniform weight.", + "searchspace": "docs/searchspace-matrix.md — Operator / Diagnose, Optimize, Verify; explicit Ascend workload qualification", + "source": { + "url": "https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/test/test_custom_ops/test_npu_rms_norm.py", + "revision": "9f23f00b1ebe54cca612b0537bc582dba409a0af", + "license": "BSD-3-Clause", + "adaptation": "Official affine RMSNorm semantics; independently authored smaller public workloads, scalar double oracle and trusted Workbench adapter. Baseline composes eager NPU operations, reference candidate calls existing fused operator. No upstream scores imported." + }, + "environment": { + "backend": "ascend", + "runtime": "torch_npu", + "dtype": "float32" + }, + "candidate": { + "entrypoint": "compute", + "editable": [ + "implementation.py" + ] + }, + "oracle": { + "operation": "rmsnorm", + "axis": 1, + "eps": 1e-05, + "rtol": 1e-05, + "atol": 1e-06 + }, + "measurement": { + "key": "metrics.suite_latency_ms", + "unit": "ms", + "boundary": "device-synchronized-eager-call", + "aggregation": "sum-of-per-case-medians", + "warmups": 10, + "samples": 25, + "input_sequence": "seeded-distinct-per-call-v1", + "case_ids": [ + "decode", + "prefill" + ] + }, + "workloads": [ + { + "id": "decode", + "shape": [ + 1, + 4096 + ], + "seed": 252, + "distribution": "normal", + "layout": "contiguous" + }, + { + "id": "prefill", + "shape": [ + 64, + 4096 + ], + "seed": 253, + "distribution": "normal", + "layout": "contiguous" + }, + { + "id": "zeros", + "shape": [ + 2, + 4096 + ], + "seed": 254, + "distribution": "zeros", + "layout": "contiguous" + }, + { + "id": "large", + "shape": [ + 4, + 4096 + ], + "seed": 255, + "distribution": "large", + "layout": "contiguous" + }, + { + "id": "tiny", + "shape": [ + 4, + 4096 + ], + "seed": 256, + "distribution": "tiny", + "layout": "contiguous" + } + ], + "coverage": [ + "Public development workloads; not holdout or Agent/skill/search efficacy", + "Single allocated Ascend NPU, contiguous float32 2D forward only; device/runtime qualification recorded separately", + "No BF16/FP16, gradients, graph replay, strided inputs, communication, capacity or service/E2E claim", + "Trusted reviewed source only; formal SkillEval NPU isolation not implemented", + "Distinct retained per-call inputs and weights; CPU oracle and transfers excluded from synchronized eager-call latency; profiling separate" + ] +} diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/LICENSE b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/LICENSE new file mode 100644 index 000000000..eaae6b11e --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2023 Anne Ouyang, Simon Guo, Azalia Mirhoseini +Scaling Intelligence Lab, Stanford University + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md new file mode 100644 index 000000000..91cef765a --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md @@ -0,0 +1,31 @@ +# softmax-cpu + +CPU adaptation of [KernelBench 23_Softmax.py](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py). +Source license is retained in LICENSE. No upstream scoring result transfers here. + +Implement `compute(x, *, eps)` in a self-contained `implementation.py`. Return a +float32 NumPy array of the same shape. Preserve input values. Compute stable softmax along axis 1. +Use only NumPy, math and locally defined helpers. Source is reviewed/trusted; +the evaluator is not a security sandbox for model-produced arbitrary code. + +The task JSON owns six base correctness workloads, seeds, layouts, numerical +tolerances and the v2 timing policy. Five varying workloads have a frozen +per-call input sequence; invariant zero inputs are correctness-only. Baseline and example candidate are separate from the scalar +double-precision oracle. The oracle uses math.fsum and is not editable by a +candidate. The task package and grader are content-frozen per comparison. + +Each measured call actually runs compute and checks its output and input +preservation outside the timed region. Input generation, reference computation, +warmups, validation, process startup and profiling are excluded. Output allocation +inside compute is included. Samples use perf_counter_ns and report milliseconds. +Sum of per-case medians is only this task's fixed-suite objective, not a universal +score. Retain every per-case sample and input hash to inspect regressions. Inputs +are generated before timing and kept alive; each precheck, warmup and sampled +input has distinct bytes. Baseline and candidate consume the same sequence. +This prevents ordinary identity/content memo hits, not hostile-code execution. +Small CPU timings still need repeatability qualification; no cold-start serving +or device synchronization is claimed. + +All tests are public. This small subset cannot demonstrate production inference +speedup, memory savings, or GPU/NPU support. A fresh task version and device +qualification are needed for changed shapes, dtype, backend or timing. diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/baseline.py b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/baseline.py new file mode 100644 index 000000000..c6c630b0f --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/baseline.py @@ -0,0 +1,8 @@ +"""NumPy CPU adaptation of the attributed KernelBench task; see LICENSE.""" + +import numpy as np + + +def compute(x, *, eps): + values = np.exp(x - np.max(x, axis=1, keepdims=True)) + return values / np.sum(values, axis=1, keepdims=True) diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/candidate.py b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/candidate.py new file mode 100644 index 000000000..f19af8c23 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/candidate.py @@ -0,0 +1,10 @@ +"""Reviewed example using temporary buffers; not an agent-generated result.""" + +import numpy as np + + +def compute(x, *, eps): + output = np.subtract(x, np.max(x, axis=1, keepdims=True)) + np.exp(output, out=output) + output /= np.sum(output, axis=1, keepdims=True) + return output diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json new file mode 100644 index 000000000..e25252d59 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json @@ -0,0 +1,117 @@ +{ + "schema_version": 1, + "id": "softmax-cpu", + "adapter": "numpy-axis1-v2", + "description": "Optimize softmax on finite float32 CPU arrays while preserving axis-1 semantics and input bytes.", + "searchspace": "docs/searchspace-matrix.md \u2014 Operator / Diagnose, Optimize, Verify; CPU adaptation, no matrix maturity promotion", + "source": { + "url": "https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py", + "revision": "423217d9fda91e0c2d67e4a43bf62f96f6d104f1", + "license": "MIT", + "adaptation": "Adapted mathematical task, not upstream scores. PyTorch/CUDA replaced by NumPy CPU; reduced shapes, public stress distributions and strided inputs added. Axis 1 retained. Baseline is a NumPy implementation; scalar math.fsum supplies independent reference." + }, + "environment": { + "backend": "cpu", + "runtime": "numpy", + "dtype": "float32" + }, + "candidate": { + "entrypoint": "compute", + "editable": [ + "implementation.py" + ] + }, + "oracle": { + "operation": "softmax", + "axis": 1, + "eps": 1e-05, + "rtol": 1e-05, + "atol": 1e-07 + }, + "measurement": { + "key": "metrics.suite_latency_ms", + "unit": "ms", + "boundary": "synchronous-compute-call", + "aggregation": "sum-of-per-case-medians", + "warmups": 2, + "samples": 5, + "input_sequence": "seeded-distinct-per-call-v1", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ] + }, + "workloads": [ + { + "id": "normal", + "shape": [ + 16, + 257 + ], + "seed": 252, + "distribution": "normal", + "layout": "contiguous" + }, + { + "id": "strided", + "shape": [ + 16, + 257 + ], + "seed": 253, + "distribution": "normal", + "layout": "strided" + }, + { + "id": "zeros", + "shape": [ + 3, + 7 + ], + "seed": 254, + "distribution": "zeros", + "layout": "contiguous" + }, + { + "id": "large", + "shape": [ + 3, + 31 + ], + "seed": 255, + "distribution": "large", + "layout": "contiguous" + }, + { + "id": "tiny", + "shape": [ + 2, + 17 + ], + "seed": 256, + "distribution": "tiny", + "layout": "contiguous" + }, + { + "id": "singleton", + "shape": [ + 2, + 1 + ], + "seed": 257, + "distribution": "normal", + "layout": "contiguous" + } + ], + "coverage": [ + "Public development workloads, not held-out generalization", + "CPU float32 finite inputs only; no float16/bfloat16, NaN/Inf inputs or gradients", + "No CUDA/ROCm/Ascend execution or accelerator timing qualification", + "No service/E2E, communication, memory-capacity or skill/search benefit claim", + "Trusted self-contained reviewed source only; no hostile-agent isolation", + "Distinct frozen per-call inputs; zeros correctness-only; allocations inside compute included, setup/oracle/profile excluded from timing" + ] +} diff --git a/src/inferrsi/data/engineering_scenarios/moe-forward-npu/LICENSE b/src/inferrsi/data/engineering_scenarios/moe-forward-npu/LICENSE new file mode 100644 index 000000000..b09cd7856 --- /dev/null +++ b/src/inferrsi/data/engineering_scenarios/moe-forward-npu/LICENSE @@ -0,0 +1,201 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/inferrsi/data/engineering_scenarios/moe-forward-npu/TASK.md b/src/inferrsi/data/engineering_scenarios/moe-forward-npu/TASK.md new file mode 100644 index 000000000..ba1922b61 --- /dev/null +++ b/src/inferrsi/data/engineering_scenarios/moe-forward-npu/TASK.md @@ -0,0 +1,47 @@ +# 双 NPU：带共享专家的 MoE FFN 完整 forward + +目标是在保持结果正确的前提下，缩短两张 Ascend NPU 上完整 MoE FFN forward 的完成时间。输入是每张卡自己的 token，输出仍保留在输入所属卡。这是一层模型计算，不涉及模型下载、训练或 serving。 + +## 输入与数学语义 + +实现 `compute(x, params, *, config, group)`，返回与 `x` 相同形状、dtype、设备的输出张量。`group` 是已建立的两 rank HCCL process group；两张空闲设备由调用方先通过 `npu run --gpus 2` 分配，候选不创建或销毁 process group、不修改设备可见性。 + +- `x`：本 rank 的 `[N_rank, H]` BF16 输入。两个 rank 的内容不同，token 数可能不同，也可能有一个 rank 没有输入。 +- `config`：`world_size=2`、`rank`、`num_experts=8`、`experts_per_rank=4`、`hidden`、`intermediate`、`top_k`、`rank_tokens`。只包含结构信息。 +- `params.router_weight`：FP32 `[8,H]`；`router_bias`：FP32 `[8]`。 +- `params.expert_gate_up`：BF16 `[4,2I,H]`；`expert_down`：BF16 `[4,H,I]`。rank 0 只有 experts 0–3，rank 1 只有 experts 4–7。 +- `params.shared_gate_up`：BF16 `[2I,H]`；`shared_down`：BF16 `[H,I]`。共享专家和 router 参数在两卡相同。 + +对每个 token，以 FP32 计算 `logits = x @ router_weight.T + router_bias`，选择最大的 `top_k` 项，再对选中的 logits 做 softmax 得到权重。每个专家执行 `W_down(silu(W_gate x) * (W_up x))`，其中 `W_gate`、`W_up` 是 `gate_up` 的前后两半。结果是选中专家输出的加权和，再加共享专家的输出。最终输出 BF16，保持本 rank 原始 token 顺序。 + +不丢 token；没有 dropout、capacity 截断、量化、residual 或额外 norm。生成输入排除了 top-k 边界近似相等的情况，避免以浮点舍入决定不同路由；路由仍必须根据本次输入计算。 + +## 候选边界 + +可以更改本实现中的计算、内存布局和通信组织，可以调用现有 PyTorch/torch_npu/HCCL 或自定义 NPU kernel。使用的工具必须来自当前已配置环境，不能更改共享软件安装。 + +输入和所有参数只读。不得使用评测 seed、输入生成器、oracle、其他 session 的代码或结果；不得根据评测阶段、文件名或计时器选择不同语义；不得省略任何 rank 的工作。不能获取或重建另一 rank 的专家权重并在未计时区内复制，以改变 EP 所有权。不能缓存输入相关的路由、专家计算或最终输出。 + +参数会变化，张量地址也可能复用。若实现中缓存权重排布或编译结果，必须正确失效；首次建立过程在首次调用中记录，其额外内存也要报告。输入相关工作全部在 `compute` 内。候选不允许外部后台计算、写结果文件、修改评测器或创建脱离评测器生命周期的进程。不得 monkeypatch PyTorch、通信库、oracle 或其他全局运行时状态以影响基线和评测器；本执行器接收经审查的可信源码，不是恶意代码隔离沙箱。 + +## 验收与计分 + +评测器使用独立 CPU FP32 数学实现，检查两个 rank 的全部输出、有限性、输入/参数保持和误差。用不同输入、权重变化、top-k=1/2、不等 token 数、空专家以及零输入 rank 检查正确性。数值容差以冻结后的 `task.json` 为准。 + +计时覆盖 host 调度、数据重排、路由、通信、专家计算、共享专家和输出完成。每次以最慢 rank 的完成时间计分；只提交通信或 kernel 而未完成输出不能获得较短时间。baseline/candidate 在同一组设备交错测量，每次计时输入都变化。各性能 case 的加速比用几何平均汇总，同时保留逐 case 数据；编译、首次调用、搜索墙钟和 token 消耗另报。 + +性能集合包含三种规模与路由分布，均为 top-k=2： + +| Case | 每 rank token 数 | Hidden H | 专家 intermediate I | 路由 | +| --- | --- | --- | --- | --- | +| balanced-small | 64 | 512 | 1024 | 均衡 | +| skewed-medium | 256 | 1024 | 2048 | 偏斜 | +| balanced-large | 1024 | 2048 | 4096 | 均衡 | + +每 case 的所有输入和 FP32 oracle 都在开始评测前共同生成，两种实现获得独立但相同的输入张量；避免只有先运行的一方在计时前经历昂贵的 CPU 参考计算。每次计时使用不同输入。入口要求调用方显式指定配对数量 `--pairs`，每组 8 个样本、AB/BA 交替；历史实验探索使用两组（每实现每 case 16 样本），最终复测使用四组（32 样本）。这些是该次实验协议，不是自动搜索轮数或通用默认值。额外的小形状边界用例仅检查正确性，不计入速度分数。准备时间和首次调用时间另存；同进程首次调用受共享底层算子缓存影响，不能解释为两个实现各自在全新进程中的冷启动对比。 + +每个计时输入还配有一个**数值不同、专家数量相同的预热输入**。评测器在 router 权重的零空间加入标准差约 0.3 的噪声，量化为 BF16 后逐 token 检查选中专家集合不变、top-k 边界间隔至少 0.02、输入 RMS 差至少 0.15；不满足时重采样，不退化成原输入、token 排列或微小扰动。这样两种实现可以预热同样的动态专家矩阵形状，避免先执行的一方单独承担新形状的底层算子准备开销。每个预热输入仍需独立数学验收、只读检查和输出清零；预热与正式 oracle 的输出必须相差超过正式容差，防止直接复用预热输出。预热成本单列，不计入 forward 速度分数；输入相关计算和输出缓存仍然禁止。 + +本题的 baseline 是可读、完整的 AllGather/本地专家/ReduceScatter 实现，已经合并 gate/up 投影，并使用 NPU SwiGLU。它是正常算法基线，不代表 vLLM-Ascend 最新生产实现的性能；对它的加速比不得解释为对 vLLM 服务或整模型的同等加速。 + +任务结构参考 vLLM-Ascend 的 [AllGather preparation](https://github.com/vllm-project/vllm-ascend/blob/0f85ed44bbbbdc0f0447f83ce62a82a8ad55afb3/vllm_ascend/ops/fused_moe/prepare_finalize.py) 和 [MoE forward](https://github.com/vllm-project/vllm-ascend/blob/0f85ed44bbbbdc0f0447f83ce62a82a8ad55afb3/vllm_ascend/ops/fused_moe/moe_comm_method.py)。本任务的代码是独立编写的最小实现，不依赖整个 vLLM 服务栈。 diff --git a/src/inferrsi/data/engineering_scenarios/moe-forward-npu/baseline.py b/src/inferrsi/data/engineering_scenarios/moe-forward-npu/baseline.py new file mode 100644 index 000000000..f7e54be2b --- /dev/null +++ b/src/inferrsi/data/engineering_scenarios/moe-forward-npu/baseline.py @@ -0,0 +1,67 @@ +"""EP=2 MoE + shared expert: AllGather/local experts/ReduceScatter baseline. + +This is a self-contained, newly written implementation of the common +AllGather-based expert-parallel MoE workload architecture, not a copy of the +vLLM serving stack. It uses a fused gate/up projection and NPU SwiGLU. +""" + +import torch +import torch.distributed as dist +import torch.nn.functional as F + + +def _mlp(x, gate_up, down): + if x.shape[0] == 0: + return x.new_empty((0, down.shape[0])) + projected = F.linear(x, gate_up) + if x.device.type == "npu": + import torch_npu + activated = torch_npu.npu_swiglu(projected, dim=-1) + else: + gate, up = projected.chunk(2, dim=-1) + activated = F.silu(gate) * up + return F.linear(activated, down) + + +def compute(x, params, *, config, group): + world_size, rank = dist.get_world_size(group), dist.get_rank(group) + if world_size != 2 or rank != config["rank"]: + raise ValueError("Expected the configured two-rank process group") + max_tokens = max(config["rank_tokens"]) + hidden = config["hidden"] + logits = ( + F.linear(x.float(), params["router_weight"], params["router_bias"]) + if x.shape[0] else torch.empty((0, 8), dtype=torch.float32, device=x.device) + ) + padded_x = F.pad(x, (0, 0, 0, max_tokens - x.shape[0])) + padded_logits = F.pad(logits, (0, 0, 0, max_tokens - x.shape[0])) + all_x = torch.empty((2 * max_tokens, hidden), dtype=x.dtype, device=x.device) + all_logits = torch.empty((2 * max_tokens, 8), dtype=torch.float32, device=x.device) + dist.all_gather_into_tensor(all_x, padded_x.contiguous(), group=group) + dist.all_gather_into_tensor(all_logits, padded_logits.contiguous(), group=group) + selected_logits, selected_experts = all_logits.topk(config["top_k"], dim=-1, sorted=True) + probabilities = selected_logits.softmax(dim=-1) + # Padded slots have zero weight and never perform an expert MLP. + for source_rank, count in enumerate(config["rank_tokens"]): + if count < max_tokens: + probabilities[source_rank * max_tokens + count:(source_rank + 1) * max_tokens] = 0 + local_contribution = torch.zeros((2 * max_tokens, hidden), dtype=torch.float32, device=x.device) + for local_expert in range(4): + expert_id = rank * 4 + local_expert + positions = torch.nonzero( + (selected_experts == expert_id) & (probabilities > 0), as_tuple=False + ) + if positions.shape[0] == 0: + continue + token_ids, slot_ids = positions[:, 0], positions[:, 1] + values = _mlp( + all_x.index_select(0, token_ids), + params["expert_gate_up"][local_expert], + params["expert_down"][local_expert], + ) + weighted = values.float() * probabilities[token_ids, slot_ids].unsqueeze(-1) + local_contribution.index_add_(0, token_ids, weighted) + local_routed = torch.empty((max_tokens, hidden), dtype=torch.float32, device=x.device) + dist.reduce_scatter_tensor(local_routed, local_contribution, group=group) + shared = _mlp(x, params["shared_gate_up"], params["shared_down"]) + return (local_routed[:x.shape[0]] + shared.float()).to(dtype=x.dtype) diff --git a/src/inferrsi/data/engineering_scenarios/moe-forward-npu/oracle.py b/src/inferrsi/data/engineering_scenarios/moe-forward-npu/oracle.py new file mode 100644 index 000000000..34035a836 --- /dev/null +++ b/src/inferrsi/data/engineering_scenarios/moe-forward-npu/oracle.py @@ -0,0 +1,89 @@ +"""Independent, unsharded CPU FP32 oracle for the complete mathematical MoE. + +No baseline/candidate, distributed collective, NPU fused operator, permutation +or combine implementation is reused. Input and weight generators are shared +only to define the problem data. +""" + +from functools import lru_cache + +import torch +from workload import _weight_key, cpu_input, cpu_parameters + + +@lru_cache(maxsize=12) +def _float_parameters(key): + hidden, intermediate, weight_seed, variant, routing = key + case = { + "hidden": hidden, "intermediate": intermediate, + "weight_seed": weight_seed, "weight_variant": variant, + "routing": routing, + } + return {name: value.float() for name, value in cpu_parameters(case, range(8)).items()} + + +def _expert(x, w_gate_up, w_down): + intermediate = w_down.shape[1] + # Separate matmuls deliberately differ from the implementation's fused + # projection, while representing exactly the same mathematical function. + gate = x @ w_gate_up[:intermediate].T + up = x @ w_gate_up[intermediate:].T + activated = (gate / (1.0 + torch.exp(-gate))) * up + return activated @ w_down.T + + +@torch.inference_mode() +def expected(case, seed, rank): + return expected_from_input(case, cpu_input(case, rank, seed)) + + +@torch.inference_mode() +def expected_from_input(case, x_cpu): + """Evaluate supplied problem data without using any candidate routing.""" + if x_cpu.device.type != "cpu" or x_cpu.dtype != torch.bfloat16: + raise ValueError("Oracle input must be CPU BF16 problem data") + if x_cpu.ndim != 2 or x_cpu.shape[1] != int(case["hidden"]): + raise ValueError("Oracle input shape does not match the case") + x = x_cpu.float() + if x.shape[0] == 0: + return x + params = _float_parameters(_weight_key(case)) + logits = x @ params["router_weight"].T + params["router_bias"] + # Sorting is an independent selection implementation. Generated input + # margins exclude top-k boundary ties. + order = torch.argsort(logits, dim=1, descending=True) + chosen = order[:, :int(case["top_k"])] + chosen_logits = logits.gather(1, chosen) + exponents = torch.exp(chosen_logits - chosen_logits.max(dim=1, keepdim=True).values) + probabilities = exponents / exponents.sum(dim=1, keepdim=True) + result = _expert(x, params["shared_gate_up"], params["shared_down"]) + for expert_id in range(8): + # Weighted output is accumulated directly at the original token row; + # no distributed packing/permutation logic is involved. + rows, slots = torch.where(chosen == expert_id) + if rows.numel() == 0: + continue + expert_values = _expert( + x[rows], params["expert_gate_up"][expert_id], params["expert_down"][expert_id] + ) + result[rows] += expert_values * probabilities[rows, slots, None] + return result.contiguous() + + +def metrics(actual, expected_output): + """Useful calibration diagnostics; verdict thresholds belong to task.json.""" + values = actual.detach().cpu().float() + reference = expected_output.detach().cpu().float() + if values.shape != reference.shape: + return {"shape_match": False, "finite": False} + if values.numel() == 0: + return {"shape_match": True, "finite": True, "max_abs": 0.0, "nrmse": 0.0} + error = values - reference + reference_rms = torch.sqrt(torch.mean(reference.square())).item() + return { + "shape_match": True, + "finite": bool(torch.isfinite(values).all()), + "max_abs": float(error.abs().max()), + "nrmse": float(torch.sqrt(torch.mean(error.square()))) / max(reference_rms, 1e-12), + "reference_rms": reference_rms, + } diff --git a/src/inferrsi/data/engineering_scenarios/moe-forward-npu/task.json b/src/inferrsi/data/engineering_scenarios/moe-forward-npu/task.json new file mode 100644 index 000000000..8855f33b7 --- /dev/null +++ b/src/inferrsi/data/engineering_scenarios/moe-forward-npu/task.json @@ -0,0 +1,181 @@ +{ + "schema_version": 1, + "id": "moe-shared-ep2-npu-v1", + "title": "Expert-parallel MoE FFN with a shared expert", + "status": "frozen", + "backend": "ascend", + "world_size": 2, + "dtype": "bfloat16", + "entrypoint": "compute(x, params, *, config, group)", + "num_experts": 8, + "experts_per_rank": 4, + "semantics": { + "router": "FP32 linear with bias; sorted top-k logits; softmax over selected logits", + "expert": "linear_gate_up -> silu(gate) * up -> linear_down", + "shared": "one replicated SwiGLU expert evaluated on this rank's input", + "output": "routed weighted sum plus shared output, BF16, original local token order", + "dropless": true, + "residual": false, + "weight_ownership": "contiguous EP partition; rank r owns experts 4*r through 4*r+3" + }, + "correctness": { + "oracle": "independent CPU FP32 calculation from BF16-quantized inputs/weights", + "rtol": 0.03, + "atol": 0.03, + "max_nrmse": 0.01, + "tolerances_frozen": true, + "preserve_inputs_and_parameters": true, + "check_all_ranks": true, + "check_all_outputs": true, + "finite_required": true + }, + "measurement": { + "unit": "ms", + "scope": "complete forward, host scheduling and all communication included", + "rank_aggregation": "maximum elapsed for each paired call", + "case_aggregation": "geometric mean of baseline/candidate per-case median latency ratios", + "warmups": 2, + "samples": 16, + "new_input_each_call": true, + "final_pairs": 4, + "timed_case_ids": [ + "balanced-small", + "skewed-medium", + "balanced-large" + ], + "samples_per_pair": 8, + "exploration_pairs": 2, + "matched_shape_warmup": { + "enabled": true, + "calls_per_implementation_per_timed_seed": 1, + "input": "distinct BF16 values from router-nullspace noise; same selected expert set for every token", + "noise_std_before_projection": 0.3, + "minimum_row_rms_input_change": 0.15, + "minimum_router_boundary_margin": 0.02, + "max_row_resampling_attempts": 32, + "verify_all_warm_outputs": true, + "require_warm_oracle_outside_timed_output_tolerance": true, + "cost": "recorded separately from scored forward latency" + } + }, + "cases": [ + { + "id": "balanced-small", + "rank_tokens": [ + 64, + 64 + ], + "hidden": 512, + "intermediate": 1024, + "top_k": 2, + "routing": "balanced", + "weight_seed": 1729, + "weight_variant": 0, + "measured": true + }, + { + "id": "skewed-medium", + "rank_tokens": [ + 256, + 256 + ], + "hidden": 1024, + "intermediate": 2048, + "top_k": 2, + "routing": "skewed", + "weight_seed": 2718, + "weight_variant": 0, + "measured": true + }, + { + "id": "balanced-large", + "rank_tokens": [ + 1024, + 1024 + ], + "hidden": 2048, + "intermediate": 4096, + "top_k": 2, + "routing": "balanced", + "weight_seed": 3141, + "weight_variant": 0, + "measured": true + }, + { + "id": "unequal-top1", + "rank_tokens": [ + 31, + 7 + ], + "hidden": 128, + "intermediate": 256, + "top_k": 1, + "routing": "balanced", + "weight_seed": 1618, + "weight_variant": 0, + "measured": false + }, + { + "id": "empty-rank-experts", + "rank_tokens": [ + 23, + 17 + ], + "hidden": 128, + "intermediate": 256, + "top_k": 2, + "routing": "empty-rank-experts", + "weight_seed": 1618, + "weight_variant": 0, + "measured": false + }, + { + "id": "zero-input-rank", + "rank_tokens": [ + 0, + 19 + ], + "hidden": 128, + "intermediate": 256, + "top_k": 2, + "routing": "balanced", + "weight_seed": 1618, + "weight_variant": 0, + "measured": false + }, + { + "id": "weight-update", + "rank_tokens": [ + 31, + 7 + ], + "hidden": 128, + "intermediate": 256, + "top_k": 1, + "routing": "balanced", + "weight_seed": 1618, + "weight_variant": 1, + "measured": false + } + ], + "source_provenance": [ + { + "project": "vLLM-Ascend", + "revision": "0f85ed44bbbbdc0f0447f83ce62a82a8ad55afb3", + "url": "https://github.com/vllm-project/vllm-ascend/blob/0f85ed44bbbbdc0f0447f83ce62a82a8ad55afb3/vllm_ascend/ops/fused_moe/prepare_finalize.py", + "role": "AllGather preparation and local expert / combine workload architecture; implementation here is newly written" + }, + { + "project": "vLLM-Ascend", + "revision": "0f85ed44bbbbdc0f0447f83ce62a82a8ad55afb3", + "url": "https://github.com/vllm-project/vllm-ascend/blob/0f85ed44bbbbdc0f0447f83ce62a82a8ad55afb3/vllm_ascend/ops/fused_moe/moe_comm_method.py", + "role": "complete dispatch / expert MLP / combine forward boundary" + } + ], + "tolerance": { + "atol": 0.03, + "rtol": 0.03, + "nrmse": 0.01, + "frozen": true + } +} diff --git a/src/inferrsi/data/engineering_scenarios/moe-forward-npu/workload.py b/src/inferrsi/data/engineering_scenarios/moe-forward-npu/workload.py new file mode 100644 index 000000000..2d1d96f64 --- /dev/null +++ b/src/inferrsi/data/engineering_scenarios/moe-forward-npu/workload.py @@ -0,0 +1,192 @@ +"""Trusted, CPU-created inputs and deterministic EP-local model parameters. + +No device import/initialization occurs until build_case receives a device. +Candidate implementations may use only the tensors/config passed to compute; +regenerating other ranks' weights or using these generator seeds is prohibited. +""" + +import json +import math +from functools import lru_cache +from pathlib import Path + +import torch + +TASK_FILE = Path(__file__).with_name("task.json") + + +def load_task(): + return json.loads(TASK_FILE.read_text(encoding="utf-8")) + + +def load_cases(): + return load_task()["cases"] + + +def _generator(seed): + return torch.Generator(device="cpu").manual_seed(int(seed) % (2**63 - 1)) + + +def _weight_key(case): + return ( + int(case["hidden"]), int(case["intermediate"]), + int(case["weight_seed"]), int(case.get("weight_variant", 0)), + str(case["routing"]), + ) + + +@lru_cache(maxsize=12) +def _common_cpu_weights(key): + hidden, intermediate, seed, variant, routing = key + gen = _generator(seed + 1_000_003 * variant) + router_weight = torch.randn(8, hidden, generator=gen) / math.sqrt(hidden) + router_bias = torch.zeros(8, dtype=torch.float32) + if routing == "skewed": + router_bias[0], router_bias[1] = 3.0, 1.5 + elif routing == "empty-rank-experts": + router_bias.fill_(-16.0) + router_bias[0], router_bias[1] = 16.0, 15.0 + elif routing != "balanced": + raise ValueError(f"Unknown routing distribution: {routing}") + shared_gate_up = ( + torch.randn(2 * intermediate, hidden, generator=gen) / math.sqrt(hidden) + ).to(torch.bfloat16) + shared_down = ( + torch.randn(hidden, intermediate, generator=gen) / math.sqrt(intermediate) + ).to(torch.bfloat16) + return router_weight, router_bias, shared_gate_up, shared_down + + +@lru_cache(maxsize=96) +def _expert_cpu_weights(key, expert_id): + hidden, intermediate, seed, variant, _ = key + gen = _generator(seed + 1_000_003 * variant + 10_007 * (int(expert_id) + 1)) + gate_up = ( + torch.randn(2 * intermediate, hidden, generator=gen) / math.sqrt(hidden) + ).to(torch.bfloat16) + down = ( + torch.randn(hidden, intermediate, generator=gen) / math.sqrt(intermediate) + ).to(torch.bfloat16) + return gate_up, down + + +def cpu_parameters(case, expert_ids): + """Fresh parameter containers; caller chooses only its owned expert IDs. + + Oracle may request all eight experts. The candidate never receives this + function, the case record, seeds, or the global expert tensors as arguments. + """ + key = _weight_key(case) + router_weight, router_bias, shared_gate_up, shared_down = _common_cpu_weights(key) + experts = [_expert_cpu_weights(key, expert_id) for expert_id in expert_ids] + return { + "router_weight": router_weight, + "router_bias": router_bias, + "expert_gate_up": torch.stack([weights[0] for weights in experts]), + "expert_down": torch.stack([weights[1] for weights in experts]), + "shared_gate_up": shared_gate_up, + "shared_down": shared_down, + } + + +def cpu_input(case, rank, seed): + """Different rank and call seeds produce different, BF16 input tensors. + + Random test inputs near a router top-k boundary are regenerated outside + measurement. This avoids making an essentially tied routing decision a + numerical-precision contest between CPU and NPU FP32 matmul implementations. + It does not precompute or provide candidate routing decisions. + """ + count = int(case["rank_tokens"][rank]) + hidden = int(case["hidden"]) + gen = _generator(int(seed) + 97_409 * (rank + 1)) + x = (0.7 * torch.randn(count, hidden, generator=gen)).to(torch.bfloat16) + if count == 0: + return x + router, bias, _, _ = _common_cpu_weights(_weight_key(case)) + for _ in range(16): + logits = torch.nn.functional.linear(x.float(), router, bias) + ordered = logits.sort(dim=-1, descending=True).values + k = int(case["top_k"]) + bad = ordered[:, k - 1] - ordered[:, k] < 0.025 + n_bad = int(bad.sum()) + if not n_bad: + return x.contiguous() + x[bad] = (0.7 * torch.randn(n_bad, hidden, generator=gen)).to(torch.bfloat16) + raise RuntimeError("Could not create unambiguous routing inputs") + + +@lru_cache(maxsize=12) +def _router_nullspace_projection(key): + router, _, _, _ = _common_cpu_weights(key) + # Solve rather than explicitly invert W W^T. The eight random router rows + # are full rank; failure is an invalid workload, never a silent fallback. + return torch.linalg.solve(router @ router.T, router) + + +def warm_input(case, rank, seed): + """Different token values with the same per-token selected expert sets. + + The timed input is never reused as its warmup, nor merely permuted. A + substantial independent perturbation in the router's nullspace preserves + expert GEMM row counts while changing the expert/shared-expert computation. + BF16 quantization slightly perturbs router logits, so every row is checked + again after quantization. Rejected rows are resampled; there is no fallback + to the original input or to smaller noise. + """ + original = cpu_input(case, rank, seed) + if original.shape[0] == 0: + return original.clone() + original_fp32 = original.float() + router, bias, _, _ = _common_cpu_weights(_weight_key(case)) + projection = _router_nullspace_projection(_weight_key(case)) + top_k = int(case["top_k"]) + original_logits = torch.nn.functional.linear(original_fp32, router, bias) + original_experts = original_logits.topk(top_k, dim=-1).indices.sort(dim=-1).values + gen = _generator(int(seed) + 30_000_019 + 1_009_117 * (rank + 1)) + pending = torch.arange(original.shape[0]) + warmed = torch.empty_like(original) + for _ in range(32): + noise = 0.3 * torch.randn(pending.numel(), original.shape[1], generator=gen) + noise = noise - (noise @ router.T) @ projection + trial = (original_fp32[pending] + noise).to(torch.bfloat16) + trial_fp32 = trial.float() + logits = torch.nn.functional.linear(trial_fp32, router, bias) + selected = logits.topk(top_k, dim=-1).indices.sort(dim=-1).values + ordered = logits.sort(dim=-1, descending=True).values + margin = ordered[:, top_k - 1] - ordered[:, top_k] + row_rms_change = (trial_fp32 - original_fp32[pending]).square().mean(dim=-1).sqrt() + accepted = ( + (selected == original_experts[pending]).all(dim=-1) + & (margin >= 0.02) + & (row_rms_change >= 0.15) + & torch.isfinite(trial_fp32).all(dim=-1) + ) + warmed[pending[accepted]] = trial[accepted] + pending = pending[~accepted] + if pending.numel() == 0: + return warmed.contiguous() + raise RuntimeError("Could not create distinct routing-shape-matched warm input") + + +def build_case(case, rank, device, seed): + """Return local input, EP-local parameters and public structural config.""" + if rank not in (0, 1): + raise ValueError("This task requires exactly two ranks") + cpu_params = cpu_parameters(case, range(rank * 4, rank * 4 + 4)) + params = {name: tensor.to(device=device, copy=True) for name, tensor in cpu_params.items()} + config = { + "world_size": 2, + "rank": rank, + "num_experts": 8, + "experts_per_rank": 4, + "hidden": int(case["hidden"]), + "intermediate": int(case["intermediate"]), + "top_k": int(case["top_k"]), + "rank_tokens": list(case["rank_tokens"]), + } + return { + "x": cpu_input(case, rank, seed).to(device=device, copy=True), + "params": params, + "config": config, + } diff --git a/src/inferrsi/harness/benchmarks/__init__.py b/src/inferrsi/harness/benchmarks/__init__.py new file mode 100644 index 000000000..21a95e916 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/__init__.py @@ -0,0 +1,5 @@ +"""Shared engineering task contracts; execution belongs to the existing harness.""" + +from .contract import TaskPackage, builtin_tasks, load_builtin_task, load_task + +__all__ = ["TaskPackage", "builtin_tasks", "load_builtin_task", "load_task"] diff --git a/src/inferrsi/harness/benchmarks/cli.py b/src/inferrsi/harness/benchmarks/cli.py new file mode 100644 index 000000000..490a74961 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/cli.py @@ -0,0 +1,84 @@ +"""CLI-first engineering benchmark operations; all execution is explicit.""" + +import json +import math +from pathlib import Path + +from ...infra.artifacts import atomic_write_json +from ..runner import run_experiment +from .consumers import grade_artifact, run_action +from .contract import builtin_tasks, load_task + + +def add_parser(commands): + root = commands.add_parser("benchmarks", help="Shared engineering tasks (trusted sources)") + actions = root.add_subparsers(dest="benchmark_action", required=True) + actions.add_parser("list") + show = actions.add_parser("show") + show.add_argument("task") + for name in ("check", "measure", "profile", "grade", "plan", "compare"): + child = actions.add_parser(name) + child.add_argument("task", help="Built-in task ID or local task package directory") + child.add_argument( + "--candidate", + type=Path, + required=True, + help="Reviewed self-contained Python source; no security sandbox", + ) + child.add_argument("--output-dir", type=Path, required=True) + if name in {"plan", "compare"}: + child.add_argument("--repo", type=Path, required=True) + child.add_argument("--repetitions", type=int, required=True) + child.add_argument("--max-minutes", type=float, required=True) + child.add_argument("--min-improvement-pct", type=float, required=True) + else: + child.add_argument("--timeout-seconds", type=float, required=True) + + +def dispatch(args) -> dict | list: + action = args.benchmark_action + if action == "list": + return builtin_tasks() + task = load_task(args.task) + if action == "show": + return { + "manifest": task.manifest, + "task_sha256": task.task_sha256, + "grader_sha256": task.grader_sha256, + } + if action in {"plan", "compare"}: + spec = task.runspec( + args.candidate, + repo=args.repo, + output_dir=args.output_dir, + repetitions=args.repetitions, + max_minutes=args.max_minutes, + min_improvement_pct=args.min_improvement_pct, + ) + if action == "plan": + return spec.to_dict() + identity = task.identity(args.candidate) + run_dir = run_experiment( + spec, + { + "benchmark": identity, + "backend": task.manifest["environment"]["backend"], + "qualification": "public development workload; see task coverage and run evidence", + }, + ) + atomic_write_json(run_dir / "task.lock.json", task.manifest) + return { + "run_dir": str(run_dir), + "status": json.loads((run_dir / "status.json").read_text(encoding="utf-8")), + "scope": "local-artifact-comparison", + "campaign_acceptance": "not established", + } + if not math.isfinite(args.timeout_seconds) or args.timeout_seconds <= 0: + raise ValueError("Timeout must be positive and finite") + if action == "grade": + return grade_artifact( + task, args.candidate, args.output_dir, timeout_seconds=args.timeout_seconds + ) + return run_action( + task, args.candidate, action, args.output_dir, timeout_seconds=args.timeout_seconds + ) diff --git a/src/inferrsi/harness/benchmarks/consumers.py b/src/inferrsi/harness/benchmarks/consumers.py new file mode 100644 index 000000000..8d2334c75 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/consumers.py @@ -0,0 +1,69 @@ +"""Thin consumers; no alternate runner, agent client or result truth.""" + +import json +import math +import time +from pathlib import Path + +from ...infra.artifacts import atomic_write_json +from ..scenarios.external import execute_trial +from ..skill_evaluation import valid_checks + + +def run_action(task, candidate: Path, action: str, output: Path, *, timeout_seconds: float) -> dict: + """Execute trusted local source. This API is not an untrusted-code sandbox.""" + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, (int, float)) + or not math.isfinite(timeout_seconds) + or not 0 < timeout_seconds <= 7 * 24 * 3600 + ): + raise ValueError("Timeout must be positive, finite and at most seven days") + command = task.command(action, candidate) + return execute_trial( + command, + candidate.resolve().parent, + output.resolve(), + time.monotonic() + timeout_seconds, + task.manifest["measurement"]["key"] if action == "measure" else None, + ) + + +def grade_artifact(task, candidate: Path, output: Path, *, timeout_seconds: float) -> dict: + """Independent correctness for offline engineering evaluation. + + Checks have SkillEval's existing shape. An agent/skill comparison must still + establish its own task delivery, trace, budget and matched conditions. + """ + identity = task.identity(candidate) + execution = run_action(task, candidate, "check", output, timeout_seconds=timeout_seconds) + result, checks = None, None + try: + result = json.loads(Path(execution["result_path"]).read_text(encoding="utf-8")) + task.assert_unchanged() + if ( + task.identity(candidate) == identity + and result.get("identity") == identity + and result.get("action") == "check" + and result.get("status") == "completed" + ): + proposed = {key: result.get(key) for key in ("passed", "assertions")} + if ( + valid_checks(proposed) + and execution["status"] in {"completed", "failed"} + and execution["returncode"] == (0 if proposed["passed"] else 1) + ): + checks = proposed + except (OSError, ValueError, TypeError): + pass + verdict = "INCONCLUSIVE" if checks is None else "PASS" if checks["passed"] else "FAIL" + record = { + "scope": "artifact-correctness-only", + "identity": identity, + "verdict": verdict, + "checks": checks, + "execution_status": execution["status"], + "agent_benefit": "not measured", + } + atomic_write_json(output / "artifact-evaluation.json", record) + return record diff --git a/src/inferrsi/harness/benchmarks/contract.py b/src/inferrsi/harness/benchmarks/contract.py new file mode 100644 index 000000000..4acac0f3a --- /dev/null +++ b/src/inferrsi/harness/benchmarks/contract.py @@ -0,0 +1,326 @@ +"""Versioned task truth and bindings to existing Workbench/SkillEval contracts.""" + +import hashlib +import json +import math +import os +import platform +import re +import sys +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version +from importlib.resources import files +from pathlib import Path + +from ...runspec import RunSpec + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def tree_digest(root: Path) -> str: + entries = {} + for path in sorted(root.rglob("*")): + if "__pycache__" in path.parts or path.suffix == ".pyc": + continue + if path.is_symlink(): + raise ValueError("Task packages must not contain symlinks") + if path.is_file(): + entries[path.relative_to(root).as_posix()] = digest(path) + return hashlib.sha256(json.dumps(entries, sort_keys=True).encode()).hexdigest() + + +def grader_digest() -> str: + return tree_digest(Path(__file__).parent) + + +def runtime_digest(environment=None) -> str: + try: + numpy_version = version("numpy") + except PackageNotFoundError: + numpy_version = "unavailable" + facts = [platform.python_version(), platform.system(), platform.machine(), numpy_version] + if environment and environment["backend"] == "ascend": + for package in ("torch", "torch-npu"): + try: + facts.append(version(package)) + except PackageNotFoundError: + facts.append("unavailable") + facts.append(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "unallocated")) + return hashlib.sha256(json.dumps(facts).encode()).hexdigest() + + +def builtin_tasks() -> list[str]: + root = files("inferrsi").joinpath("data/engineering_benchmarks") + return sorted(p.name for p in root.iterdir() if p.joinpath("task.json").is_file()) + + +def _text(value, field): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a nonempty string") + + +def _positive(value, field, *, integer=False): + if ( + isinstance(value, bool) + or not isinstance(value, int if integer else (int, float)) + or not math.isfinite(value) + or value <= 0 + ): + raise ValueError(f"{field} must be positive and finite") + + +@dataclass(frozen=True) +class TaskPackage: + root: Path + frozen_manifest: str + task_sha256: str + grader_sha256: str + + @property + def manifest(self) -> dict: + # Callers cannot mutate the task by changing a returned dictionary. + return json.loads(self.frozen_manifest) + + def assert_unchanged(self) -> None: + if tree_digest(self.root) != self.task_sha256 or grader_digest() != self.grader_sha256: + raise ValueError("Frozen task or grader changed") + + def identity(self, candidate: Path) -> dict: + self.assert_unchanged() + candidate = candidate.absolute() + if candidate.is_symlink() or not candidate.is_file(): + raise ValueError("Candidate must be a regular source file") + return { + "task_id": self.manifest["id"], + "task_sha256": self.task_sha256, + "grader_sha256": self.grader_sha256, + "candidate_sha256": digest(candidate), + "runtime_sha256": runtime_digest(self.manifest["environment"]), + } + + def command(self, action: str, candidate: Path) -> tuple[str, ...]: + if action not in {"check", "measure", "profile"}: + raise ValueError("Unknown task action") + identity = self.identity(candidate) + return ( + sys.executable, + "-B", + "-m", + "inferrsi.harness.benchmarks.worker", + action, + "--package", + str(self.root), + "--candidate", + str(candidate.resolve()), + "--task-sha256", + identity["task_sha256"], + "--grader-sha256", + identity["grader_sha256"], + "--candidate-sha256", + identity["candidate_sha256"], + "--runtime-sha256", + identity["runtime_sha256"], + ) + + def runspec( + self, + candidate: Path, + *, + repo: Path, + output_dir: Path, + repetitions: int, + max_minutes: float, + min_improvement_pct: float, + ) -> RunSpec: + """Bind one task to AB/BA; the caller owns repetition/budget/gain gates.""" + metric = self.manifest["measurement"] + spec = RunSpec( + repo=str(repo.resolve()), + repo_type="engineering_task", + backend=self.manifest["environment"]["backend"], + baseline_command=self.command("measure", self.root / "baseline.py"), + candidate_command=self.command("measure", candidate), + validation_command=self.command("check", candidate), + metric_key=metric["key"], + metric_unit=metric["unit"], + comparison_scope=self.manifest["id"] + ":" + metric["boundary"], + direction="minimize", + repetitions=repetitions, + max_minutes=max_minutes, + min_improvement_pct=min_improvement_pct, + workload_id=self.task_sha256, + output_dir=str(output_dir.resolve()), + mode="code", + ) + spec.validate() + return spec + + +def load_task(name: str | Path) -> TaskPackage: + """Load a reviewed path or a packaged name for the trusted Workbench API.""" + root = Path(name) + if not root.is_dir(): + return load_builtin_task(str(name)) + return _load_task_root(root) + + +def load_builtin_task(name: str) -> TaskPackage: + """Resolve only packaged task resources, never a same-named cwd directory.""" + if not isinstance(name, str) or name not in builtin_tasks(): + raise ValueError(f"Unknown built-in task package: {name}") + root = Path(str(files("inferrsi").joinpath("data/engineering_benchmarks", name))) + package = _load_task_root(root) + if package.manifest["id"] != name: + raise ValueError("Built-in task identity does not match its resource name") + return package + + +def _load_task_root(root: Path) -> TaskPackage: + if root.is_symlink(): + raise ValueError("Task root must not be a symlink") + root = root.resolve() + data = json.loads((root / "task.json").read_text(encoding="utf-8")) + required = { + "schema_version", + "id", + "description", + "searchspace", + "source", + "environment", + "candidate", + "oracle", + "measurement", + "workloads", + "coverage", + "adapter", + } + if ( + not isinstance(data, dict) + or set(data) != required + or type(data["schema_version"]) is not int + ): + raise ValueError("Invalid task fields") + if data["schema_version"] != 1 or data["adapter"] not in { + "numpy-axis1-v2", "torch-npu-rmsnorm-v1" + }: + raise ValueError("Unsupported task schema or adapter") + npu = data["adapter"] == "torch-npu-rmsnorm-v1" + if not isinstance(data["id"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]*", data["id"]): + raise ValueError("Invalid task id") + for field in ("description", "searchspace"): + _text(data[field], field) + source = data["source"] + if not isinstance(source, dict) or set(source) != {"url", "revision", "license", "adaptation"}: + raise ValueError("Source provenance is required") + for key, value in source.items(): + _text(value, "source." + key) + if not re.fullmatch(r"[0-9a-f]{40}", source["revision"]): + raise ValueError("Source revision must be a full commit") + env = data["environment"] + expected_env = ( + {"backend": "ascend", "runtime": "torch_npu", "dtype": "float32"} if npu + else {"backend": "cpu", "runtime": "numpy", "dtype": "float32"} + ) + if env != expected_env: + raise ValueError("Unsupported environment; no implicit device fallback") + if data["candidate"] != {"entrypoint": "compute", "editable": ["implementation.py"]}: + raise ValueError("Unsupported candidate contract") + oracle = data["oracle"] + if ( + not isinstance(oracle, dict) + or set(oracle) != {"operation", "axis", "eps", "rtol", "atol"} + or oracle["operation"] not in {"softmax", "rmsnorm"} + or type(oracle["axis"]) is not int + or oracle["axis"] != 1 + ): + raise ValueError("Unsupported oracle") + for key in ("eps", "rtol", "atol"): + _positive(oracle[key], "oracle." + key) + if npu and oracle["operation"] != "rmsnorm": + raise ValueError("NPU adapter only supports affine RMSNorm") + metric = data["measurement"] + if ( + not isinstance(metric, dict) + or set(metric) + != { + "key", + "unit", + "boundary", + "aggregation", + "warmups", + "samples", + "input_sequence", + "case_ids", + } + or metric["key"] != "metrics.suite_latency_ms" + or metric["unit"] != "ms" + or metric["boundary"] != ( + "device-synchronized-eager-call" if npu else "synchronous-compute-call" + ) + or metric["aggregation"] != "sum-of-per-case-medians" + or metric["input_sequence"] != "seeded-distinct-per-call-v1" + ): + raise ValueError("Unsupported measurement contract") + for key in ("warmups", "samples"): + _positive(metric[key], "measurement." + key, integer=True) + if ( + not isinstance(metric["case_ids"], list) + or not metric["case_ids"] + or any(not isinstance(name, str) for name in metric["case_ids"]) + or len(set(metric["case_ids"])) != len(metric["case_ids"]) + ): + raise ValueError("Distinct measured case IDs are required") + if not isinstance(data["workloads"], list) or not data["workloads"]: + raise ValueError("Workloads are required") + names = set() + for case in data["workloads"]: + if not isinstance(case, dict) or set(case) != { + "id", + "shape", + "seed", + "distribution", + "layout", + }: + raise ValueError("Invalid workload fields") + _text(case["id"], "workload id") + if case["id"] in names: + raise ValueError("Duplicate workload id") + names.add(case["id"]) + if ( + not isinstance(case["shape"], list) + or not 2 <= len(case["shape"]) <= 4 + or any(type(n) is not int or n < 1 for n in case["shape"]) + or math.prod(case["shape"]) > 1048576 + ): + raise ValueError("Invalid or oversized workload") + if npu and (len(case["shape"]) != 2 or case["layout"] != "contiguous"): + raise ValueError("NPU task requires contiguous 2D inputs") + if type(case["seed"]) is not int or case["seed"] < 0: + raise ValueError("Seed must be a nonnegative integer") + if case["distribution"] not in {"normal", "zeros", "large", "tiny"}: + raise ValueError("Unknown distribution") + if case["layout"] not in {"contiguous", "strided"}: + raise ValueError("Unknown layout") + measured_cases = [case for case in data["workloads"] if case["id"] in metric["case_ids"]] + if [case["id"] for case in measured_cases] != metric["case_ids"]: + raise ValueError("Measured case IDs must follow workload order") + if any(case["distribution"] == "zeros" for case in measured_cases): + raise ValueError("Invariant zero workloads are correctness-only") + prepared_elements = sum(math.prod(case["shape"]) for case in data["workloads"]) + prepared_elements += (metric["warmups"] + metric["samples"]) * sum( + math.prod(case["shape"]) for case in measured_cases + ) + if prepared_elements > (16777216 if npu else 1048576): + raise ValueError("Prepared input sequence exceeds the workload bound") + if ( + not isinstance(data["coverage"], list) + or not data["coverage"] + or any(not isinstance(v, str) or not v.strip() for v in data["coverage"]) + ): + raise ValueError("Coverage limits are required") + for path in ("baseline.py", "candidate.py", "LICENSE", "README.md"): + if not (root / path).is_file(): + raise ValueError(f"Task package missing {path}") + return TaskPackage(root, json.dumps(data, sort_keys=True), tree_digest(root), grader_digest()) diff --git a/src/inferrsi/harness/benchmarks/npu_task.py b/src/inferrsi/harness/benchmarks/npu_task.py new file mode 100644 index 000000000..2303c534b --- /dev/null +++ b/src/inferrsi/harness/benchmarks/npu_task.py @@ -0,0 +1,211 @@ +"""Trusted affine RMSNorm workload on an explicitly allocated Ascend NPU. + +The independent CPU oracle and host/device transfers are outside timing. Timing +is synchronized eager-call latency, not pure kernel time. No CPU fallback and no +untrusted-code sandbox are provided by this Workbench adapter. +""" + +import hashlib +import json +import math +import os +import platform +import statistics +import time +from pathlib import Path + +from .numpy_task import input_record, inputs + + +def reference(x, weight, eps, np): + """Scalar double arithmetic, independent of the device reduction/fused op.""" + expected = np.empty_like(x) + for index, row in enumerate(x): + values = [float(value) for value in row] + scale = math.sqrt(math.fsum(value * value for value in values) / len(values) + eps) + expected[index] = [value / scale * float(w) for value, w in zip(values, weight)] + return expected + + +def environment(torch, torch_npu, np): + from torch_npu.npu.utils import get_cann_version + + return { + "backend": "ascend", + "device": torch.npu.get_device_name(0), + "logical_device": 0, + "python": platform.python_version(), + "torch": torch.__version__, + "torch_npu": torch_npu.__version__, + "numpy": np.__version__, + "cann": str(get_cann_version()), + "os": platform.system(), + "machine": platform.machine(), + "allocation": "caller-owned; one visible device", + } + + +def check_output(output, x, weight, original, original_weight, expected, oracle, torch, np): + tensor = isinstance(output, torch.Tensor) + shape_dtype_device = ( + tensor and tuple(output.shape) == original.shape + and output.dtype == torch.float32 and output.device == torch.device("npu:0") + ) + def preserved(tensor, original): + return (tuple(tensor.shape) == original.shape and tensor.dtype == torch.float32 + and tensor.device == torch.device("npu:0") and tensor.is_contiguous() + and tensor.cpu().numpy().tobytes() == original.tobytes()) + + # Reading results and checking preservation happens after device synchronization. + values = output.detach().cpu().numpy() if shape_dtype_device else None + return { + "shape_dtype_device": shape_dtype_device, + "finite": values is not None and bool(np.isfinite(values).all()), + "input_preserved": preserved(x, original), + "weight_preserved": preserved(weight, original_weight), + "values": values is not None + and bool(np.allclose(values, expected, rtol=oracle["rtol"], atol=oracle["atol"])), + } + + +def device_kernels(document): + """Read actual Ascend hardware records, not CPU launches/flow arrows.""" + events = document.get("traceEvents", []) if isinstance(document, dict) else document + hardware = {event["pid"] for event in events + if event.get("name") == "process_name" + and event.get("args", {}).get("name") == "Ascend Hardware"} + return [event for event in events + if event.get("ph") == "X" and event.get("pid") in hardware + and str(event.get("args", {}).get("Task Type", "")).startswith("KERNEL_")] + + +def evaluate(task, candidate: Path, action: str, result_dir: Path) -> dict: + # The caller owns resource admission. This adapter does not reserve devices + # or reinterpret the scheduler's selected physical device IDs. + allocated = os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "") + if not allocated or len(allocated.split(",")) != 1: + raise RuntimeError("One explicitly allocated visible NPU is required") + import numpy as np + import torch + import torch_npu + + if not torch.npu.is_available() or torch.npu.device_count() != 1: + raise RuntimeError("One NPU must be available; there is no CPU fallback") + torch.npu.set_device(0) + manifest = task.manifest + oracle, measurement = manifest["oracle"], manifest["measurement"] + namespace = {"__name__": "benchmark_candidate", "__file__": str(candidate)} + exec(compile(candidate.read_bytes(), "implementation.py", "exec"), namespace) + compute = namespace.get("compute") + if not callable(compute): + raise ValueError("Candidate must define compute(x, weight, *, eps)") + + prepared, records, seen = [], [], set() + for case in manifest["workloads"]: + count = 1 + if case["id"] in measurement["case_ids"]: + count += measurement["warmups"] + measurement["samples"] + for index in range(count): + original = inputs(case, np, sequence=index) + rng = np.random.default_rng(np.random.SeedSequence([case["seed"], index, 7])) + original_weight = rng.uniform(-1.5, 1.5, case["shape"][-1]).astype(np.float32) + expected = reference(original, original_weight, oracle["eps"], np) + record = input_record(case, original) + record["weight_sha256"] = hashlib.sha256(original_weight.tobytes()).hexdigest() + record["call_index"] = index + record["phase"] = ( + "precheck" if index == 0 + else "warmup" if index <= measurement["warmups"] else "measurement" + ) + # x itself varies: a weight-only change cannot evade x-output replay checks. + if record["input_sha256"] in seen: + raise ValueError("Repeated input bytes in NPU sequence") + seen.add(record["input_sha256"]) + x = torch.from_numpy(original.copy()).to("npu:0") + weight = torch.from_numpy(original_weight.copy()).to("npu:0") + prepared.append((case, x, weight, original, original_weight, expected, record)) + records.append(record) + torch.npu.synchronize() + checks, executed, samples = {}, [], {key: [] for key in measurement["case_ids"]} + result = { + "status": "completed", "action": action, "passed": False, + "assertions": checks, "environment": environment(torch, torch_npu, np), + "coverage": manifest["coverage"], "input_sequence": records, "executed_calls": executed, + } + + def call(entry, *, timed=False): + case, x, weight, original, original_weight, expected, record = entry + torch.npu.synchronize() + started = time.perf_counter_ns() if timed else None + with torch.inference_mode(): + output = compute(x, weight, eps=oracle["eps"]) + torch.npu.synchronize() + elapsed = (time.perf_counter_ns() - started) / 1e6 if timed else None + assertions = check_output( + output, x, weight, original, original_weight, expected, oracle, torch, np + ) + checks.update({f"{case['id']}.call-{record['call_index']}.{k}": v + for k, v in assertions.items()}) + executed.append(record) + if not all(assertions.values()): + result["failed_call"] = {**record, "usable_measurement": False} + return False + if timed: + if not math.isfinite(elapsed) or elapsed <= 0: + raise ValueError("Timer did not resolve positive latency") + samples[case["id"]].append({"elapsed_ms": elapsed, **record}) + return True + + # All correctness cases are checked before any timed input. + for entry in prepared: + if entry[-1]["phase"] == "precheck" and not call(entry): + return result + + if action == "profile": + # Warm all measured shapes before attaching the profiler. Profile one + # fresh input per shape; ordinary measurement never enables instrumentation. + for entry in prepared: + if entry[-1]["phase"] == "warmup" and not call(entry): + return result + selected = [entry for entry in prepared if entry[-1]["phase"] == "measurement" + and entry[-1]["call_index"] == measurement["warmups"] + 1] + with torch_npu.profiler.profile( + activities=[torch_npu.profiler.ProfilerActivity.CPU, + torch_npu.profiler.ProfilerActivity.NPU], + record_shapes=True, profile_memory=False, with_stack=False, + ) as profiler: + for entry in selected: + with torch.profiler.record_function("rmsnorm-candidate-call"): + if not call(entry): + return result + trace = result_dir / "npu-trace.json" + profiler.export_chrome_trace(str(trace)) + # Raw profiler output may contain local identifiers: retain locally; + # publication should use a reviewed, path-free kernel summary. + document = json.loads(trace.read_text(encoding="utf-8")) + kernels = device_kernels(document) + if not kernels: + raise RuntimeError("NPU profiler produced no device kernel events") + result["diagnostic"] = { + "kind": "ascend-torch-profiler", "artifact": "npu-trace.json", + "sha256": hashlib.sha256(trace.read_bytes()).hexdigest(), + "performance_verdict": None, + "kernels": [{"name": event["name"], "duration_us": event.get("dur")} + for event in kernels], + "limitation": "Instrumented diagnostic; includes verification copies; not a score", + } + else: + for entry in prepared: + if entry[-1]["phase"] == "precheck": + continue + if not call(entry, timed=action == "measure" and entry[-1]["phase"] == "measurement"): + return result + if action == "measure": + cases = [{"id": key, "samples_ms": [v["elapsed_ms"] for v in values], + "sample_input_sha256": [v["input_sha256"] for v in values], + "median_ms": statistics.median(v["elapsed_ms"] for v in values)} + for key, values in samples.items()] + result.update(cases=cases, measurement=measurement, + metrics={"suite_latency_ms": sum(case["median_ms"] for case in cases)}) + result["passed"] = all(checks.values()) + return result diff --git a/src/inferrsi/harness/benchmarks/numpy_task.py b/src/inferrsi/harness/benchmarks/numpy_task.py new file mode 100644 index 000000000..883c7f559 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/numpy_task.py @@ -0,0 +1,211 @@ +"""CPU task payload; never schedules processes or makes performance verdicts. + +Only reviewed, trusted implementations may run here. Hash checks detect drift, +not malicious code. The public input suite is not a hidden evaluation set. +""" + +import cProfile +import hashlib +import math +import platform +import pstats +import statistics +import time +from pathlib import Path + + +def inputs(case, np, *, sequence=0): + seed = case["seed"] if sequence == 0 else np.random.SeedSequence([case["seed"], sequence]) + rng = np.random.default_rng(seed) + x = rng.standard_normal(case["shape"]).astype(np.float32) + if case["distribution"] == "zeros": + x.fill(0) + elif case["distribution"] == "large": + x = x * np.float32(100) + np.float32(1000) + elif case["distribution"] == "tiny": + x *= np.float32(1e-7) + if case["layout"] == "strided": + storage = np.empty((*x.shape[:-1], x.shape[-1] * 2), dtype=np.float32) + view = storage[..., ::2] + view[...] = x + x = view + return x + + +def reference(x, oracle, np): + """Scalar double-precision oracle, independent of NumPy reduction kernels.""" + rows = np.moveaxis(x, 1, -1) + output = np.empty_like(rows) + for index in np.ndindex(rows.shape[:-1]): + values = [float(v) for v in rows[index]] + if oracle["operation"] == "softmax": + maximum = max(values) + exps = [math.exp(v - maximum) for v in values] + total = math.fsum(exps) + result = [v / total for v in exps] + else: + scale = math.sqrt(math.fsum(v * v for v in values) / len(values) + oracle["eps"]) + result = [v / scale for v in values] + output[index] = result + return np.moveaxis(output, -1, 1) + + +def check_output(output, x, original, expected, oracle, np): + return { + "shape_dtype": isinstance(output, np.ndarray) + and output.shape == x.shape + and output.dtype == x.dtype, + "finite": isinstance(output, np.ndarray) and bool(np.isfinite(output).all()), + "input_preserved": x.tobytes() == original.tobytes(), + "values": isinstance(output, np.ndarray) + and output.shape == expected.shape + and bool( + np.allclose(output, expected, rtol=oracle["rtol"], atol=oracle["atol"], equal_nan=False) + ), + } + + +def input_record(case, x): + return { + "id": case["id"], + "input_sha256": hashlib.sha256(x.tobytes()).hexdigest(), + "shape": list(x.shape), + "dtype": str(x.dtype), + "strides": list(x.strides), + } + + +def evaluate(task, candidate: Path, action: str, result_dir: Path) -> dict: + # Optional dependency is loaded only for execution, never task discovery. + import numpy as np + + manifest = task.manifest + source = candidate.read_bytes() + namespace = {"__name__": "benchmark_candidate", "__file__": str(candidate)} + exec(compile(source, "implementation.py", "exec"), namespace) + compute = namespace.get("compute") + if not callable(compute): + raise ValueError("Candidate must define compute(x, *, eps)") + oracle = manifest["oracle"] + measurement = manifest["measurement"] + checks, cases, workload_records, sequence_records, executed = {}, [], [], [], [] + prepared, sequences = [], [] + # Prepare all inputs/oracles before candidate calls. Keep every input alive: + # a previous call must not prime a later call's object/content memo entry. + for case in manifest["workloads"]: + x = inputs(case, np) + original = x.copy() + expected = reference(x, oracle, np) + workload_records.append(input_record(case, x)) + prepared.append((case, x, original, expected)) + seen = {record["input_sha256"] for record in workload_records} + for case in manifest["workloads"]: + if case["id"] not in measurement["case_ids"]: + continue + calls = [] + for index in range(measurement["warmups"] + measurement["samples"]): + x = inputs(case, np, sequence=index + 1) + record = input_record(case, x) + if record["input_sha256"] in seen: + raise ValueError("Input sequence repeats earlier input bytes") + seen.add(record["input_sha256"]) + record.update( + call_index=index, + phase="warmup" if index < measurement["warmups"] else "measurement", + ) + sequence_records.append(record) + calls.append((x, x.copy(), reference(x, oracle, np), record)) + sequences.append((case, calls)) + for (case, x, original, expected), record in zip(prepared, workload_records, strict=True): + executed.append({**record, "phase": "precheck"}) + output = compute(x, eps=oracle["eps"]) + assertions = check_output(output, x, original, expected, oracle, np) + checks.update({case["id"] + "." + k: v for k, v in assertions.items()}) + result = { + "status": "completed", + "action": action, + "passed": all(checks.values()), + "assertions": checks, + "environment": { + "backend": "cpu", + "python": platform.python_version(), + "numpy": np.__version__, + "os": platform.system(), + "machine": platform.machine(), + }, + "coverage": manifest["coverage"], + "workloads": workload_records, + "input_sequence": sequence_records, + "executed_calls": executed, + } + if not result["passed"]: + return result + + # Correctness and profiling do not masquerade as a paired measurement. + profiler = cProfile.Profile() if action == "profile" else None + for case, calls in sequences: + samples = [] + for index, (x, original, expected, record) in enumerate(calls): + executed.append(record) + elapsed = None + if action == "check": + output = compute(x, eps=oracle["eps"]) + else: + if profiler: + profiler.enable() + started = time.perf_counter_ns() + output = compute(x, eps=oracle["eps"]) + elapsed = (time.perf_counter_ns() - started) / 1e6 + if profiler: + profiler.disable() + assertions = check_output(output, x, original, expected, oracle, np) + checks.update({f"{case['id']}.call-{index}.{k}": v for k, v in assertions.items()}) + if not all(assertions.values()): + result.update( + passed=False, + cases=cases, + failed_call={ + "case": case["id"], + "call_index": index, + "phase": "warmup" if index < measurement["warmups"] else "measurement", + "elapsed_ms": elapsed, + "prior_samples_ms": samples, + "usable_measurement": False, + }, + ) + return result + if action != "check" and index >= measurement["warmups"]: + if not math.isfinite(elapsed) or elapsed <= 0: + raise ValueError("Timer did not resolve a positive duration") + samples.append(elapsed) + if action != "check": + cases.append( + { + "id": case["id"], + "samples_ms": samples, + "median_ms": statistics.median(samples), + "sample_input_sha256": [ + record["input_sha256"] + for _, _, _, record in calls[measurement["warmups"] :] + ], + } + ) + if action == "check": + return result + if profiler: + # Strip local installation paths from the textual profile before capture. + with (result_dir / "profile.txt").open("w", encoding="utf-8") as stream: + pstats.Stats(profiler, stream=stream).strip_dirs().sort_stats( + "cumulative" + ).print_stats() + result["diagnostic"] = { + "kind": "cpu-cprofile", + "artifact": "profile.txt", + "performance_verdict": None, + "limitation": "Instrumented CPU calls; no accelerator timeline", + } + else: + result["cases"] = cases + result["measurement"] = measurement + result["metrics"] = {"suite_latency_ms": sum(c["median_ms"] for c in cases)} + return result diff --git a/src/inferrsi/harness/benchmarks/skill_verifier.py b/src/inferrsi/harness/benchmarks/skill_verifier.py new file mode 100644 index 000000000..2ae3e3a74 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/skill_verifier.py @@ -0,0 +1,55 @@ +"""SkillEval verifier for a controller-selected, reviewed retained artifact. + +This is an offline handoff after artifact review, not live generated-code +execution. It deliberately leaves the production engineering sandbox unchanged. +""" + +import json +import os +import sys +from pathlib import Path + +from inferrsi.harness.benchmarks.consumers import grade_artifact +from inferrsi.harness.benchmarks.contract import load_task +from inferrsi.infra.artifacts import atomic_write_json + + +def task_descriptor(task, candidate: Path) -> dict: + return { + "id": task.manifest["id"], + "split": "development", + "request": "Submit the SHA-256 of the controller-reviewed retained implementation.", + "provenance": task.manifest["source"]["url"], + "benchmark": { + "package": str(task.root), + "candidate": str(candidate.resolve()), + "identity": task.identity(candidate), + }, + } + + +def verify(task: dict, answer: dict, output_dir: Path) -> dict: + binding = task["benchmark"] + package = load_task(binding["package"]) + candidate = Path(binding["candidate"]) + if package.identity(candidate) != binding["identity"]: + raise ValueError("Retained artifact or task identity changed") + if answer != {"candidate_sha256": binding["identity"]["candidate_sha256"]}: + return {"passed": False, "assertions": {"artifact_identity": False}} + record = grade_artifact(package, candidate, output_dir, timeout_seconds=25) + if record["checks"] is None: + raise ValueError("Artifact verification did not complete") + return record["checks"] + + +def main(): + destination = Path(os.environ["INFERRSI_RESULT_PATH"]) + task = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) + answer = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) + result = verify(task, answer, destination.parent / "artifact") + atomic_write_json(destination, result) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/inferrsi/harness/benchmarks/worker.py b/src/inferrsi/harness/benchmarks/worker.py new file mode 100644 index 000000000..d1aca05bf --- /dev/null +++ b/src/inferrsi/harness/benchmarks/worker.py @@ -0,0 +1,63 @@ +"""Execute one frozen task action under an existing InferRSI executor.""" + +import argparse +import json +import os +from pathlib import Path + +from ...infra.artifacts import atomic_write_json +from .contract import load_task + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("check", "measure", "profile")) + for field in ( + "package", + "candidate", + "task-sha256", + "grader-sha256", + "candidate-sha256", + "runtime-sha256", + ): + parser.add_argument("--" + field, required=True) + args = parser.parse_args() + destination = Path(os.environ["INFERRSI_RESULT_PATH"]) + result = {"status": "error", "action": args.action, "passed": False} + try: + task = load_task(args.package) + candidate = Path(args.candidate) + expected = { + "task_id": task.manifest["id"], + "task_sha256": args.task_sha256, + "grader_sha256": args.grader_sha256, + "candidate_sha256": args.candidate_sha256, + "runtime_sha256": args.runtime_sha256, + } + if task.identity(candidate) != expected: + raise ValueError("Frozen task, grader or candidate changed") + if task.manifest["adapter"] == "torch-npu-rmsnorm-v1": + from .npu_task import evaluate + else: + from .numpy_task import evaluate + + result = evaluate(task, candidate, args.action, destination.parent) + if task.identity(candidate) != expected: + raise ValueError("Frozen task, grader or candidate changed during execution") + result["identity"] = expected + except Exception as error: + # Errors are incomplete evidence, never numerical correctness failures. + # Do not capture arbitrary exception text that may expose private paths. + result = { + "status": "error", + "action": args.action, + "passed": False, + "error_type": type(error).__name__, + } + atomic_write_json(destination, result) + print(json.dumps({k: result[k] for k in ("status", "action", "passed")})) + return 0 if result["status"] == "completed" and result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/inferrsi/harness/scenarios/_moe_processes.py b/src/inferrsi/harness/scenarios/_moe_processes.py new file mode 100644 index 000000000..ec9d7b785 --- /dev/null +++ b/src/inferrsi/harness/scenarios/_moe_processes.py @@ -0,0 +1,70 @@ +"""Clean this MoE invocation's torchrun workers, including detached sessions. + +torchrun workers may create their own sessions, outside the executor's process +group. Only the launcher receives a fresh marker; it cannot reach the invoking +shell or allocator. This is lifecycle supervision for reviewed code, not a sandbox. +The marker must be generated in this driver after it starts, as evaluate() does. +""" + +import os +import signal +import time +from pathlib import Path + +MARKER = "INFERRSI_MOE_RUN_ID" + + +def _members(token: str, proc: Path = Path("/proc")) -> list[dict]: + marker = (MARKER + "=" + token).encode() + found = [] + # A fresh marker can only be inherited by processes started after this + # driver. Older same-user services may have unreadable environments; they + # cannot own this invocation. Keep same-tick processes in scope because + # Linux start times have tick granularity, not a strict ordering within it. + driver_fields = (proc / "self" / "stat").read_text().rsplit(")", 1)[1].split() + driver_start = int(driver_fields[19]) + # A missing procfs must raise, never be interpreted as successful cleanup. + for entry in proc.iterdir(): + if not entry.name.isdigit() or int(entry.name) == os.getpid(): + continue + try: + if entry.stat().st_uid != os.getuid(): + continue + fields = (entry / "stat").read_text().rsplit(")", 1)[1].split() + if fields[0] == "Z": + continue + start_ticks = int(fields[19]) + if start_ticks < driver_start: + continue + if marker in (entry / "environ").read_bytes().split(b"\0"): + found.append({"pid": int(entry.name), "start_ticks": start_ticks}) + except (FileNotFoundError, ProcessLookupError): + continue + # Unreadable in-scope live same-user processes still make the cleanup + # check inconclusive; in particular never swallow PermissionError. + return found + + +def cleanup(token: str, *, grace_seconds: float = 5.0) -> dict: + """Clean a marker minted by this driver; preserve PID-reuse protection.""" + killed = set() + deadline = time.monotonic() + grace_seconds + try: + while True: + members = _members(token) + if not members: + return {"verified": True, "signalled_pids": sorted(killed), "remaining": []} + for item in members: + try: + fields = Path(f"/proc/{item['pid']}/stat").read_text().rsplit(")", 1)[1].split() + if int(fields[19]) == item["start_ticks"]: + os.kill(item["pid"], signal.SIGKILL) + killed.add(item["pid"]) + except (FileNotFoundError, ProcessLookupError): + pass + if time.monotonic() >= deadline: + return {"verified": False, "signalled_pids": sorted(killed), + "remaining": _members(token)} + time.sleep(0.05) + except (OSError, ValueError, IndexError) as exc: + return {"verified": False, "signalled_pids": sorted(killed), "error": str(exc)} diff --git a/src/inferrsi/harness/scenarios/moe.py b/src/inferrsi/harness/scenarios/moe.py new file mode 100644 index 000000000..914393026 --- /dev/null +++ b/src/inferrsi/harness/scenarios/moe.py @@ -0,0 +1,273 @@ +"""Evaluate one reviewed MoE candidate on two caller-allocated Ascend NPUs. + +No agent, scheduler, SSH, search policy or token accounting is started here. +Importing this module and reading task resources require no device libraries. +""" + +import argparse +import hashlib +import json +import math +import os +import statistics +import sys +import time +import uuid +from importlib.resources import files +from pathlib import Path + +from ...infra.artifacts import atomic_write_json +from ._moe_processes import MARKER, cleanup +from .external import execute_trial + +TASK_FILES = ("task.json", "TASK.md", "baseline.py", "oracle.py", "workload.py", "LICENSE") + + +def task_resources(): + return files("inferrsi").joinpath("data", "engineering_scenarios", "moe-forward-npu") + + +def _positive(value): + return type(value) in (int, float) and math.isfinite(value) and value > 0 + + +def _preflight(candidate: Path, output: Path, pairs: int, timeout_seconds: float): + if type(pairs) is not int or pairs <= 0: + raise ValueError("pairs must be a positive integer") + if not _positive(timeout_seconds): + raise ValueError("timeout-seconds must be finite and positive") + if not candidate.is_file() or candidate.suffix != ".py": + raise ValueError("candidate must be a reviewed, self-contained Python file") + if output.exists(): + raise FileExistsError("output-dir must be new; use a separate directory per candidate") + if sys.platform != "linux": + raise OSError("MoE execution requires Linux, readable /proc and two allocated Ascend NPUs") + Path("/proc/self/stat").read_text() + visible = os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "").split(",") + if len(visible) != 2 or any(not x.strip().isdigit() for x in visible): + raise ValueError("Allocate two NPUs first (npu run --gpus 2); do not choose fixed card IDs") + if len({int(x.strip()) for x in visible}) != 2: + raise ValueError("ASCEND_RT_VISIBLE_DEVICES must contain two distinct allocated devices") + + +def _hashes(root: Path, names) -> dict: + return {name: hashlib.sha256((root / name).read_bytes()).hexdigest() for name in names} + + +def _prepare(candidate: Path, output: Path, pairs: int) -> dict: + output.mkdir(parents=True, exist_ok=False) + for directory in ("task", "candidate", "runner", "results"): + (output / directory).mkdir() + for name in TASK_FILES: + (output / "task" / name).write_bytes(task_resources().joinpath(name).read_bytes()) + (output / "candidate" / "implementation.py").write_bytes(candidate.read_bytes()) + grader = files("inferrsi.harness.scenarios").joinpath("moe_grader.py").read_bytes() + (output / "runner" / "grade.py").write_bytes(grader) + atomic_write_json(output / "request.json", {"repetitions": pairs}) + names = ["task/" + name for name in TASK_FILES] + names += ["candidate/implementation.py", "runner/grade.py", "request.json"] + identity = _hashes(output, names) + atomic_write_json(output / "identity.json", identity) + return identity + + +def _launch(root: Path, token: str): + # exec starts a process with the marker in its initial environment, visible + # in /proc and inherited even by torchrun children with their own sessions. + os.environ[MARKER] = token + argv = [sys.executable, "-B", "-m", "torch.distributed.run", "--standalone", + "--nnodes=1", "--nproc-per-node=2", str(root / "runner" / "grade.py"), + "--run-dir", str(root)] + os.execv(sys.executable, argv) + + +def _expected_calls(task: dict, pairs: int) -> dict: + expected = {} + measure = task["measurement"] + for index, case in enumerate(task["cases"]): + base = 100000 + index * 10000 + calls = [(base + i, "correctness", "standard", None, None, False) for i in range(2)] + if case["measured"]: + calls += [(base + 10 + i, "warmup", "standard", None, None, False) + for i in range(measure["warmups"])] + calls += [(base + 100 + pair * 100 + sample, phase, variant, pair, sample, timed) + for pair in range(pairs) for sample in range(measure["samples_per_pair"]) + for phase, variant, timed in (("shape_warmup", "shape_warmup", False), + ("measurement", "standard", True))] + for seed, phase, variant, pair, sample, timed in calls: + for label in ("baseline", "candidate"): + expected[(case["id"], seed, label, phase, variant, pair, sample)] = timed + return expected + + +def _score(task: dict, ranks: list[dict], summary: dict, pairs: int) -> dict: + """Recompute from complete raw calls/samples; a rank-0 summary is insufficient.""" + if (len(ranks) != 2 or {r.get("rank") for r in ranks} != {0, 1} + or any(type(r.get("rank")) is not int or r.get("completed") is not True + or r.get("correctness") != "PASS" or r.get("error") for r in ranks)): + raise ValueError("Both ranks must complete correctly without errors") + expected = _expected_calls(task, pairs) + checks = ("shape_dtype_device", "input_parameters_preserved", "finite", "values", "nrmse_pass") + by_rank = [] + for rank in ranks: + observed = {} + for call in rank["calls"]: + key = tuple(call[k] for k in ("case_id", "seed", "label", "phase", "variant", + "pair", "sample")) + if (key in observed or key not in expected + or call["included_in_score"] is not expected[key] + or call["all_ranks_correct"] is not True + or not all(call["checks"].get(k) is True for k in checks) + or not _positive(call["local_elapsed_ms"])): + raise ValueError("Missing, duplicated or invalid forward verification") + observed[key] = call + if set(observed) != set(expected): + raise ValueError("Incomplete correctness/warmup/measurement calls") + samples = {} + for sample in rank["samples"]: + key = tuple(sample[k] for k in ("case_id", "seed", "label", "pair", "sample")) + call_key = key[:3] + ("measurement", "standard") + key[3:] + if (key in samples or call_key not in observed + or sample["local_elapsed_ms"] != observed[call_key]["local_elapsed_ms"] + or sample["input_sha256"] != observed[call_key]["input_sha256"] + or not _positive(sample["slowest_rank_elapsed_ms"])): + raise ValueError("Invalid timed sample") + samples[key] = sample + if len(samples) != sum(expected.values()): + raise ValueError("Incomplete timed samples") + by_rank.append(samples) + if by_rank[0].keys() != by_rank[1].keys(): + raise ValueError("Ranks disagree on timed calls") + for key, first in by_rank[0].items(): + second = by_rank[1][key] + maximum = max(first["local_elapsed_ms"], second["local_elapsed_ms"]) + if (first["slowest_rank_elapsed_ms"] != second["slowest_rank_elapsed_ms"] + or not math.isclose(first["slowest_rank_elapsed_ms"], maximum, rel_tol=1e-6)): + raise ValueError("Timing does not use the maximum of both ranks") + cases = [] + for case in task["cases"]: + if not case["measured"]: + continue + grouped = {label: [s["slowest_rank_elapsed_ms"] for s in by_rank[0].values() + if s["case_id"] == case["id"] and s["label"] == label] + for label in ("baseline", "candidate")} + b, c = (statistics.median(grouped[label]) for label in ("baseline", "candidate")) + cases.append({"id": case["id"], "baseline_median_ms": b, "candidate_median_ms": c, + "speedup": b / c, "samples_ms": grouped}) + speedup = math.exp(statistics.mean(math.log(c["speedup"]) for c in cases)) + score = {"completed": True, "correctness": "PASS", "cases": cases, "speedup": speedup, + "aggregation": "geometric_mean_case_speedups", + "measurement_protocol": + "shape-matched-different-values-warmup-both-labels-before-paired-timing", + "metrics": {"suite_latency_ms": sum(c["candidate_median_ms"] for c in cases)}} + if summary != score or any(r["cases"] != cases or r["speedup"] != speedup for r in ranks): + raise ValueError("Summary disagrees with raw two-rank samples") + return score + + +def _adjudicate(root: Path, execution: dict, guard: dict, identity: dict, pairs: int) -> dict: + result = {"status": "infrastructure_failure", "completed": False, + "correctness": "INCONCLUSIVE", "speedup": None, "metrics": {}, + "round_consumed": False, "process_cleanup": guard, "execution": execution} + ranks = [] + try: + malformed_ranks = [] + for rank in range(2): + path = root / "results" / f"rank-{rank}.json" + if path.exists(): + try: + record = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(record, dict) or record.get("rank") != rank: + raise ValueError("Rank/file identity mismatch") + except (OSError, ValueError): + malformed_ranks.append(rank) + continue + ranks.append(record) + # A peer may leave a truncated record when the job is stopped. + # Preserve observed candidate work before parsing the next rank. + phase, kind, calls = (record.get(k) for k in ("phase", "failure_kind", "calls")) + if ((isinstance(phase, str) and phase.startswith("candidate_")) + or (isinstance(kind, str) and kind.startswith("candidate_")) + or (isinstance(calls, list) and any( + isinstance(c, dict) and c.get("label") == "candidate" for c in calls))): + result["round_consumed"] = True + result["rank_status"] = [{k: r.get(k) for k in ( + "rank", "phase", "completed", "correctness", "failure_kind", "error")} for r in ranks] + failed = any(r.get("phase", "").startswith("candidate_") + or r.get("failure_kind", "").startswith("candidate_") for r in ranks) + # A later cleanup/integrity failure cannot erase observed candidate work + # and turn a consumed attempt into a free infrastructure retry. + if failed: + result.update(status="candidate_failure", correctness="FAIL", round_consumed=True) + if malformed_ranks: + raise ValueError("Incomplete or malformed rank records; no usable score") + if not guard.get("verified"): + raise ValueError("Workload process cleanup was not verified") + if _hashes(root, identity) != identity: + raise ValueError("Frozen task, grader, request or candidate changed during execution") + if failed: + raise ValueError("Candidate failed or did not complete; inspect retained rank logs") + if (execution.get("status") != "completed" or execution.get("returncode") != 0 + or execution.get("metric_error") or not _positive(execution.get("metric"))): + raise ValueError("Executor did not complete with a valid result; no usable score") + summary = json.loads((root / "results" / "summary.json").read_text(encoding="utf-8")) + task = json.loads((root / "task" / "task.json").read_text(encoding="utf-8")) + score = _score(task, ranks, summary, pairs) + if execution["metric"] != score["metrics"]["suite_latency_ms"]: + raise ValueError("Execution metric disagrees with raw samples") + result.update(score, status="completed", round_consumed=True) + except (OSError, ValueError, TypeError, KeyError, AttributeError, OverflowError) as exc: + result["error"] = str(exc) + return result + + +def evaluate(candidate: Path, output: Path, *, pairs: int, timeout_seconds: float) -> dict: + started = time.monotonic() + candidate, output = candidate.resolve(), output.resolve() + _preflight(candidate, output, pairs, timeout_seconds) + identity = _prepare(candidate, output, pairs) + token = uuid.uuid4().hex + # This small launcher is frozen too and receives the marker only after the + # executor starts it; the calling shell and npu allocator never inherit it. + launcher = output / "runner" / "launch.py" + launcher.write_text("from inferrsi.harness.scenarios.moe import _launch\n" + "from pathlib import Path\nimport sys\n" + "_launch(Path(sys.argv[1]), sys.argv[2])\n", encoding="utf-8") + identity.update(_hashes(output, ["runner/launch.py"])) + atomic_write_json(output / "identity.json", identity) + execution = {"status": "error", "metric": None, "returncode": None} + try: + execution = execute_trial( + (sys.executable, "-B", str(launcher), str(output), token), output, + output / "execution", started + timeout_seconds, "metrics.suite_latency_ms") + except (OSError, ValueError, RuntimeError) as exc: + execution["error"] = str(exc) + finally: + guard = cleanup(token) + result = _adjudicate(output, execution, guard, identity, pairs) + result["elapsed_seconds"] = time.monotonic() - started + result["pairs"] = pairs + result["timeout_seconds"] = timeout_seconds + atomic_write_json(output / "result.json", result) + return result + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--pairs", type=int, required=True) + parser.add_argument("--timeout-seconds", type=float, required=True) + args = parser.parse_args(argv) + try: + result = evaluate(args.candidate, args.output_dir, pairs=args.pairs, + timeout_seconds=args.timeout_seconds) + except (OSError, ValueError) as exc: + parser.exit(1, str(exc) + "\n") + print(json.dumps({k: result.get(k) for k in ( + "status", "correctness", "speedup", "metrics", "elapsed_seconds", "error")})) + return 0 if result["completed"] else 2 if result["status"] == "candidate_failure" else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/inferrsi/harness/scenarios/moe_grader.py b/src/inferrsi/harness/scenarios/moe_grader.py new file mode 100644 index 000000000..99588a0f0 --- /dev/null +++ b/src/inferrsi/harness/scenarios/moe_grader.py @@ -0,0 +1,476 @@ +"""Trusted two-rank correctness and full-forward timer, inside torchrun only.""" + +import argparse +import copy +import datetime as dt +import hashlib +import importlib.util +import json +import math +import os +import statistics +import sys +import time +import traceback +from pathlib import Path + + +def stamp(): + return dt.datetime.now(dt.timezone.utc).isoformat() + + +def write(path, value): + temp = path.with_suffix(path.suffix + ".tmp") + temp.write_text(json.dumps(value, indent=2, ensure_ascii=False), encoding="utf-8") + temp.replace(path) + + +def load(path, name): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if not callable(getattr(module, "compute", None)): + raise ValueError("Candidate must define compute(x, params, *, config, group)") + return module.compute + + +def tensor_snapshot(tensor): + # BF16 is represented as bytes without asking numpy to support its dtype. + cpu = tensor.detach().cpu().contiguous() + return { + "shape": list(tensor.shape), + "dtype": str(tensor.dtype), + "device": str(tensor.device), + "stride": list(tensor.stride()), + "storage_offset": tensor.storage_offset(), + "sha256": hashlib.sha256(cpu.view(torch.uint8).numpy().tobytes()).hexdigest(), + } + + +def snapshots(item): + return { + "x": tensor_snapshot(item["x"]), + "params": {k: tensor_snapshot(v) for k, v in item["params"].items()}, + "config": copy.deepcopy(item["config"]), + } + + +def check_output(output, item, expected, before, tolerance): + spec = before["x"] + shape = ( + isinstance(output, torch.Tensor) + and list(output.shape) == spec["shape"] + and tuple(output.shape) == tuple(expected.shape) + ) + identity = shape and str(output.dtype) == spec["dtype"] and str(output.device) == spec["device"] + result = { + "shape_dtype_device": bool(identity), + "input_parameters_preserved": snapshots(item) == before, + } + if not identity: + result.update(finite=False, values=False, nrmse=None, max_abs_error=None) + return result + actual = output.detach().float().cpu() + finite = bool(torch.isfinite(actual).all()) + diff = actual - expected + denom = max(float(expected.square().mean().sqrt()), 1e-12) if expected.numel() else 1.0 + nrmse = float(diff.square().mean().sqrt()) / denom if expected.numel() else 0.0 + result.update( + finite=finite, + values=finite + and bool(torch.allclose(actual, expected, atol=tolerance["atol"], rtol=tolerance["rtol"])), + nrmse=nrmse, + nrmse_pass=math.isfinite(nrmse) and nrmse <= tolerance["nrmse"], + max_abs_error=float(diff.abs().max()) if diff.numel() else 0.0, + ) + return result + + +def valid(checks): + return all( + checks.get(k) is True + for k in ( + "shape_dtype_device", + "input_parameters_preserved", + "finite", + "values", + "nrmse_pass", + ) + ) + + +def main(): + global torch + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-dir", type=Path, required=True) + args = parser.parse_args() + ROOT = args.run_dir.resolve() + OUT = ROOT / "results" + sys.path.insert(0, str(ROOT / "task")) + rank = int(os.environ["RANK"]) + local_rank = int(os.environ["LOCAL_RANK"]) + record = { + "rank": rank, + "started_at_utc": stamp(), + "completed": False, + "correctness": "INCONCLUSIVE", + "calls": [], + "samples": [], + "phase": "initializing", + } + path = OUT / f"rank-{rank}.json" + write(path, record) + initialized = False + try: + import torch + import torch.distributed as dist + import torch_npu + from oracle import expected as oracle_expected + from oracle import expected_from_input + from torch_npu.npu.utils import get_cann_version + from workload import build_case, cpu_input, load_cases, load_task, warm_input + + torch.set_num_threads(8) + assert int(os.environ["WORLD_SIZE"]) == 2 + assert torch.npu.device_count() == 2 + assert rank in (0, 1) and local_rank in (0, 1) and rank == local_rank + torch.npu.set_device(local_rank) + torch.npu.matmul.allow_hf32 = False + record["phase"] = "hccl_initialization" + write(path, record) + dist.init_process_group("hccl", timeout=dt.timedelta(seconds=150)) + initialized = True + dist.barrier() + torch.npu.synchronize() + request = json.loads((ROOT / "request.json").read_text()) + task = load_task() + cases = load_cases() + tolerance = task["tolerance"] + measure = task["measurement"] + record["runtime"] = { + "torch": torch.__version__, + "torch_npu": torch_npu.__version__, + "cann": str(get_cann_version()), + "python": sys.version.split()[0], + "device": torch.npu.get_device_name(local_rank), + "logical_device": local_rank, + "matmul_allow_hf32": bool(torch.npu.matmul.allow_hf32), + } + record["phase"] = "baseline_import" + write(path, record) + functions = {"baseline": load(ROOT / "task" / "baseline.py", "trusted_baseline")} + record["phase"] = "candidate_import" + write(path, record) + functions["candidate"] = load( + ROOT / "candidate" / "implementation.py", "evaluated_candidate" + ) + + parameter_bundles = {} + configs = {} + expected_cache = {} + prepared_inputs = {} + + def prepare_case(case, case_index): + """Prepare both sides before timing, including all expensive CPU work. + + Computing the oracle just before the first implementation of a pair + creates an asymmetric idle gap. Neither side may pay that gap alone. + """ + started = time.monotonic() + record["phase"] = "trusted_case_preparation" + record["active_case"] = case["id"] + write(path, record) + base_seed = 100000 + case_index * 10000 + seeds = [base_seed, base_seed + 1] + timed_seeds = [] + if case["measured"]: + seeds.extend(base_seed + 10 + i for i in range(measure["warmups"])) + timed_seeds = [ + base_seed + 100 + pair * 100 + sample + for pair in range(request["repetitions"]) + for sample in range(measure["samples_per_pair"]) + ] + seeds.extend(timed_seeds) + # All margin checks, FP32 oracle GEMMs and host tensors are prepared + # before either label's correctness/warmup/measurement starts. + host_inputs = {(seed, "standard"): cpu_input(case, rank, seed) for seed in seeds} + expected_cache.clear() + expected_cache.update( + {(seed, "standard"): oracle_expected(case, seed, rank) for seed in seeds} + ) + warm_started = time.monotonic() + warm_pairs = [] + for seed in timed_seeds: + warmed = warm_input(case, rank, seed) + warm_expected = expected_from_input(case, warmed) + original, original_expected = ( + host_inputs[(seed, "standard")], + expected_cache[(seed, "standard")], + ) + if original.numel(): + assert not torch.equal(original, warmed), ( + "Shape warmup must use different input values" + ) + assert not torch.allclose( + original_expected, + warm_expected, + atol=tolerance["atol"], + rtol=tolerance["rtol"], + ), "Shape warmup must have a different output" + input_delta = (warmed.float() - original.float()).square().mean(dim=1).sqrt() + output_delta = (warm_expected - original_expected).square().mean().sqrt() + output_scale = max(float(original_expected.square().mean().sqrt()), 1e-12) + difference_nrmse = float(output_delta) / output_scale + assert math.isfinite(difference_nrmse) and difference_nrmse > 0.1, ( + "Shape warmup output must prevent replay" + ) + warm_pairs.append( + { + "seed": seed, + "minimum_row_input_rms_difference": float(input_delta.min()), + "output_nrmse_difference": difference_nrmse, + "output_outside_tolerance": True, + } + ) + host_inputs[(seed, "shape_warmup")] = warmed + expected_cache[(seed, "shape_warmup")] = warm_expected + warm_preparation_seconds = time.monotonic() - warm_started + prepared_inputs.clear() + for label in ("baseline", "candidate"): + bundle_key = (case["id"], label) + item = build_case(case, rank, torch.device(f"npu:{local_rank}"), base_seed) + if case["id"] == "weight-update": + previous = parameter_bundles[("unequal-top1", label)] + for name, value in item["params"].items(): + previous[name].copy_(value) + item["params"] = previous + parameter_bundles[bundle_key] = item["params"] + configs[bundle_key] = copy.deepcopy(item["config"]) + for (seed, variant), host_input in host_inputs.items(): + prepared_inputs[(seed, label, variant)] = host_input.to( + f"npu:{local_rank}", copy=True + ) + torch.npu.synchronize() + record.setdefault("case_preparation", []).append( + { + "case_id": case["id"], + "distinct_seeds": len(seeds), + "shape_warmup_inputs": len(timed_seeds), + "shape_warmup_preparation_seconds": warm_preparation_seconds, + "shape_warmup_pairs": warm_pairs, + "elapsed_seconds": time.monotonic() - started, + } + ) + write(path, record) + + def call( + case, seed, label, phase, *, timed=False, pair=None, sample=None, variant="standard" + ): + record["phase"] = "trusted_pre_call_validation" + record["active_case"] = case["id"] + write(path, record) + # Production model parameters stay stable across forward calls. The + # two implementations own separate, identical parameter tensors. + bundle_key = (case["id"], label) + item = { + "x": prepared_inputs[(seed, label, variant)], + "params": parameter_bundles[bundle_key], + "config": copy.deepcopy(configs[bundle_key]), + } + before = snapshots(item) + expected = expected_cache[(seed, variant)] + assert expected.device.type == "cpu" and expected.dtype == torch.float32 + # This barrier is outside the measured interval. synchronize completes + # HCCL/device work before starting the full host-to-device call timer. + record["phase"] = label + "_" + phase + write(path, record) + try: + dist.barrier() + torch.npu.synchronize() + except BaseException: + record["phase"] = "trusted_rank_alignment_failure" + write(path, record) + raise + started = time.perf_counter_ns() + with torch.inference_mode(): + output = functions[label]( + item["x"], item["params"], config=item["config"], group=dist.group.WORLD + ) + torch.npu.synchronize() + local_ms = (time.perf_counter_ns() - started) / 1e6 + record["phase"] = "trusted_output_validation" + write(path, record) + checks = check_output(output, item, expected, before, tolerance) + good = torch.tensor([int(valid(checks))], device=item["x"].device, dtype=torch.int32) + dist.all_reduce(good, op=dist.ReduceOp.MIN) + torch.npu.synchronize() + call_record = { + "case_id": case["id"], + "seed": seed, + "label": label, + "phase": phase, + "variant": variant, + "pair": pair, + "sample": sample, + "checks": checks, + "input_sha256": before["x"]["sha256"], + "all_ranks_correct": bool(good.item()), + "local_elapsed_ms": local_ms, + "included_in_score": timed, + } + record["calls"].append(call_record) + if not call_record["all_ranks_correct"]: + record["correctness"] = "FAIL" if label == "candidate" else "INCONCLUSIVE" + record["failure_kind"] = ( + "candidate_invalid_output" + if label == "candidate" + else "baseline_invalid_output" + ) + write(path, record) + raise ValueError(f"{label} {case['id']} failed independent correctness") + if timed: + elapsed = torch.tensor([local_ms], device=item["x"].device, dtype=torch.float32) + dist.all_reduce(elapsed, op=dist.ReduceOp.MAX) + torch.npu.synchronize() + slowest = float(elapsed.item()) + if not math.isfinite(slowest) or slowest <= 0: + raise ValueError("Invalid completed-forward time") + record["samples"].append( + { + **{ + k: call_record[k] + for k in ("case_id", "seed", "label", "pair", "sample", "input_sha256") + }, + "local_elapsed_ms": local_ms, + "slowest_rank_elapsed_ms": slowest, + } + ) + # Explicitly overwrite returned storage after verification. Retaining + # an output object for replay cannot satisfy a later changed input. + record["phase"] = "trusted_output_reset" + write(path, record) + with torch.inference_mode(): + output.zero_() + torch.npu.synchronize() + write(path, record) + + for case_index, case in enumerate(cases): + prepare_case(case, case_index) + # Each verification and timed input changes. Both implementations use + # exactly the same inputs/weights, constructed independently. + base_seed = 100000 + case_index * 10000 + for variant in range(2): + for label in ("baseline", "candidate"): + call(case, base_seed + variant, label, "correctness") + if not case["measured"]: + continue + for warmup in range(measure["warmups"]): + for label in ("baseline", "candidate"): + call(case, base_seed + 10 + warmup, label, "warmup") + for pair in range(request["repetitions"]): + order = ("baseline", "candidate") if pair % 2 == 0 else ("candidate", "baseline") + for sample in range(measure["samples_per_pair"]): + seed = base_seed + 100 + pair * 100 + sample + # Both implementations see their own shape-matched but + # numerically different input before either timed call. This + # pays dynamic operator preparation independently of whether + # the other implementation happens to use the same operators. + for label in order: + call( + case, + seed, + label, + "shape_warmup", + pair=pair, + sample=sample, + variant="shape_warmup", + ) + for label in order: + call(case, seed, label, "measurement", timed=True, pair=pair, sample=sample) + + record["phase"] = "trusted_summary" + write(path, record) + result_cases = [] + for case in cases: + if not case["measured"]: + continue + grouped = { + label: [ + s["slowest_rank_elapsed_ms"] + for s in record["samples"] + if s["case_id"] == case["id"] and s["label"] == label + ] + for label in ("baseline", "candidate") + } + expected_count = request["repetitions"] * measure["samples_per_pair"] + assert all(len(values) == expected_count for values in grouped.values()) + b, c = (statistics.median(grouped[label]) for label in ("baseline", "candidate")) + result_cases.append( + { + "id": case["id"], + "baseline_median_ms": b, + "candidate_median_ms": c, + "speedup": b / c, + "samples_ms": grouped, + } + ) + speedup = math.exp(statistics.mean(math.log(c["speedup"]) for c in result_cases)) + record.update( + completed=True, + correctness="PASS", + phase="completed", + cases=result_cases, + speedup=speedup, + aggregation="geometric_mean_case_speedups", + measurement_protocol="shape-matched-different-values-warmup-both-labels-before-paired-timing", + shape_warmup_cost={ + label: { + "calls": sum( + c["phase"] == "shape_warmup" and c["label"] == label + for c in record["calls"] + ), + "local_forward_ms": sum( + c["local_elapsed_ms"] + for c in record["calls"] + if c["phase"] == "shape_warmup" and c["label"] == label + ), + } + for label in ("baseline", "candidate") + }, + peak_allocated_bytes=int(torch.npu.max_memory_allocated(local_rank)), + ) + if rank == 0: + summary = { + k: record[k] + for k in ( + "completed", + "correctness", + "cases", + "speedup", + "aggregation", + "measurement_protocol", + ) + } + summary["metrics"] = { + "suite_latency_ms": sum(c["candidate_median_ms"] for c in result_cases) + } + write(OUT / "summary.json", summary) + write(Path(os.environ["INFERRSI_RESULT_PATH"]), summary) + except BaseException as error: + record["error"] = repr(error) + record["traceback"] = traceback.format_exc() + finally: + record["completed_at_utc"] = stamp() + write(path, record) + if initialized: + # Fatal candidate exceptions can leave the peer in a collective. + # torchrun plus the outer bounded driver owns full-job termination. + try: + dist.destroy_process_group() + except BaseException: + pass + return 0 if record["completed"] else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/inferrsi/harness/skill_eval_benchmark.py b/src/inferrsi/harness/skill_eval_benchmark.py new file mode 100644 index 000000000..224a6c7e4 --- /dev/null +++ b/src/inferrsi/harness/skill_eval_benchmark.py @@ -0,0 +1,333 @@ +"""Shared CPU tasks for SkillEval's captured-artifact path. + +The trusted parent owns inputs and the scalar oracle. Submitted source runs only +inside the existing Linux bubblewrap boundary, without the repository or grader. +This adapter grades correctness, never child-reported timing or performance. +""" + +import base64 +import binascii +import hashlib +import json +import os +import signal +import subprocess +import tempfile +from pathlib import Path + +from . import skill_eval_engineering as engineering +from .benchmarks import numpy_task +from .benchmarks.contract import TaskPackage, load_builtin_task + +ADAPTER = "engineering-benchmark-v1" +ARTIFACT = "implementation.py" +MAX_SOURCE_BYTES = engineering.MAX_SOURCE_BYTES +MAX_OUTPUT_BYTES = 32 * 1024 * 1024 +SandboxUnavailable = engineering.SandboxUnavailable +read_regular = engineering.read_regular +SCHEMA = { + "type": "object", "additionalProperties": False, + "required": ["artifact", "summary"], + "properties": { + "artifact": {"type": "string", "enum": [ARTIFACT]}, + "summary": {"type": "string"}, + }, +} +PROMPT = ( + "Read task.json and the assigned knowledge. Change only implementation.py, " + "implementing compute(x, *, eps) under the public shared CPU task contract. " + "Use NumPy, math and self-contained helpers; preserve input contents. " + "Do not change frozen inputs or create other files. No network, installs, " + "other agents, commits or publication. Return artifact='implementation.py' " + "and a concise summary, not a verdict or claimed speedup. The controller " + "retains the actual file and independently checks its behavior in isolation." +) + + +def task_package(name: str) -> TaskPackage: + """The artifact sandbox supports only the packaged NumPy CPU contract.""" + package = load_builtin_task(name) + manifest = package.manifest + if (manifest["adapter"] != "numpy-axis1-v2" + or manifest["environment"] != { + "backend": "cpu", "runtime": "numpy", "dtype": "float32"}): + raise ValueError("Formal benchmark artifacts require the NumPy CPU adapter") + return package + + +def task_descriptor(name: str) -> dict: + package = task_package(name) + manifest = package.manifest + return { + "id": name, + "split": "development", + "request": ( + manifest["description"] + " Implement compute(x, *, eps) in implementation.py. " + "Inputs and output are NumPy float32 arrays; preserve shape and input bytes. " + "Reduce axis 1; RMSNorm workloads include four-dimensional inputs. Use eps for " + "RMSNorm. Public finite inputs include strided layouts, zeros, large and tiny " + "values and singleton reductions. The shared task's base and distinct-per-call " + "input sequence is checked. This SkillEval adapter grades correctness only; " + "no latency, hardware, holdout generalization or skill-benefit claim follows." + ), + "provenance": manifest["source"]["url"], + "contract": manifest, + "benchmark": { + "task_id": name, + "task_sha256": package.task_sha256, + "grader_sha256": package.grader_sha256, + }, + } + + +def validate_task(task): + if not isinstance(task, dict) or not isinstance(task.get("benchmark"), dict): + raise ValueError("Shared benchmark task requires a canonical descriptor") + name = task["benchmark"].get("task_id") + if not isinstance(name, str) or task != task_descriptor(name): + raise ValueError("Shared benchmark descriptor or frozen task identity changed") + + +def starter(task) -> bytes: + validate_task(task) + return (task_package(task["benchmark"]["task_id"]).root / "candidate.py").read_bytes() + + +def input_hashes(workspace: Path) -> dict: + return engineering.input_hashes(workspace, editable=ARTIFACT) + + +def _numpy(): + try: + import numpy as np + except (ImportError, OSError): + raise SandboxUnavailable("Shared benchmark verification requires NumPy") from None + return np + + +# Expected outputs, task packages, and the grader are never mounted in the child. +# All arrays remain alive and compute is loaded once for the complete sequence. +RUNNER = '''import base64, json, platform, resource, sys +resource.setrlimit(resource.RLIMIT_CPU, (10, 10)) +resource.setrlimit(resource.RLIMIT_AS, (1073741824, 1073741824)) +resource.setrlimit(resource.RLIMIT_FSIZE, (33554432, 33554432)) +resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64)) +resource.setrlimit(resource.RLIMIT_NPROC, (0, 0)) +resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +sys.path.insert(0, "/runtime") +import numpy as np +if sys.argv[1:] == ["probe"]: + print(json.dumps({"ready": True, "numpy": np.__version__, + "python": platform.python_version(), "machine": platform.machine()})) +else: + print("INFERRSI_BENCHMARK_READY", file=sys.stderr, flush=True) + request = json.loads(sys.stdin.read(16777217)) + arrays = [] + for record in request["inputs"]: + x = np.frombuffer(base64.b64decode(record["data"], validate=True), + dtype=np.float32).reshape(record["shape"]).copy() + if record["layout"] == "strided": + storage = np.empty((*x.shape[:-1], x.shape[-1] * 2), dtype=np.float32) + view = storage[..., ::2] + view[...] = x + x = view + arrays.append(x) + namespace = {"__name__": "benchmark_candidate", "__file__": "/work/implementation.py"} + with open("/work/implementation.py", encoding="utf-8") as source: + exec(compile(source.read(), "implementation.py", "exec"), namespace) + compute = namespace["compute"] + outputs = [] + for x in arrays: + y = compute(x, eps=request["eps"]) + output = None + if isinstance(y, np.ndarray) and y.shape == x.shape and y.dtype == np.float32: + output = {"shape": list(y.shape), "dtype": str(y.dtype), + "data": base64.b64encode(y.tobytes()).decode("ascii")} + outputs.append({"output": output, + "input_after": base64.b64encode(x.tobytes()).decode("ascii")}) + print(json.dumps({"outputs": outputs}, allow_nan=False)) +''' + + +def sandbox_command(work: Path, *, probe=False): + # Reuse the established system-Python namespaces and read-only mount policy. + command = engineering.sandbox_command(work, probe=probe) + np = _numpy() + + query = ( + "import json,numpy,pathlib; " + "print(json.dumps({'version':numpy.__version__," + "'root':str(pathlib.Path(numpy.__file__).resolve().parent)}))" + ) + try: + runtime = subprocess.run( + ["/usr/bin/python3", "-I", "-c", query], check=True, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + timeout=5, env={"PATH": "/usr/bin:/bin", "LANG": "C"}, + ) + info = json.loads(runtime.stdout) + root = Path(info["root"]) + if (info["version"] != np.__version__ or root.name != "numpy" + or not root.is_absolute() or not root.is_dir()): + raise ValueError("Unsupported NumPy runtime") + except (OSError, subprocess.SubprocessError, ValueError, KeyError, TypeError): + raise SandboxUnavailable("Matching system Python NumPy runtime is required") from None + mounts = ["--ro-bind", str(root), "/runtime/numpy"] + libraries = root.parent / "numpy.libs" + if libraries.is_dir(): + mounts += ["--ro-bind", str(libraries.resolve()), "/runtime/numpy.libs"] + mounts += ["--setenv", "OPENBLAS_NUM_THREADS", "1", "--setenv", "OMP_NUM_THREADS", "1"] + # Create the narrow mount destinations before the final read-only root remount. + position = command.index("--remount-ro") + return command[:position] + mounts + command[position:] + + +def _invoke(work, request=None, *, probe=False): + with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr: + try: + process = subprocess.Popen( + sandbox_command(work, probe=probe), stdin=subprocess.PIPE, + stdout=stdout, stderr=stderr, + env={"PATH": "/usr/bin:/bin", "LANG": "C"}, start_new_session=True, + ) + except OSError: + raise SandboxUnavailable("Cannot start benchmark artifact sandbox") from None + timed_out = False + try: + process.communicate(json.dumps(request).encode(), timeout=15) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + timed_out = True + if not probe: + stderr.seek(0) + if not stderr.read(128).startswith(b"INFERRSI_BENCHMARK_READY\n"): + raise SandboxUnavailable("Benchmark worker did not establish its trusted boundary") + if timed_out: + return None + stdout.seek(0) + raw = stdout.read(MAX_OUTPUT_BYTES + 1) + if process.returncode or len(raw) > MAX_OUTPUT_BYTES: + return None + try: + return json.loads(raw) + except (ValueError, UnicodeError, RecursionError): + return None + + +def _prepare(package, np): + manifest = package.manifest + oracle, measurement = manifest["oracle"], manifest["measurement"] + prepared, base, sequence = [], [], [] + for case in manifest["workloads"]: + x = numpy_task.inputs(case, np) + record = numpy_task.input_record(case, x) + base.append(record) + prepared.append((case, x, numpy_task.reference(x, oracle, np), + {**record, "phase": "precheck"}, case["id"])) + seen = {record["input_sha256"] for record in base} + for case in manifest["workloads"]: + if case["id"] not in measurement["case_ids"]: + continue + for index in range(measurement["warmups"] + measurement["samples"]): + x = numpy_task.inputs(case, np, sequence=index + 1) + record = numpy_task.input_record(case, x) + if record["input_sha256"] in seen: + raise ValueError("Input sequence repeats earlier bytes") + seen.add(record["input_sha256"]) + record.update(call_index=index, + phase="warmup" if index < measurement["warmups"] else "measurement") + sequence.append(record) + prepared.append((case, x, numpy_task.reference(x, oracle, np), record, + f"{case['id']}.call-{index}")) + return prepared, base, sequence + + +def _array(value, original, np): + if (not isinstance(value, dict) or set(value) != {"shape", "dtype", "data"} + or value["shape"] != list(original.shape) or value["dtype"] != "float32" + or not isinstance(value["data"], str)): + return None + try: + raw = base64.b64decode(value["data"], validate=True) + except (ValueError, binascii.Error): + return None + if len(raw) != original.nbytes: + return None + return np.frombuffer(raw, dtype=np.float32).reshape(original.shape) + + +def verify(task, output): + validate_task(task) + valid = (isinstance(output, dict) + and set(output) == {"artifact", "summary", "artifact_source"} + and output["artifact"] == ARTIFACT + and isinstance(output["summary"], str) + and isinstance(output["artifact_source"], str)) + if not valid: + return {"passed": False, "assertions": {"artifact_schema": False}} + try: + source = output["artifact_source"].encode("utf-8") + except UnicodeError: + source = b"" + if not 0 < len(source) <= MAX_SOURCE_BYTES: + return {"passed": False, "assertions": {"artifact_size": False}} + np = _numpy() + + package = task_package(task["benchmark"]["task_id"]) + prepared, base, sequence = _prepare(package, np) + request = { + "eps": package.manifest["oracle"]["eps"], + "inputs": [{"shape": list(x.shape), "layout": case["layout"], + "data": base64.b64encode(x.tobytes()).decode("ascii")} + for case, x, _, _, _ in prepared], + } + with tempfile.TemporaryDirectory(prefix="skill-benchmark-") as temporary: + work = Path(temporary) + (work / "runner.py").write_text(RUNNER, encoding="utf-8") + (work / ARTIFACT).write_bytes(source) + runtime = _invoke(work, probe=True) + if (not isinstance(runtime, dict) + or set(runtime) != {"ready", "numpy", "python", "machine"} + or runtime["ready"] is not True or runtime["numpy"] != np.__version__ + or any(not isinstance(runtime[k], str) or not runtime[k] + for k in ("python", "machine"))): + raise SandboxUnavailable("Benchmark sandbox preflight failed; no host fallback") + actual = _invoke(work, request) + package.assert_unchanged() + receipt = {"identity": task["benchmark"], "workloads": base, "input_sequence": sequence, + "artifact_sha256": hashlib.sha256(source).hexdigest(), + "runtime": {k: runtime[k] for k in ("numpy", "python", "machine")}, + "returned_calls": [], "scope": "isolated-artifact-correctness-only", + "timing": "not measured", "public_inputs": True, + "coverage": ( + "returned_calls indexes well-formed response rows, not independently " + "observed invocations; a crash may leave no rows. Isolation protects " + "the host and keeps the oracle outside, but does not prevent runtime " + "protocol spoofing by submitted source or public-suite overfitting." + )} + if (not isinstance(actual, dict) or set(actual) != {"outputs"} + or not isinstance(actual["outputs"], list) + or len(actual["outputs"]) != len(prepared)): + return {"passed": False, "assertions": {"artifact_response": False}, "benchmark": receipt} + checks = {} + for (case, original, expected, record, prefix), result in zip( + prepared, actual["outputs"], strict=True + ): + if not isinstance(result, dict) or set(result) != {"output", "input_after"}: + checks[prefix + ".response"] = False + continue + received = _array(result["output"], original, np) + after = _array({"shape": list(original.shape), "dtype": "float32", + "data": result["input_after"]}, original, np) + assertions = numpy_task.check_output( + received, original if after is None else after, original, expected, + package.manifest["oracle"], np, + ) + assertions["input_preserved"] = after is not None and assertions["input_preserved"] + checks.update({prefix + "." + key: value for key, value in assertions.items()}) + receipt["returned_calls"].append(record) + return {"passed": all(checks.values()), "assertions": checks, "benchmark": receipt} diff --git a/src/inferrsi/harness/skill_eval_engineering.py b/src/inferrsi/harness/skill_eval_engineering.py index 3fdb79904..218075ab6 100644 --- a/src/inferrsi/harness/skill_eval_engineering.py +++ b/src/inferrsi/harness/skill_eval_engineering.py @@ -171,7 +171,7 @@ def read_regular(path: Path, limit: int) -> bytes: return data -def input_hashes(workspace: Path) -> dict: +def input_hashes(workspace: Path, *, editable: str = "parser.py") -> dict: """Frozen engineering inputs only; special nodes are never opened.""" if any(path.is_symlink() for path in (workspace.absolute(), *workspace.absolute().parents)): raise ValueError("Workspace paths must not contain symlinks") @@ -181,7 +181,7 @@ def input_hashes(workspace: Path) -> dict: mode = path.lstat().st_mode if stat.S_ISLNK(mode) or not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)): raise ValueError("Workspace contains a symlink or special node") - if stat.S_ISREG(mode) and name != "parser.py": + if stat.S_ISREG(mode) and name != editable: hashes[name] = hashlib.sha256(read_regular(path, 16 * 1024 * 1024)).hexdigest() return hashes diff --git a/src/inferrsi/harness/skill_evaluation.py b/src/inferrsi/harness/skill_evaluation.py index 27ce1ed3e..7ca4da9f3 100644 --- a/src/inferrsi/harness/skill_evaluation.py +++ b/src/inferrsi/harness/skill_evaluation.py @@ -190,12 +190,16 @@ def run_trial( timeout_seconds: float | None = None, engineering_artifact: bool = False, generation_only: bool = False, + artifact_adapter=None, ) -> dict: - if generation_only and not engineering_artifact: + adapter = artifact_adapter if artifact_adapter is not None else engineering + is_artifact = engineering_artifact or artifact_adapter is not None + artifact_name = getattr(adapter, "ARTIFACT", "parser.py") + if generation_only and (not is_artifact or adapter is not engineering): raise ValueError("Generation-only requires the engineering artifact adapter") trial_dir.mkdir(parents=True) final = trial_dir / "answer.json" - before = engineering.input_hashes(workspace) if engineering_artifact else { + before = adapter.input_hashes(workspace) if is_artifact else { p.relative_to(workspace).as_posix(): sha256(p) for p in workspace.rglob("*") if p.is_file() } @@ -238,7 +242,7 @@ def run_trial( try: answer_limit = 1048576 if generation_only else 65536 answer_bytes = (engineering.read_regular(final, answer_limit) - if engineering_artifact + if is_artifact else final.read_bytes()) output = json.loads(answer_bytes.decode("utf-8")) except (ValueError, UnicodeError): @@ -248,25 +252,25 @@ def run_trial( output = engineering.materialize_generated_artifact(workspace, output) except (ValueError, OSError): evaluation_error = "invalid_generated_artifact" - if evaluation_error is None and engineering_artifact: - artifact = workspace / "parser.py" + if evaluation_error is None and is_artifact: + artifact = workspace / artifact_name if not isinstance(output, dict) or set(output) != {"artifact", "summary"}: evaluation_error = "invalid_artifact_answer" else: try: - artifact_bytes = engineering.read_regular(artifact, engineering.MAX_SOURCE_BYTES) + artifact_bytes = engineering.read_regular(artifact, adapter.MAX_SOURCE_BYTES) source = artifact_bytes.decode("utf-8") if any(path.is_symlink() for path in (trial_dir, *trial_dir.absolute().parents)): raise ValueError("Retained directory contains a symlink") if any((trial_dir / name).exists() or (trial_dir / name).is_symlink() - for name in ("agent-answer.json", "parser.py", "verification")): + for name in ("agent-answer.json", artifact_name, "verification")): raise ValueError("Agent created reserved evidence paths") except (ValueError, UnicodeError): evaluation_error = "missing_or_invalid_artifact" else: # A single bounded snapshot supplies both retained bytes and grading. (trial_dir / "agent-answer.json").write_bytes(answer_bytes) - (trial_dir / "parser.py").write_bytes(artifact_bytes) + (trial_dir / artifact_name).write_bytes(artifact_bytes) output = {**output, "artifact_source": source} atomic_write_json(final, output) verification_dir = trial_dir / "verification" @@ -279,14 +283,19 @@ def run_trial( evaluation_error = "artifact_verification_deferred" else: try: - checks = engineering.verify(task, output) - except engineering.SandboxUnavailable: + raw_checks = adapter.verify(task, output) + except adapter.SandboxUnavailable: evaluation_error = "artifact_sandbox_unavailable" else: - atomic_write_json(verification_dir / "result.json", checks) - verification = {"status": "completed" if checks["passed"] else "failed", - "returncode": 0 if checks["passed"] else 1} - if evaluation_error is None and not engineering_artifact: + atomic_write_json(verification_dir / "result.json", raw_checks) + proposed = {key: raw_checks.get(key) for key in ("passed", "assertions")} + if not valid_checks(proposed): + evaluation_error = "invalid_verifier_result" + else: + checks = proposed + verification = {"status": "completed" if checks["passed"] else "failed", + "returncode": 0 if checks["passed"] else 1} + if evaluation_error is None and not is_artifact: verifier_environment = getattr(client, "verifier_environment", None) options = {"environment": verifier_environment()} if verifier_environment else {} verification = execute_command( @@ -315,7 +324,7 @@ def run_trial( if verification["returncode"] != (0 if checks["passed"] else 1): evaluation_error = "verifier_exit" try: - after = engineering.input_hashes(workspace) if engineering_artifact else { + after = adapter.input_hashes(workspace) if is_artifact else { p.relative_to(workspace).as_posix(): sha256(p) for p in workspace.rglob("*") if p.is_file() } @@ -343,11 +352,11 @@ def run_trial( "input_hashes": before, "output_sha256": sha256(final) if final.is_file() and not final.is_symlink() else None, } - if engineering_artifact: - retained = trial_dir / "parser.py" + if is_artifact: + retained = trial_dir / artifact_name record["artifact_sha256"] = (sha256(retained) if retained.is_file() and not retained.is_symlink() else None) - record["artifact_kind"] = engineering.ADAPTER + record["artifact_kind"] = adapter.ADAPTER if generation_only: record["execution_mode"] = engineering.GENERATION_MODE atomic_write_json(trial_dir / "result.json", record) diff --git a/tests/check_installed.py b/tests/check_installed.py index d412f14ce..a67928436 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -9,11 +9,25 @@ import inferrsi from inferrsi.catalog import knowledge_records, repository_pack, search_knowledge from inferrsi.cli import main +from inferrsi.harness.benchmarks import builtin_tasks, load_task +from inferrsi.harness.scenarios.moe import TASK_FILES, task_resources from inferrsi.knowledge_operations import KnowledgeReader assert Path(inferrsi.__file__).resolve().parent.parent.name != "src", ( "This check requires an installed package" ) +assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-npu"} +assert "torch" not in sys.modules and "torch_npu" not in sys.modules +for resource_name in TASK_FILES: + assert task_resources().joinpath(resource_name).is_file(), resource_name +assert files("inferrsi.harness.scenarios").joinpath("moe_grader.py").is_file() +for benchmark_id in builtin_tasks(): + benchmark = load_task(benchmark_id) + benchmark.assert_unchanged() + assert (benchmark.root / "baseline.py").is_file() + assert (benchmark.root / "candidate.py").is_file() + assert (benchmark.root / "LICENSE").is_file() + records = knowledge_records() assert len(records) == 127, "Knowledge is missing from the installation" assert ( diff --git a/tests/test_engineering_benchmarks.py b/tests/test_engineering_benchmarks.py new file mode 100644 index 000000000..97dc1f11b --- /dev/null +++ b/tests/test_engineering_benchmarks.py @@ -0,0 +1,445 @@ +"""Real payload checks and consumer regressions; no model/device calls.""" + +import json +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from inferrsi.harness.benchmarks import builtin_tasks, load_task, skill_verifier +from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action +from inferrsi.harness.benchmarks.contract import load_builtin_task +from inferrsi.harness.runner import run_experiment +from inferrsi.harness.scenarios.external import execute_trial +from inferrsi.harness.skill_evaluation import run_trial + + +@pytest.fixture +def numpy_runtime(): + return pytest.importorskip("numpy") + + +def read_result(execution): + return json.loads(Path(execution["result_path"]).read_text(encoding="utf-8")) + + +def test_real_baseline_candidate_measurement_and_profile(tmp_path, numpy_runtime): + task = load_task("softmax-cpu") + checked = [] + measurement = task.manifest["measurement"] + expected_calls = len(task.manifest["workloads"]) + len(measurement["case_ids"]) * ( + measurement["warmups"] + measurement["samples"] + ) + for role in ("baseline", "candidate"): + candidate = task.root / (role + ".py") + grade = grade_artifact(task, candidate, tmp_path / role, timeout_seconds=20) + assert grade["verdict"] == "PASS" + assert len(grade["checks"]["assertions"]) == 4 * expected_calls + checked.append(json.loads((tmp_path / role / "result.json").read_text())) + candidate = task.root / "candidate.py" + measured = read_result( + run_action(task, candidate, "measure", tmp_path / "timed", timeout_seconds=20) + ) + assert measured["passed"] + assert measured["metrics"]["suite_latency_ms"] > 0 + assert [case["id"] for case in measured["cases"]] == measurement["case_ids"] + assert all(len(c["samples_ms"]) == 5 for c in measured["cases"]) + profile = read_result( + run_action(task, candidate, "profile", tmp_path / "profile", timeout_seconds=20) + ) + assert "metrics" not in profile + assert profile["diagnostic"]["performance_verdict"] is None + assert "compute" in (tmp_path / "profile/profile.txt").read_text() + for result in [*checked, profile, measured]: + assert result["workloads"] == measured["workloads"] + assert result["input_sequence"] == measured["input_sequence"] + assert result["executed_calls"] == measured["executed_calls"] + assert len(result["executed_calls"]) == expected_calls + hashes = [call["input_sha256"] for call in result["executed_calls"]] + assert len(hashes) == len(set(hashes)) + assert "zeros" not in measurement["case_ids"] + assert any(call["id"] == "zeros" for call in measured["executed_calls"]) + for case in measured["cases"]: + assert case["sample_input_sha256"] == [ + call["input_sha256"] + for call in measured["input_sequence"] + if call["id"] == case["id"] and call["phase"] == "measurement" + ] + + +@pytest.mark.parametrize("key", ["id(x)", "x.tobytes()"]) +def test_timed_calls_cannot_hit_precheck_or_warmup_memo(key, tmp_path, numpy_runtime): + task = load_task("softmax-cpu") + candidate = tmp_path / "implementation.py" + operation = "y = np.exp(x-x.max(axis=1,keepdims=True)); y /= y.sum(axis=1,keepdims=True)" + candidate.write_text( + "import numpy as np\nfrom pathlib import Path\ncache = {}\n" + "def compute(x, *, eps):\n" + f" key = {key}\n" + " if key in cache:\n" + " Path(__file__).with_suffix('.hits').write_text('cache hit')\n" + " return cache[key]\n" + f" {operation}\n" + " cache[key] = y\n" + " return y\n" + ) + execution = run_action(task, candidate, "measure", tmp_path / "run", timeout_seconds=20) + result = read_result(execution) + assert execution["status"] == "completed" and result["passed"] + assert result["metrics"]["suite_latency_ms"] > 0 + # Correct memoization remains valid, but every timed call must be a miss. + assert not candidate.with_suffix(".hits").exists() + + +def test_shape_cached_output_replay_is_rejected_after_precheck(tmp_path, numpy_runtime): + task = load_task("softmax-cpu") + candidate = tmp_path / "implementation.py" + operation = "y = np.exp(x-x.max(axis=1,keepdims=True)); y /= y.sum(axis=1,keepdims=True)" + candidate.write_text( + "import numpy as np\ncache = {}\ndef compute(x, *, eps):\n" + " key = (x.shape, x.strides)\n" + " if key in cache: return cache[key]\n" + f" {operation}\n" + " cache[key] = y\n" + " return y\n" + ) + execution = run_action(task, candidate, "measure", tmp_path / "run", timeout_seconds=20) + result = read_result(execution) + assert execution["status"] == "failed" and not result["passed"] + assert "metrics" not in result + assert result["failed_call"]["case"] == "normal" + assert result["failed_call"]["phase"] == "warmup" + assert all(value for key, value in result["assertions"].items() if ".call-" not in key) + + +@pytest.mark.parametrize( + "body", + [ + "return np.zeros_like(x)", + "return np.full_like(x, np.nan)", + "return np.ones_like(x, dtype=np.float64)", + "return np.ones((1,), dtype=np.float32)", + "y = np.exp(x); return y / y.sum(axis=1, keepdims=True)", + "x[:] = 0; return np.full_like(x, 1 / x.shape[1])", + "y = np.exp(x - x.max(axis=0, keepdims=True)); return y / y.sum(axis=0, keepdims=True)", + ], +) +def test_bad_candidates_cannot_supply_performance(body, tmp_path, numpy_runtime): + candidate = tmp_path / "implementation.py" + candidate.write_text("import numpy as np\ndef compute(x, *, eps):\n " + body + "\n") + task = load_task("softmax-cpu") + grade = grade_artifact(task, candidate, tmp_path / "grade", timeout_seconds=20) + assert grade["verdict"] == "FAIL" + execution = run_action(task, candidate, "measure", tmp_path / "measure", timeout_seconds=20) + assert execution["status"] == "failed" + assert "metrics" not in read_result(execution) + + +def test_checks_repeated_timed_calls_not_only_initial_call(tmp_path, numpy_runtime): + candidate = tmp_path / "implementation.py" + candidate.write_text( + "import numpy as np\nn = 0\ndef compute(x, *, eps):\n" + " global n\n n += 1\n" + " if n > 6: return np.zeros_like(x)\n" + " y = np.exp(x-x.max(axis=1,keepdims=True))\n" + " return y/y.sum(axis=1,keepdims=True)\n" + ) + execution = run_action( + load_task("softmax-cpu"), candidate, "measure", tmp_path / "run", timeout_seconds=20 + ) + assert execution["status"] == "failed" + result = read_result(execution) + assert not result["passed"] and "metrics" not in result + + +def test_frozen_package_and_candidate_drift(tmp_path, numpy_runtime): + root = tmp_path / "package" + shutil.copytree(load_task("softmax-cpu").root, root) + task = load_task(root) + candidate = root / "candidate.py" + command = task.command("check", candidate) + candidate.write_text(candidate.read_text() + "\n# changed\n") + with pytest.raises(ValueError, match="changed"): + task.command("check", candidate) + execution = execute_trial(command, root, tmp_path / "run", time.monotonic() + 20, None) + assert execution["status"] == "failed" + assert read_result(execution)["status"] == "error" + + +@pytest.mark.parametrize( + "field,value", + [ + ("schema_version", 2), + ("schema_version", True), + ("adapter", "unknown"), + ("adapter", "numpy-axis1-v1"), + ("environment", {"backend": "cuda", "runtime": "numpy", "dtype": "float32"}), + ("coverage", []), + ("workloads", []), + ], +) +def test_unsupported_contract_rejected(field, value, tmp_path): + root = tmp_path / "package" + shutil.copytree(load_task("softmax-cpu").root, root) + document = json.loads((root / "task.json").read_text(encoding="utf-8")) + document[field] = value + (root / "task.json").write_text(json.dumps(document)) + with pytest.raises(ValueError): + load_task(root) + + +def test_descriptor_does_not_mutate_task(): + task = load_task("softmax-cpu") + manifest = task.manifest + manifest["oracle"]["axis"] = 0 + assert task.manifest["oracle"]["axis"] == 1 + assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-npu"} + with pytest.raises(ValueError, match="Unknown built-in task package"): + load_builtin_task("rmsnorm-cpu") + + +@pytest.mark.parametrize( + "field,value", + [ + ("input_sequence", "repeated"), + ("case_ids", []), + ("case_ids", ["missing"]), + ("case_ids", ["normal", "normal"]), + ("case_ids", ["normal", "zeros"]), + ("samples", 1000), + ], +) +def test_invalid_input_sequence_contract_rejected(field, value, tmp_path): + root = tmp_path / "package" + shutil.copytree(load_task("softmax-cpu").root, root) + document = json.loads((root / "task.json").read_text()) + document["measurement"][field] = value + (root / "task.json").write_text(json.dumps(document)) + with pytest.raises(ValueError): + load_task(root) + + +def test_duplicate_generated_input_bytes_refuse_measurement(tmp_path, numpy_runtime): + root = tmp_path / "package" + shutil.copytree(load_task("softmax-cpu").root, root) + document = json.loads((root / "task.json").read_text()) + # Same values under different layouts still prime a bytes-keyed memo. + document["workloads"][1]["seed"] = document["workloads"][0]["seed"] + (root / "task.json").write_text(json.dumps(document)) + task = load_task(root) + execution = run_action( + task, root / "candidate.py", "measure", tmp_path / "run", timeout_seconds=20 + ) + result = read_result(execution) + assert execution["status"] == "failed" + assert result["status"] == "error" and "metrics" not in result + assert result["error_type"] == "ValueError" + + +def test_real_workbench_reuses_frozen_task(tmp_path, numpy_runtime): + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True) + task = load_task("softmax-cpu") + candidate = repo / "implementation.py" + shutil.copyfile(task.root / "candidate.py", candidate) + subprocess.run(["git", "-C", str(repo), "add", "."], check=True) + subprocess.run( + [ + "git", + "-C", + str(repo), + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-qm", + "fixture", + ], + check=True, + ) + spec = task.runspec( + candidate, + repo=repo, + output_dir=tmp_path / "runs", + repetitions=2, + max_minutes=1, + min_improvement_pct=0, + ) + run = run_experiment(spec, {"scope": "CPU software integration"}) + evaluation = json.loads((run / "evaluation.json").read_text()) + assert evaluation["source_integrity"]["verified"] + assert evaluation["baseline"]["excluded_trials"] == 0 + assert evaluation["candidate"]["excluded_trials"] == 0 + assert evaluation["statistical_significance"] == "not established" + events = [json.loads(line) for line in (run / "events.jsonl").read_text().splitlines()] + assert [e["side"] for e in events if e["phase"] == "measurement"] == [ + "baseline", + "candidate", + "candidate", + "baseline", + ] + identity = json.loads((run / "candidate/trial-001/result.json").read_text())["identity"] + assert identity == task.identity(candidate) + baseline = json.loads((run / "baseline/trial-001/result.json").read_text()) + for side in ("baseline", "candidate"): + for path in (run / side).glob("trial-*/result.json"): + result = json.loads(path.read_text()) + assert result["input_sequence"] == baseline["input_sequence"] + assert result["executed_calls"] == baseline["executed_calls"] + + +@pytest.mark.parametrize( + "case,expected_code,expected_state,expected_verdict", + [ + ("unmet-gain", 1, "completed", "FAIL"), + ("wrong-output", 2, "failed", "INCONCLUSIVE"), + ("timeout", 2, "timeout", "INCONCLUSIVE"), + ], +) +def test_compare_cli_reports_retained_workbench_outcome( + case, expected_code, expected_state, expected_verdict, tmp_path, numpy_runtime, capsys +): + from inferrsi.cli import main + + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True) + candidate = repo / "implementation.py" + candidate.write_text( + "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n" + if case == "wrong-output" + else (load_task("softmax-cpu").root / "candidate.py").read_text(), + encoding="utf-8", + ) + subprocess.run(["git", "-C", str(repo), "add", "."], check=True) + subprocess.run( + [ + "git", "-C", str(repo), "-c", "user.name=Fixture", "-c", + "user.email=fixture@example.invalid", "commit", "-qm", "fixture", + ], + check=True, + ) + code = main( + [ + "benchmarks", "compare", "softmax-cpu", "--candidate", str(candidate), + "--repo", str(repo), "--output-dir", str(tmp_path / "runs"), + "--repetitions", "1", "--max-minutes", "1e-10" if case == "timeout" else "1", + # A complete positive-latency comparison cannot reach a 100% reduction. + "--min-improvement-pct", "100", + ] + ) + response = json.loads(capsys.readouterr().out) + run_dir = Path(response["run_dir"]) + retained = json.loads((run_dir / "status.json").read_text()) + evaluation = json.loads((run_dir / "evaluation.json").read_text()) + assert response["status"] == retained + assert retained["state"] == expected_state + assert retained["verdict"] == evaluation["verdict"] == expected_verdict + assert code == expected_code + + +class RetainedControl: + """Local authored control for the real trial machinery; no model or token usage.""" + + timeout_seconds = 20 + + def __init__(self, identity): + self.identity = identity + + def command(self, workspace, final, schema, prompt): + script = ( + "import json, pathlib, sys; " + "pathlib.Path(sys.argv[1]).write_text(json.dumps({'candidate_sha256':sys.argv[2]})); " + "print(json.dumps({'type':'turn.completed'}))" + ) + return sys.executable, "-B", "-c", script, str(final), self.identity + + +@pytest.mark.parametrize("good", [True, False]) +def test_actual_skilleval_trial_uses_same_oracle(good, tmp_path, numpy_runtime): + package = load_task("softmax-cpu") + candidate = tmp_path / "implementation.py" + candidate.write_text( + (package.root / "candidate.py").read_text() + if good + else "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n" + ) + task = skill_verifier.task_descriptor(package, candidate) + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "task.json").write_text(json.dumps(task)) + client = RetainedControl(task["benchmark"]["identity"]["candidate_sha256"]) + record = run_trial( + client, + task, + "N", + 1, + workspace, + tmp_path / "trial", + tmp_path / "unused-schema.json", + Path(skill_verifier.__file__), + "Offline authored control", + time.monotonic() + 30, + ) + assert record["verdict"] == ("PASS" if good else "FAIL") + assert record["inputs_unchanged"] + assert record["observations"]["usage"] is None + raw = json.loads((tmp_path / "trial/verification/artifact/result.json").read_text()) + assert raw["identity"] == task["benchmark"]["identity"] + + +def test_missing_numpy_has_no_fallback(tmp_path): + # An import blocker exercises dependency failure in a real worker process. + blocker = tmp_path / "numpy.py" + blocker.write_text("raise ImportError('numpy deliberately unavailable')\n") + task = load_task("softmax-cpu") + result = tmp_path / "result.json" + env = {**os.environ, "PYTHONPATH": str(tmp_path), "INFERRSI_RESULT_PATH": str(result)} + process = subprocess.run( + task.command("measure", task.root / "candidate.py"), + env=env, + cwd=tmp_path, + capture_output=True, + ) + assert process.returncode == 1 + document = json.loads(result.read_text()) + assert document["status"] == "error" and "metrics" not in document + + +def test_timeout_is_incomplete_and_retains_execution(tmp_path, numpy_runtime): + candidate = tmp_path / "implementation.py" + candidate.write_text("import time\ndef compute(x, *, eps):\n time.sleep(10)\n") + result = grade_artifact( + load_task("softmax-cpu"), candidate, tmp_path / "timeout", timeout_seconds=0.5 + ) + assert result["verdict"] == "INCONCLUSIVE" + assert result["execution_status"] == "timeout" + assert result["checks"] is None + assert (tmp_path / "timeout/execution.json").is_file() + + +def test_runtime_and_grader_drift_rejected(tmp_path, numpy_runtime): + task = load_task("softmax-cpu") + for field in ("--runtime-sha256", "--grader-sha256", "--candidate-sha256"): + command = list(task.command("check", task.root / "candidate.py")) + command[command.index(field) + 1] = "0" * 64 + execution = execute_trial( + tuple(command), task.root, tmp_path / field[2:], time.monotonic() + 20, None + ) + assert execution["status"] == "failed" + assert read_result(execution)["status"] == "error" + + +def test_skill_verifier_binds_answer_to_retained_artifact(tmp_path, numpy_runtime): + package = load_task("softmax-cpu") + candidate = package.root / "candidate.py" + descriptor = skill_verifier.task_descriptor(package, candidate) + result = skill_verifier.verify(descriptor, {"candidate_sha256": "0" * 64}, tmp_path / "unused") + assert result == {"passed": False, "assertions": {"artifact_identity": False}} + assert not (tmp_path / "unused").exists() diff --git a/tests/test_evidence_inventory.py b/tests/test_evidence_inventory.py index 9edd30863..25b751ef7 100644 --- a/tests/test_evidence_inventory.py +++ b/tests/test_evidence_inventory.py @@ -145,6 +145,6 @@ def test_checked_in_inventory(): repo = Path(__file__).resolve().parents[1] result = validate_inventory(repo) assert result["errors"] == [] - assert len(result["bundles"]) == 21 - assert result["checked_hashes"] == 430 + assert len(result["bundles"]) == 22 + assert result["checked_hashes"] == 460 assert result["unavailable"] # Historical raw artifacts are not in this checkout. diff --git a/tests/test_moe_benchmark.py b/tests/test_moe_benchmark.py new file mode 100644 index 000000000..502c8e0d1 --- /dev/null +++ b/tests/test_moe_benchmark.py @@ -0,0 +1,343 @@ +"""No-device contract tests; these do not claim NPU performance acceptance.""" + +import copy +import errno +import json +import os +import subprocess +import sys +import time +import uuid +from pathlib import Path + +import pytest + +from inferrsi.harness.scenarios import _moe_processes, moe +from inferrsi.harness.scenarios.external import execute_trial + + +def _write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding="utf-8") + + +@pytest.fixture +def fixture_run(tmp_path): + """A hand-authored two-case transcript with one timed sample per label. + + Rank 0 takes 10/5ms and rank 1 takes 12/6ms. The expected score is 2, + and suite latency is 6, using rank MAX rather than rank-0 latency. + """ + task = {"cases": [{"id": "timed", "measured": True}, + {"id": "edge", "measured": False}], + "measurement": {"warmups": 0, "samples_per_pair": 1}} + _write(tmp_path / "task/task.json", task) + cases = [{"id": "timed", "baseline_median_ms": 12.0, "candidate_median_ms": 6.0, + "speedup": 2.0, "samples_ms": {"baseline": [12.0], "candidate": [6.0]}}] + summary = {"completed": True, "correctness": "PASS", "cases": cases, "speedup": 2.0, + "aggregation": "geometric_mean_case_speedups", + "measurement_protocol": + "shape-matched-different-values-warmup-both-labels-before-paired-timing", + "metrics": {"suite_latency_ms": 6.0}} + ranks = [] + for rank in (0, 1): + calls = [] + for case, seed in (("timed", 100000), ("timed", 100001), + ("edge", 110000), ("edge", 110001)): + for label in ("baseline", "candidate"): + calls.append({"case_id": case, "seed": seed, "label": label, + "phase": "correctness", "variant": "standard", + "pair": None, "sample": None, "included_in_score": False}) + for phase, variant, scored in (("shape_warmup", "shape_warmup", False), + ("measurement", "standard", True)): + for label in ("baseline", "candidate"): + calls.append({"case_id": "timed", "seed": 100100, "label": label, + "phase": phase, "variant": variant, "pair": 0, "sample": 0, + "included_in_score": scored}) + samples = [] + for call in calls: + latency = (10.0 + 2 * rank) / (2 if call["label"] == "candidate" else 1) + call.update(all_ranks_correct=True, local_elapsed_ms=latency, + input_sha256=f"rank{rank}-{call['seed']}-{call['variant']}", + checks={k: True for k in ("shape_dtype_device", "finite", "values", + "nrmse_pass", "input_parameters_preserved")}) + if call["included_in_score"]: + sample = {k: call[k] for k in ("case_id", "seed", "label", "pair", "sample", + "local_elapsed_ms", "input_sha256")} + sample["slowest_rank_elapsed_ms"] = 6.0 if call["label"] == "candidate" else 12.0 + samples.append(sample) + record = {"rank": rank, "completed": True, "correctness": "PASS", "phase": "completed", + "calls": calls, "samples": samples, "cases": cases, "speedup": 2.0} + ranks.append(record) + _write(tmp_path / f"results/rank-{rank}.json", record) + _write(tmp_path / "results/summary.json", summary) + execution = {"status": "completed", "returncode": 0, "metric": 6.0, "metric_error": None} + identity = moe._hashes(tmp_path, ["task/task.json"]) + return tmp_path, task, ranks, summary, execution, identity + + +def _adjudicate(fixture_run, **changes): + root, _, _, _, execution, identity = fixture_run + execution = {**execution, **changes} + return moe._adjudicate(root, execution, {"verified": True}, identity, 1) + + +def test_complete_two_rank_result_is_recomputed(fixture_run): + root = fixture_run[0] + before = (root / "results/rank-0.json").read_bytes() + result = _adjudicate(fixture_run) + assert result["completed"] and result["correctness"] == "PASS" + assert result["speedup"] == 2.0 and result["metrics"]["suite_latency_ms"] == 6.0 + assert (root / "results/rank-0.json").read_bytes() == before + + +@pytest.mark.parametrize("status,returncode", [("timeout", 0), ("failed", 2), ("cancelled", -9)]) +def test_failed_execution_cannot_reuse_successful_summary(fixture_run, status, returncode): + result = _adjudicate(fixture_run, status=status, returncode=returncode) + assert not result["completed"] and result["correctness"] == "INCONCLUSIVE" + assert result["speedup"] is None and result["metrics"] == {} + + +@pytest.mark.parametrize("change", ["missing_rank", "duplicate_rank", "missing_call", "bad_check", + "duplicate_sample", "wrong_max", "wrong_summary"]) +def test_incomplete_or_inconsistent_evidence_has_no_score(fixture_run, change): + root, _, ranks, summary, _, _ = fixture_run + if change == "missing_rank": + (root / "results/rank-1.json").unlink() + elif change == "wrong_summary": + summary["speedup"] = 99 + _write(root / "results/summary.json", summary) + else: + record = copy.deepcopy(ranks[1]) + if change == "duplicate_rank": + record["rank"] = 0 + elif change == "missing_call": + record["calls"].pop(0) + elif change == "bad_check": + record["calls"][0]["checks"]["finite"] = False + elif change == "duplicate_sample": + record["samples"].append(record["samples"][0]) + elif change == "wrong_max": + record["samples"][0]["slowest_rank_elapsed_ms"] = 10.0 + _write(root / "results/rank-1.json", record) + result = _adjudicate(fixture_run) + assert not result["completed"] and result["speedup"] is None + + +@pytest.mark.parametrize("phase,kind,expected", [ + ("candidate_import", "", "FAIL"), ("candidate_measurement", "", "FAIL"), + ("trusted_output_validation", "candidate_invalid_output", "FAIL"), + ("baseline_import", "", "INCONCLUSIVE"), + ("trusted_output_validation", "baseline_invalid_output", "INCONCLUSIVE"), +]) +def test_candidate_failure_is_not_an_infrastructure_retry(fixture_run, phase, kind, expected): + root, _, ranks, _, _, _ = fixture_run + ranks[0].update(completed=False, calls=[], phase="trusted_rank_alignment_failure") + _write(root / "results/rank-0.json", ranks[0]) + ranks[1].update(completed=False, correctness="INCONCLUSIVE", phase=phase, failure_kind=kind, + calls=[]) + _write(root / "results/rank-1.json", ranks[1]) + result = _adjudicate(fixture_run, status="failed", returncode=2) + assert result["correctness"] == expected and result["speedup"] is None + assert result["round_consumed"] is (expected == "FAIL") + + +def test_cleanup_failure_does_not_erase_consumed_candidate(fixture_run): + root, _, ranks, _, execution, identity = fixture_run + ranks[1].update(completed=False, phase="candidate_import", error="candidate exception") + _write(root / "results/rank-1.json", ranks[1]) + result = moe._adjudicate(root, execution, {"verified": False}, identity, 1) + assert result["round_consumed"] and result["correctness"] == "FAIL" + assert not result["completed"] and result["speedup"] is None + assert "cleanup" in result["error"] + + +@pytest.mark.parametrize("rank", [0, 1]) +def test_malformed_peer_does_not_erase_consumed_candidate(fixture_run, rank): + root = fixture_run[0] + (root / f"results/rank-{rank}.json").write_text("{") + result = _adjudicate(fixture_run, status="timeout", returncode=None) + assert result["round_consumed"] and result["status"] == "infrastructure_failure" + assert not result["completed"] and result["speedup"] is None + + +@pytest.mark.parametrize("status,exit_code", [("infrastructure_failure", 1), + ("candidate_failure", 2)]) +def test_exit_status_is_separate_from_consumed_attempt(monkeypatch, status, exit_code): + monkeypatch.setattr(moe, "evaluate", lambda *args, **kwargs: { + "status": status, "completed": False, "round_consumed": True}) + assert moe.main(["--candidate", "candidate.py", "--output-dir", "new-run", + "--pairs", "2", "--timeout-seconds", "10"]) == exit_code + + +def test_source_drift_and_failed_cleanup_invalidate_score(fixture_run): + root, _, _, _, execution, identity = fixture_run + result = moe._adjudicate(root, execution, {"verified": False}, identity, 1) + assert result["speedup"] is None and not result["completed"] + (root / "task/task.json").write_text("{}") + result = _adjudicate(fixture_run) + assert result["speedup"] is None and "changed" in result["error"] + + +@pytest.mark.parametrize("pairs,timeout", [(0, 10), (-1, 10), (True, 10), (1.5, 10), + (1, 0), (1, -1), (1, float("nan")), + (1, float("inf"))]) +def test_invalid_limits_fail_before_output_creation(tmp_path, pairs, timeout): + candidate = tmp_path / "implementation.py" + candidate.write_text("def compute(*a, **kw): pass\n") + with pytest.raises(ValueError): + moe.evaluate(candidate, tmp_path / "run", pairs=pairs, timeout_seconds=timeout) + assert not (tmp_path / "run").exists() + + +@pytest.mark.parametrize("visible", ["", "0", "0,0", "2,", "0,1,2", "bad,1"]) +def test_bad_device_allocation_rejected_without_launch(tmp_path, monkeypatch, visible): + candidate = tmp_path / "implementation.py" + candidate.write_text("def compute(*a, **kw): pass\n") + monkeypatch.setattr(sys, "platform", "linux") + monkeypatch.setattr(Path, "read_text", lambda *a, **k: "") + monkeypatch.setenv("ASCEND_RT_VISIBLE_DEVICES", visible) + with pytest.raises(ValueError, match="NPUs|VISIBLE"): + moe.evaluate(candidate, tmp_path / "run", pairs=1, timeout_seconds=10) + assert not (tmp_path / "run").exists() + + +def test_existing_output_is_never_overwritten(tmp_path): + candidate = tmp_path / "implementation.py" + candidate.write_text("candidate evidence") + with pytest.raises(FileExistsError): + moe.evaluate(candidate, tmp_path, pairs=1, timeout_seconds=10) + assert candidate.read_text() == "candidate evidence" + + +def test_evaluate_freezes_input_uses_executor_and_retains_failure(tmp_path, monkeypatch): + candidate = tmp_path / "implementation.py" + candidate.write_text("raise RuntimeError('candidate import failed')\n") + output = tmp_path / "run" + monkeypatch.setattr(moe, "_preflight", lambda *args: None) + monkeypatch.setattr(moe, "cleanup", lambda token: {"verified": True}) + before = time.monotonic() + + def execute(command, cwd, trial, deadline, metric): + assert cwd == output and trial == output / "execution" + assert before < deadline <= time.monotonic() + 10 + assert metric == "metrics.suite_latency_ms" + assert command[:2] == (sys.executable, "-B") + identity = json.loads((output / "identity.json").read_text()) + assert moe._hashes(output, identity) == identity + assert (output / "candidate/implementation.py").read_bytes() == candidate.read_bytes() + _write(output / "results/rank-0.json", { + "rank": 0, "phase": "candidate_import", "completed": False, + "correctness": "INCONCLUSIVE", "error": "candidate import failed"}) + return {"status": "failed", "returncode": 2, "metric": None} + + monkeypatch.setattr(moe, "execute_trial", execute) + result = moe.evaluate(candidate, output, pairs=2, timeout_seconds=10) + assert result["status"] == "candidate_failure" and result["round_consumed"] + assert result["speedup"] is None + assert json.loads((output / "result.json").read_text()) == result + + +def test_resources_stage_without_importing_candidate_or_device(tmp_path): + candidate = tmp_path / "implementation.py" + candidate.write_text("raise RuntimeError('must not import while staging')\n") + output = tmp_path / "run" + identity = moe._prepare(candidate, output, 2) + task = json.loads((output / "task/task.json").read_text()) + assert task["world_size"] == 2 and task["num_experts"] == 8 + assert len(task["cases"]) == 7 and sum(c["measured"] for c in task["cases"]) == 3 + assert task["tolerance"] == {"atol": 0.03, "rtol": 0.03, "nrmse": 0.01, "frozen": True} + assert moe._hashes(output, identity) == identity + assert (output / "candidate/implementation.py").read_bytes() == candidate.read_bytes() + assert json.loads((output / "request.json").read_text()) == {"repetitions": 2} + + +def _fake_proc_entry(proc, pid, marker, *, state="S", start_ticks=123): + entry = proc / str(pid) + entry.mkdir(parents=True, exist_ok=True) + (entry / "environ").write_bytes((_moe_processes.MARKER + "=" + marker + "\0").encode()) + fields = [state] + ["0"] * 18 + [str(start_ticks)] + (entry / "stat").write_text(f"{pid} (python) " + " ".join(fields)) + return entry + + +def test_exact_marker_and_zombie_filter(tmp_path, monkeypatch): + monkeypatch.setattr(os, "getuid", lambda: (tmp_path.stat().st_uid), raising=False) + _fake_proc_entry(tmp_path, "self", "", start_ticks=100) + for pid, marker, state in ((111, "token", "S"), (222, "other", "S"), (333, "token", "Z")): + _fake_proc_entry(tmp_path, pid, marker, state=state) + assert _moe_processes._members("token", tmp_path) == [{"pid": 111, "start_ticks": 123}] + + +@pytest.mark.parametrize("start_ticks,verified", [(99, True), (100, False), (101, False)]) +def test_unreadable_environment_only_excludes_strictly_older_process( + tmp_path, monkeypatch, start_ticks, verified +): + monkeypatch.setattr(os, "getuid", lambda: tmp_path.stat().st_uid, raising=False) + _fake_proc_entry(tmp_path, "self", "", start_ticks=100) + entry = _fake_proc_entry(tmp_path, 111, "other", start_ticks=start_ticks) + read_bytes = Path.read_bytes + + def unreadable(path): + if path == entry / "environ": + raise PermissionError(errno.EACCES, "Permission denied", str(path)) + return read_bytes(path) + + members = _moe_processes._members + monkeypatch.setattr(Path, "read_bytes", unreadable) + monkeypatch.setattr(_moe_processes, "_members", lambda token: members(token, tmp_path)) + guard = _moe_processes.cleanup("token", grace_seconds=0) + assert guard["verified"] is verified, guard + assert guard["signalled_pids"] == [] + if not verified: + assert repr(str(entry / "environ")) in guard["error"] + + +def test_same_tick_owned_process_remains_visible(tmp_path, monkeypatch): + monkeypatch.setattr(os, "getuid", lambda: tmp_path.stat().st_uid, raising=False) + _fake_proc_entry(tmp_path, "self", "", start_ticks=100) + _fake_proc_entry(tmp_path, 111, "token", start_ticks=100) + assert _moe_processes._members("token", tmp_path) == [{"pid": 111, "start_ticks": 100}] + + +def test_reused_pid_is_not_signalled(monkeypatch): + scans = iter([[{"pid": 111, "start_ticks": 123}], []]) + monkeypatch.setattr(_moe_processes, "_members", lambda token: next(scans)) + # The PID was reused after enumeration; its new start time must be checked + # immediately before signalling even though the original marker matched. + fields = ["S"] + ["0"] * 18 + ["124"] + monkeypatch.setattr(Path, "read_text", lambda *a, **k: "111 (python) " + " ".join(fields)) + signalled = [] + monkeypatch.setattr(os, "kill", lambda *args: signalled.append(args)) + monkeypatch.setattr(time, "sleep", lambda _: None) + guard = _moe_processes.cleanup("token") + assert guard["verified"] and not signalled, guard + + +@pytest.mark.skipif(sys.platform != "linux", reason="real /proc and detached process sessions") +def test_deadline_cleans_detached_workers_but_preserves_unrelated_process(tmp_path): + token = uuid.uuid4().hex + script = tmp_path / "fake_torchrun.py" + script.write_text( + "import json, os, pathlib, subprocess, sys, time\n" + f"os.environ[{_moe_processes.MARKER!r}] = sys.argv[1]\n" + "children = [subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], " + "start_new_session=True) for _ in range(2)]\n" + "pathlib.Path('workers.json').write_text(json.dumps([p.pid for p in children]))\n" + "time.sleep(60)\n", encoding="utf-8") + unrelated = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"], + start_new_session=True) + try: + execution = execute_trial((sys.executable, str(script), token), tmp_path, + tmp_path / "execution", time.monotonic() + 2, None) + assert execution["status"] == "timeout" + pids = json.loads((tmp_path / "workers.json").read_text()) + guard = _moe_processes.cleanup(token) + assert guard["verified"], guard + assert set(pids) <= set(guard["signalled_pids"]), guard + assert not _moe_processes._members(token) + assert unrelated.poll() is None + finally: + _moe_processes.cleanup(token) + unrelated.kill() + unrelated.wait(timeout=5) diff --git a/tests/test_npu_engineering_benchmark.py b/tests/test_npu_engineering_benchmark.py new file mode 100644 index 000000000..979eee2e3 --- /dev/null +++ b/tests/test_npu_engineering_benchmark.py @@ -0,0 +1,139 @@ +"""Device-free contract controls; actual NPU execution is explicitly opted in.""" + +import json +import os +import shutil +import subprocess +import sys +from types import SimpleNamespace + +import pytest + +from inferrsi.harness.benchmarks import load_builtin_task, load_task +from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action +from inferrsi.harness.benchmarks.npu_task import check_output, device_kernels, reference +from inferrsi.harness.skill_eval_benchmark import task_descriptor +from tools.npu_benchmark_validate import validation_passed + + +def test_discovery_has_no_torch_dependency_and_formal_cpu_path_refuses_npu(): + task = load_builtin_task("rmsnorm-npu") + assert task.manifest["environment"]["backend"] == "ascend" + subprocess.run([sys.executable, "-c", "import sys; " + "from inferrsi.harness.benchmarks import load_builtin_task; " + "load_builtin_task('rmsnorm-npu'); " + "assert 'torch' not in sys.modules and 'torch_npu' not in sys.modules"], + check=True) + with pytest.raises(ValueError, match="CPU"): + task_descriptor("rmsnorm-npu") + + +@pytest.mark.parametrize("field,value", [ + ("environment", {"backend": "cpu", "runtime": "numpy", "dtype": "float32"}), + ("oracle.operation", "softmax"), ("measurement.boundary", "synchronous-compute-call"), + ("workloads.0.layout", "strided"), ("workloads.0.shape", [2, 3, 4]), +]) +def test_npu_contract_rejects_wrong_execution_semantics(field, value, tmp_path): + root = tmp_path / "task" + shutil.copytree(load_builtin_task("rmsnorm-npu").root, root) + manifest = json.loads((root / "task.json").read_text(encoding="utf-8")) + current = manifest + parts = field.split(".") + for key in parts[:-1]: + current = current[int(key)] if isinstance(current, list) else current[key] + current[parts[-1]] = value + (root / "task.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError): + load_task(root) + + +def test_unallocated_device_is_inconclusive_without_import_or_fallback(tmp_path, monkeypatch): + monkeypatch.delenv("ASCEND_RT_VISIBLE_DEVICES", raising=False) + task = load_builtin_task("rmsnorm-npu") + grade = grade_artifact(task, task.root / "candidate.py", tmp_path / "run", timeout_seconds=20) + assert grade["verdict"] == "INCONCLUSIVE" + result = json.loads((tmp_path / "run" / "result.json").read_text(encoding="utf-8")) + assert result["error_type"] == "RuntimeError" and "metrics" not in result + + +def test_scalar_affine_oracle_preserves_epsilon_and_weight(): + np = pytest.importorskip("numpy") + x = np.array([[0, 0], [1e-7, -1e-7]], dtype=np.float32) + weight = np.array([2, -3], dtype=np.float32) + y = reference(x, weight, 1e-5, np) + np.testing.assert_allclose(y[0], [0, 0]) + np.testing.assert_allclose(y[1], [2e-7 / (1e-5 + 1e-14)**0.5, + 3e-7 / (1e-5 + 1e-14)**0.5], rtol=1e-6) + + +def test_profile_requires_hardware_kernel_records(): + launch = {"pid": 1, "ph": "X", "name": "npu_rms_norm", "cat": "cpu_op"} + metadata = {"pid": 2, "name": "process_name", "args": {"name": "Ascend Hardware"}} + flow = {"pid": 2, "ph": "f", "cat": "async_npu"} + kernel = {"pid": 2, "ph": "X", "name": "RmsNorm", "dur": 5.0, + "args": {"Task Type": "KERNEL_AIVEC"}} + assert device_kernels([launch, metadata, flow]) == [] + assert device_kernels({"traceEvents": [launch, metadata, flow, kernel]}) == [kernel] + + +@pytest.mark.parametrize("status,code,passed,expected", [ + ("completed", 0, True, True), ("failed", 1, False, False), + ("timeout", None, True, False), ("completed", 0, False, False), +]) +def test_noop_qualification_requires_final_quality_pass(status, code, passed, expected): + assert validation_passed({"execution_status": status, "returncode": code, + "result": {"status": "completed", "passed": passed}}) is expected + + +def test_changed_input_shape_and_broadcast_output_cannot_pass(): + np = pytest.importorskip("numpy") + + class Tensor: + dtype = "float32" + device = "npu:0" + + def __init__(self, array): + self.array, self.shape = array, array.shape + + def cpu(self): + return self + + def detach(self): + return self + + def numpy(self): + return self.array + + def is_contiguous(self): + return True + + torch = SimpleNamespace(Tensor=Tensor, float32="float32", device=lambda value: value) + original = np.array([[1., 2., 3.]], dtype=np.float32) + weight = np.ones(3, dtype=np.float32) + # Identical bytes, wrong metadata, and values that allclose could broadcast. + squeezed = Tensor(original.reshape(3)) + checks = check_output(squeezed, squeezed, Tensor(weight), original, weight, original, + {"rtol": 1e-5, "atol": 1e-6}, torch, np) + assert checks["shape_dtype_device"] is False + assert checks["input_preserved"] is False + assert checks["values"] is False + + +@pytest.mark.skipif(os.environ.get("INFERRSI_REQUIRE_NPU_BENCHMARK") != "1", + reason="Requires explicitly scheduled Ascend NPU execution") +def test_real_npu_baseline_fused_wrong_output_and_profile(tmp_path): + task = load_builtin_task("rmsnorm-npu") + for name in ("baseline", "candidate"): + report = grade_artifact(task, task.root / f"{name}.py", tmp_path / name, + timeout_seconds=180) + assert report["verdict"] == "PASS" + wrong = tmp_path / "wrong.py" + wrong.write_text("import torch\ndef compute(x, weight, *, eps): return torch.zeros_like(x)\n") + result = run_action(task, wrong, "measure", tmp_path / "wrong", timeout_seconds=180) + record = json.loads(open(result["result_path"]).read()) + assert not record["passed"] and "metrics" not in record + result = run_action(task, task.root / "candidate.py", "profile", tmp_path / "profile", + timeout_seconds=180) + record = json.loads(open(result["result_path"]).read()) + assert record["passed"] and "metrics" not in record + assert any("RmsNorm" in kernel["name"] for kernel in record["diagnostic"]["kernels"]) diff --git a/tests/test_skill_eval_benchmark.py b/tests/test_skill_eval_benchmark.py new file mode 100644 index 000000000..04b1018d7 --- /dev/null +++ b/tests/test_skill_eval_benchmark.py @@ -0,0 +1,325 @@ +"""Shared-oracle and isolated artifact controls; never model-benefit evidence.""" + +import base64 +import json +import os +import shutil +from dataclasses import replace +from pathlib import Path +from unittest import mock + +import numpy as np +import pytest + +from inferrsi.harness import skill_eval_benchmark as benchmark +from inferrsi.harness import skill_eval_engineering as engineering +from inferrsi.harness.benchmarks import numpy_task +from inferrsi.harness.benchmarks.contract import load_builtin_task, load_task + +READY = {"ready": True, "numpy": np.__version__, "python": "3.10.0", "machine": "x86_64"} + + +def answer(name="softmax-cpu", source=None): + task = benchmark.task_descriptor(name) + return task, {"artifact": benchmark.ARTIFACT, "summary": "Authored software control", + "artifact_source": source or benchmark.starter(task).decode()} + + +def oracle_transport(task): + """Synthetic transport exercises parent grading only, never executes source.""" + package = load_builtin_task(task["benchmark"]["task_id"]) + prepared, _, _ = benchmark._prepare(package, np) + outputs = [{"output": {"shape": list(x.shape), "dtype": "float32", + "data": base64.b64encode(expected.tobytes()).decode()}, + "input_after": base64.b64encode(x.tobytes()).decode()} + for _, x, expected, _, _ in prepared] + return {"outputs": outputs} + + +def test_descriptor_freezes_builtin_task_and_shared_grader(): + name = "softmax-cpu" + task, _ = answer(name) + benchmark.validate_task(task) + package = load_task(name) + assert task["contract"] == package.manifest + assert task["benchmark"] == { + "task_id": name, "task_sha256": package.task_sha256, + "grader_sha256": package.grader_sha256, + } + assert benchmark.starter(task) == (package.root / "candidate.py").read_bytes() + + +def shadow_softmax(directory): + """A reviewed-path task is valid but must never replace the formal rubric.""" + canonical = load_builtin_task("softmax-cpu") + shadow = directory / "softmax-cpu" + shutil.copytree(canonical.root, shadow, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + manifest = canonical.manifest + manifest["oracle"]["atol"] = 1 + (shadow / "task.json").write_text(json.dumps(manifest), encoding="utf-8") + (shadow / "candidate.py").write_text( + "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n", + encoding="utf-8", + ) + return canonical, shadow + + +def test_cwd_shadow_cannot_replace_descriptor_starter_or_parent_oracle(tmp_path, monkeypatch): + canonical, shadow = shadow_softmax(tmp_path) + original, _ = answer("softmax-cpu") + monkeypatch.chdir(tmp_path) + # Trusted callers can still deliberately load an external reviewed task. + assert load_task("softmax-cpu").root == shadow + assert load_task(shadow).manifest["oracle"]["atol"] == 1 + assert load_builtin_task("softmax-cpu").root == canonical.root + task, output = answer("softmax-cpu") + assert task == original + assert task["benchmark"]["task_sha256"] == canonical.task_sha256 + assert benchmark.starter(task) == (canonical.root / "candidate.py").read_bytes() + assert benchmark.starter(task) != (shadow / "candidate.py").read_bytes() + # Synthetic returned arrays test parent grading, not sandbox source execution. + transport = oracle_transport(task) + for record in transport["outputs"]: + record["output"]["data"] = base64.b64encode( + bytes(len(base64.b64decode(record["output"]["data"]))) + ).decode() + with mock.patch.object(benchmark, "_invoke", side_effect=[READY, transport]): + result = benchmark.verify(task, output) + assert not result["passed"] + assert not result["assertions"]["normal.values"] + assert "metrics" not in result + + +@pytest.mark.parametrize("change", ["adapter", "environment"]) +def test_formal_artifact_path_rejects_new_backend_packages(change, monkeypatch): + package = load_builtin_task("softmax-cpu") + manifest = package.manifest + if change == "adapter": + manifest["adapter"] = "torch-npu-rmsnorm-v1" + else: + manifest["environment"] = {"backend": "ascend", "runtime": "torch_npu", + "dtype": "float32"} + unsupported = replace(package, frozen_manifest=json.dumps(manifest)) + monkeypatch.setattr(benchmark, "load_builtin_task", lambda _: unsupported) + with pytest.raises(ValueError, match="NumPy CPU adapter"): + benchmark.task_descriptor("rmsnorm-npu") + + +@pytest.mark.parametrize("change", ["hash", "holdout", "path", "request", "extra", "contract"]) +def test_descriptor_drift_and_noncanonical_tasks_rejected(change): + task, _ = answer() + if change == "hash": + task["benchmark"]["task_sha256"] = "0" * 64 + elif change == "path": + task["benchmark"]["task_id"] = "../softmax-cpu" + elif change == "holdout": + task["split"] = "holdout" + elif change == "request": + task["request"] += " instead use a new oracle" + elif change == "contract": + task["contract"]["oracle"]["atol"] = 1 + else: + task["expected"] = True + with pytest.raises(ValueError): + benchmark.validate_task(task) + + +@pytest.mark.parametrize("change", [{"artifact": "parser.py"}, {"artifact_source": ""}, + {"artifact_source": "x" * 65537}, {"passed": True}]) +def test_invalid_artifact_does_not_start_source(change): + task, output = answer() + with mock.patch.object(benchmark, "_invoke") as invoke: + assert not benchmark.verify(task, {**output, **change})["passed"] + invoke.assert_not_called() + + +def test_no_sandbox_means_explicit_unavailable_never_host_execution(tmp_path): + marker = tmp_path / "must-not-exist" + task, output = answer(source=f"open({str(marker)!r}, 'w').write('bad')") + with mock.patch.object(benchmark, "_invoke", return_value=None): + with pytest.raises(benchmark.SandboxUnavailable): + benchmark.verify(task, output) + assert not marker.exists() + + +def test_missing_numpy_is_explicit_unavailable(): + task, output = answer() + with mock.patch.dict("sys.modules", {"numpy": None}): + with pytest.raises(benchmark.SandboxUnavailable, match="requires NumPy"): + benchmark.verify(task, output) + + +def test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call(): + name = "softmax-cpu" + task, output = answer(name) + transport = oracle_transport(task) + calls = [] + + def invoke(work, request=None, *, probe=False): + assert sorted(p.name for p in work.iterdir()) == ["implementation.py", "runner.py"] + assert (work / "implementation.py").read_bytes() == output["artifact_source"].encode() + if probe: + return READY + calls.append(request) + assert set(request) == {"eps", "inputs"} + assert all(set(x) == {"shape", "layout", "data"} for x in request["inputs"]) + return transport + + with mock.patch.object(benchmark, "_invoke", side_effect=invoke): + result = benchmark.verify(task, output) + package = load_task(name) + # Use only the repository-authored candidate for comparison, not submitted source. + trusted = numpy_task.evaluate(package, package.root / "candidate.py", "check", Path(".")) + assert result["passed"] and len(result["assertions"]) == 164 + assert result["assertions"] == trusted["assertions"] + for key in ("workloads", "input_sequence"): + assert result["benchmark"][key] == trusted[key] + assert result["benchmark"]["returned_calls"] == trusted["executed_calls"] + assert len(calls) == 1 and len(calls[0]["inputs"]) == 41 + assert result["benchmark"]["timing"] == "not measured" + + +@pytest.mark.parametrize("wrong", ["dtype", "values", "mutation", "truncated", "malformed"]) +def test_parent_rejects_wrong_or_malformed_returned_arrays(wrong): + task, output = answer() + transport = oracle_transport(task) + record = transport["outputs"][10] + if wrong == "dtype": + record["output"]["dtype"] = "float64" + elif wrong == "values": + record["output"]["data"] = base64.b64encode( + bytes(len(base64.b64decode(record["output"]["data"]))) + ).decode() + elif wrong == "mutation": + record["input_after"] = base64.b64encode( + bytes(len(base64.b64decode(record["input_after"]))) + ).decode() + elif wrong == "truncated": + transport["outputs"].pop() + else: + record["output"]["data"] = "!bad-base64" + with mock.patch.object(benchmark, "_invoke", + side_effect=[READY, transport]): + result = benchmark.verify(task, output) + assert not result["passed"] + assert "metrics" not in result + + +def test_workspace_freeze_excludes_only_implementation(tmp_path): + (tmp_path / "task.json").write_text("{}") + (tmp_path / "implementation.py").write_text("original") + (tmp_path / "parser.py").write_text("frozen") + before = benchmark.input_hashes(tmp_path) + assert set(before) == {"task.json", "parser.py"} + (tmp_path / "implementation.py").write_text("candidate") + assert benchmark.input_hashes(tmp_path) == before + (tmp_path / "parser.py").write_text("changed") + assert benchmark.input_hashes(tmp_path) != before + + +def test_numpy_mounts_extend_existing_boundary_without_host_venv_or_repository(): + original = ["bwrap", "--unshare-all", "--clearenv", "--ro-bind", "/safe/work", "/work", + "--remount-ro", "/", "--", "/usr/bin/python3", "-I", "-S", "-B", + "/work/runner.py"] + runtime = mock.Mock(stdout=json.dumps({"version": np.__version__, + "root": "/opt/public-runtime/numpy"}).encode()) + with mock.patch.object(engineering, "sandbox_command", return_value=original), \ + mock.patch.object(benchmark.subprocess, "run", return_value=runtime), \ + mock.patch.object(Path, "is_dir", return_value=True), \ + mock.patch.object(Path, "is_absolute", return_value=True), \ + mock.patch.object(Path, "resolve", return_value=Path("/opt/public-runtime/numpy.libs")): + command = benchmark.sandbox_command(Path("/safe/work")) + assert "--unshare-all" in command and "--clearenv" in command + assert command[-5:] == original[-5:] + assert command.count("--ro-bind") == 3 + assert "/runtime/numpy" in command and "/runtime/numpy.libs" in command + assert command.index("/runtime/numpy.libs") < command.index("--remount-ro") + assert "--share-net" not in command + assert "OPENBLAS_NUM_THREADS" in command + + +def test_runtime_version_mismatch_is_unavailable(): + runtime = mock.Mock(stdout=json.dumps({"version": "different", "root": "/runtime/numpy"})) + with mock.patch.object(engineering, "sandbox_command", return_value=["--"]), \ + mock.patch.object(benchmark.subprocess, "run", return_value=runtime): + with pytest.raises(benchmark.SandboxUnavailable): + benchmark.sandbox_command(Path("/safe/work")) + + +@pytest.mark.parametrize("ready", [False, True]) +def test_worker_startup_is_distinct_from_artifact_failure(ready): + process = mock.Mock(returncode=1) + + def start(*args, **kwargs): + assert kwargs["env"] == {"PATH": "/usr/bin:/bin", "LANG": "C"} + if ready: + kwargs["stderr"].write(b"INFERRSI_BENCHMARK_READY\n") + return process + + with mock.patch.object(benchmark, "sandbox_command", return_value=["sandbox"]), \ + mock.patch.object(benchmark.subprocess, "Popen", side_effect=start): + if ready: + assert benchmark._invoke(Path("/unused"), {}) is None + else: + with pytest.raises(benchmark.SandboxUnavailable): + benchmark._invoke(Path("/unused"), {}) + + +def run_real(task, output): + try: + return benchmark.verify(task, output) + except benchmark.SandboxUnavailable as error: + if os.environ.get("INFERRSI_REQUIRE_BENCHMARK_SANDBOX") == "1": + pytest.fail(str(error)) + pytest.skip(str(error)) + + +def test_real_sandbox_correct_and_wrong_authored_controls(): + task, output = answer() + result = run_real(task, output) + assert result["passed"] and len(result["assertions"]) == 164 + wrong = {**output, "artifact_source": + "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n"} + rejected = run_real(task, wrong) + assert not rejected["passed"] and "metrics" not in rejected + + +def test_real_sandbox_rejects_zero_output_despite_cwd_shadow(tmp_path, monkeypatch): + canonical, shadow = shadow_softmax(tmp_path) + monkeypatch.chdir(tmp_path) + task, wrong = answer("softmax-cpu", source=(shadow / "candidate.py").read_text()) + assert task["benchmark"]["task_sha256"] == canonical.task_sha256 + result = run_real(task, wrong) + assert not result["passed"] and not result["assertions"]["normal.values"] + assert "metrics" not in result + + +def test_real_sandbox_preserves_sequence_state_and_rejects_cached_output_replay(): + source = """import numpy as np +cache = {} +def compute(x, *, eps): + key = (x.shape, x.strides) + if key not in cache: + y = np.exp(x - x.max(axis=1, keepdims=True)) + cache[key] = y / y.sum(axis=1, keepdims=True) + return cache[key] +""" + task, output = answer(source=source) + result = run_real(task, output) + assert not result["passed"] + assert result["assertions"]["normal.values"] + assert not result["assertions"]["normal.call-0.values"] + + +def test_real_sandbox_cannot_read_host_sentinel_or_oracle(tmp_path): + sentinel = tmp_path / "host-only-secret.txt" + sentinel.write_text("must not be visible to submitted code") + task, output = answer() + source = output["artifact_source"] + f""" +from pathlib import Path +assert not Path({str(sentinel)!r}).exists() +assert sorted(p.name for p in Path('/work').iterdir()) == ['implementation.py', 'runner.py'] +assert not Path('/proc/1/root{str(sentinel)}').exists() +""" + result = run_real(task, {**output, "artifact_source": source}) + assert result["passed"] diff --git a/tests/test_skill_eval_benchmark_compare.py b/tests/test_skill_eval_benchmark_compare.py new file mode 100644 index 000000000..1165286b2 --- /dev/null +++ b/tests/test_skill_eval_benchmark_compare.py @@ -0,0 +1,330 @@ +"""Formal two-arm plumbing with authored producers, not agent/skill efficacy. + +The producer is a real local subprocess. These controller tests replace the +artifact verifier; they do not establish that Linux/NumPy sandbox execution ran. +The separate benchmark verifier tests own real sandbox software controls. +""" + +import hashlib +import json +import os +import shutil +import subprocess +import sys + +import pytest + +from inferrsi.harness import skill_eval_benchmark as benchmark +from inferrsi.harness.benchmarks.contract import load_builtin_task, load_task +from inferrsi.infra.artifacts import sha256, write_json +from tests.fixtures.skill_eval_compare_support import configuration +from tools import skill_eval_compare as compare +from tools import skill_eval_local as local + +PRODUCER = """import json, pathlib, sys +work, final = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]) +source, mode = sys.argv[3:] +artifact = work / 'implementation.py' +artifact.write_text(source, encoding='utf-8', newline='') +answer = {'artifact': 'implementation.py', 'summary': 'Authored software control, no model'} +if mode == 'changed-input': (work / 'task.json').write_text('{}') +if mode == 'extra-file': (work / 'unexpected.txt').write_text('unexpected') +if mode == 'missing-artifact': artifact.unlink() +if mode == 'oversized': artifact.write_bytes(b'x' * 65537) +if mode == 'invalid-answer': answer['passed'] = True +final.write_text(json.dumps(answer), encoding='utf-8') +event = {'type': 'turn.completed'} +if mode != 'unknown-usage': event['usage'] = {'input_tokens': 3, 'output_tokens': 2} +print(json.dumps(event), flush=True) +""" + + +class AuthoredProducer: + """Write candidate bytes without importing or executing them on the host.""" + + timeout_seconds = 30 + + def __init__(self, modes=None): + self.modes = modes or {} + self.calls = [] + + def command(self, workspace, final, schema, prompt): + task = compare.read_json(workspace / "task.json") + role = "baseline" if "-baseline-" in final.parent.name else "candidate" + repeat = int(final.parent.name.rsplit("-", 1)[1]) + mode = self.modes.get( + (task["id"], role, repeat), self.modes.get((task["id"], role), "correct")) + source = ("import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n" + if mode == "wrong-output" else benchmark.starter(task).decode("utf-8")) + self.calls.append({ + "task": task, "role": role, "repeat": repeat, "workspace": workspace, + "schema_sha256": sha256(schema), "source": source, + "inputs": {p.relative_to(workspace).as_posix(): p.read_bytes() + for p in workspace.rglob("*") if p.is_file()}, + }) + return (sys.executable, "-B", "-c", PRODUCER, str(workspace), str(final), source, mode) + + +def prepared(tmp_path, names=("softmax-cpu",), repeats=1): + compare.require_git_version(subprocess.check_output(("git", "--version"), text=True)) + config_path, repo = configuration(tmp_path, ids=names, repeats=repeats) + config = compare.read_json(config_path) + config["adapter"] = benchmark.ADAPTER + write_json(config_path, config) + tasks = [benchmark.task_descriptor(name) for name in names] + write_json(tmp_path / "cases.json", tasks) + output = tmp_path / "run" + digest = compare.prepare(config_path, repo, output) + return output, digest, config["snapshots"]["candidate"]["git_revision"] + + +def software_control_verifier(task, answer): + """A source-identity stub, deliberately not presented as numerical grading.""" + benchmark.validate_task(task) + passed = answer["artifact_source"] == benchmark.starter(task).decode("utf-8") + return { + "passed": passed, "assertions": {"authored_source_control": passed}, + "benchmark": {"scope": "mocked controller test only; no sandbox execution"}, + } + + +def test_real_authored_processes_use_frozen_task_and_retain_both_sources(tmp_path, monkeypatch): + # Two repetitions retain AB/BA isolation and both report outcomes with one task. + output, digest, _ = prepared(tmp_path, repeats=2) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + client = AuthoredProducer({("softmax-cpu", "baseline", 1): "wrong-output"}) + result = compare.execute(output, digest, client=client) + assert result["adapter"] == benchmark.ADAPTER + assert result["counts"]["fix"] == result["counts"]["unchanged_correct"] == 1 + assert len(client.calls) == 4 + assert [call["repeat"] for call in client.calls] == [1, 1, 2, 2] + assert [call["role"] for call in client.calls] == [ + "baseline", "candidate", "candidate", "baseline"] + for left, right in (client.calls[:2], client.calls[2:]): + assert left["task"] == right["task"] + assert left["inputs"]["task.json"] == right["inputs"]["task.json"] + assert left["inputs"]["implementation.py"] == right["inputs"]["implementation.py"] + assert left["schema_sha256"] == right["schema_sha256"] + assert left["inputs"]["knowledge/selected/SKILL.md"] != ( + right["inputs"]["knowledge/selected/SKILL.md"]) + assert len({call["workspace"] for call in client.calls}) == 4 + assert all(not call["workspace"].exists() for call in client.calls) + for pair in result["pairs"]: + for role in compare.ROLES: + trial = pair[role] + artifact = output / trial["evidence"]["implementation.py"] + assert sha256(artifact) == trial["artifact_sha256"] + answer = json.loads(trial["output"]) + assert artifact.read_text(encoding="utf-8") == answer["artifact_source"] + assert trial["artifact_kind"] == benchmark.ADAPTER + assert trial["inputs_unchanged"] is True + assert trial["grader_trust"] == "reverified" + assert set(trial["checks"]) == {"passed", "assertions"} + assert "metrics" not in trial + before = (output / "report.json").read_bytes() + compare.report(output) + assert len(client.calls) == 4 + assert (output / "report.json").read_bytes() == before + assert str(tmp_path) not in before.decode("utf-8") + + +@pytest.mark.parametrize("kind", ["task", "schema", "starter"]) +def test_changed_frozen_input_prevents_any_producer_call(tmp_path, kind): + output, digest, _ = prepared(tmp_path) + plan = compare.load_plan(output) + name = {"task": "plan/tasks.json", "schema": "plan/schema.json"}.get(kind) + if name is None: + name = next(name for name in plan["files"] if name.endswith("/implementation.py")) + path = output / name + path.write_bytes(path.read_bytes() + b"\n") + client = AuthoredProducer() + with pytest.raises(ValueError, match="Frozen input changed"): + compare.execute(output, digest, client=client) + assert client.calls == [] + + +def test_task_grader_and_oracle_dependencies_are_frozen_before_producer(tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path) + plan = compare.load_plan(output) + names = ( + "src/inferrsi/harness/skill_eval_benchmark.py", + "src/inferrsi/harness/benchmarks/numpy_task.py", + "src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json", + ) + for name in names: + assert plan["controllers"][name] == sha256(compare.ROOT / name) + changed_path = compare.ROOT / name + + def modified_digest(path): + return "0" * 64 if path == changed_path else sha256(path) + + client = AuthoredProducer() + with monkeypatch.context() as scope: + scope.setattr(compare, "sha256", modified_digest) + with pytest.raises(ValueError, match="Controller changed after freeze"): + compare.execute(output, digest, client=client) + assert client.calls == [] + + +def test_prepare_and_revalidation_use_packaged_task_despite_cwd_shadow(tmp_path, monkeypatch): + canonical = load_builtin_task("softmax-cpu") + shadow = tmp_path / "softmax-cpu" + shutil.copytree(canonical.root, shadow, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + manifest = canonical.manifest + manifest["oracle"]["atol"] = 1 + write_json(shadow / "task.json", manifest) + (shadow / "candidate.py").write_text("# substituted starter\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + output, _, _ = prepared(tmp_path, names=("softmax-cpu",)) + plan = compare.load_plan(output) + frozen = load_task(output / "plan/benchmark-tasks/softmax-cpu") + assert frozen.task_sha256 == canonical.task_sha256 + assert frozen.manifest["oracle"] == canonical.manifest["oracle"] + assert (output / "plan/starters/softmax-cpu/implementation.py").read_bytes() == ( + canonical.root / "candidate.py").read_bytes() + name = "src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json" + assert plan["controllers"][name] == sha256(canonical.root / "task.json") + assert not any(name.startswith("softmax-cpu/") for name in plan["controllers"]) + # A later cwd change cannot make the frozen package validate against a new rubric. + manifest["oracle"]["atol"] = 2 + write_json(shadow / "task.json", manifest) + assert compare.load_plan(output) == plan + + +@pytest.mark.parametrize("mode,error", [ + ("changed-input", "inputs_changed"), + ("extra-file", "inputs_changed"), + ("missing-artifact", "missing_or_invalid_artifact"), + ("oversized", "missing_or_invalid_artifact"), + ("invalid-answer", "invalid_artifact_answer"), +]) +def test_invalid_handoff_is_not_a_clean_comparison(tmp_path, monkeypatch, mode, error): + output, digest, _ = prepared(tmp_path) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + client = AuthoredProducer({("softmax-cpu", "candidate"): mode}) + result = compare.execute(output, digest, client=client) + trial = result["pairs"][0]["candidate"] + assert trial["verdict"] == "INCONCLUSIVE" + retained = compare.read_json(output / trial["evidence"]["result.json"]) + assert retained["evaluation_error"] == error + # Report prioritizes missing artifact integrity over the original capture + # error, which remains available in the unchanged result.json record. + expected_report_error = ("missing_retained_artifact" if mode in { + "missing-artifact", "oversized", "invalid-answer"} else error) + assert trial["evaluation_error"] == expected_report_error + assert result["counts"]["inconclusive"] == 1 + + +def test_no_unsafe_default_launch_or_silent_sandbox_fallback(tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path) + with pytest.raises(ValueError, match="confined local client"): + compare.execute(output, digest) + assert not (output / "results/start.json").exists() + + def unavailable(*_): + raise benchmark.SandboxUnavailable("authored unsupported-host control") + + monkeypatch.setattr(benchmark, "verify", unavailable) + result = compare.execute(output, digest, client=AuthoredProducer()) + for role in compare.ROLES: + trial = result["pairs"][0][role] + assert trial["verdict"] == "INCONCLUSIVE" + assert trial["evaluation_error"] == "artifact_sandbox_unavailable" + assert trial["artifact_sha256"] is not None + + +def test_retained_source_tamper_invalidates_report_without_rerunning_producer( + tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + client = AuthoredProducer() + result = compare.execute(output, digest, client=client) + trial = result["pairs"][0]["candidate"] + artifact = output / trial["evidence"]["implementation.py"] + artifact.write_text("# substituted source\n", encoding="utf-8") + report = compare.read_report(output) + assert report["pairs"][0]["candidate"]["verdict"] == "INCONCLUSIVE" + assert report["pairs"][0]["candidate"]["evaluation_error"] == "retained_artifact_hash_mismatch" + assert len(client.calls) == 2 + + +def test_unknown_usage_stops_new_trials_without_inventing_cost(tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + client = AuthoredProducer({("softmax-cpu", "baseline"): "unknown-usage"}) + result = compare.execute(output, digest, client=client) + assert len(client.calls) == 1 + assert result["status"]["stop_reason"] == "unknown_usage" + assert result["counts"]["missing_pair"] == 1 + assert result["conditions"]["baseline"]["cost"] is None + + +def test_manual_handoff_keeps_author_reports_distinct_from_artifact_checks(tmp_path, monkeypatch): + output, digest, head = prepared(tmp_path) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + submission = tmp_path / "author-logs" + submission.mkdir() + write_json(submission / "execution.json", { + "model": compare.load_plan(output)["model"], "environment": "authored software control", + "access": "not a real agent run", "resources": "no model calls", "usage": None, + "elapsed_seconds": 1.0, "agent_exit_code": 0, "summary": "controller test only", + }) + for name in ("stdout.log", "stderr.log"): + (submission / name).write_text("authored control\n", encoding="utf-8") + for role in compare.ROLES: + workspace = tmp_path / role + local.stage(output, digest, "softmax-cpu", role, 1, workspace, head) + if role == "baseline": + (workspace / benchmark.ARTIFACT).write_bytes(b"# authored bad control\r\n") + result = local.capture(output, digest, "softmax-cpu", role, 1, workspace, head, submission) + assert result["counts"]["artifact_improved"] == 1 + pair = result["pairs"][0] + assert pair["comparability"] == "unverified_author_reported" + assert pair["baseline"]["verdict"] == "FAIL" + assert pair["candidate"]["verdict"] == "PASS" + assert pair["candidate"]["author_reported_execution"] is True + assert pair["candidate"]["observations"]["usage"] is None + artifact = output / pair["baseline"]["evidence"]["implementation.py"] + assert artifact.read_bytes() == b"# authored bad control\r\n" + assert pair["baseline"]["artifact_sha256"] == hashlib.sha256(artifact.read_bytes()).hexdigest() + + +@pytest.mark.parametrize("change", [ + {"adapter": "unregistered-benchmark"}, + {"execution_mode": "generation-only"}, + {"schema": "custom-schema.json"}, + {"grader": "custom-grader.py"}, +]) +def test_adapter_does_not_accept_unfrozen_alternate_paths(tmp_path, change): + config_path, _ = configuration(tmp_path, ids=("softmax-cpu",)) + config = compare.read_json(config_path) + config.update(adapter=benchmark.ADAPTER) + config.update(change) + with pytest.raises(ValueError): + compare.validate(config, [benchmark.task_descriptor("softmax-cpu")]) + + +def test_real_benchmark_sandbox_two_arm_handoff_or_explicit_unavailable(tmp_path): + """Actual authored source replay; never represents an LLM or skill benefit.""" + output, digest, _ = prepared(tmp_path) + client = AuthoredProducer({("softmax-cpu", "baseline"): "wrong-output"}) + result = compare.execute(output, digest, client=client) + pair = result["pairs"][0] + if any(pair[role]["evaluation_error"] == "artifact_sandbox_unavailable" + for role in compare.ROLES): + if os.environ.get("INFERRSI_REQUIRE_BENCHMARK_SANDBOX") == "1": + pytest.fail("Required benchmark sandbox unavailable; no host fallback") + pytest.skip("Benchmark sandbox unavailable; two-arm numerical execution unqualified") + assert len(client.calls) == 2 + assert pair["baseline"]["verdict"] == "FAIL" + assert pair["candidate"]["verdict"] == "PASS" + assert result["counts"]["fix"] == 1 + assert len(pair["candidate"]["checks"]["assertions"]) == 164 + assert not all(pair["baseline"]["checks"]["assertions"].values()) + for role in compare.ROLES: + trial = pair[role] + assert trial["evaluation_error"] is None + assert trial["grader_trust"] == "reverified" + assert "suite_latency_ms" not in json.dumps(trial["checks"]) + artifact = output / trial["evidence"]["implementation.py"] + assert sha256(artifact) == trial["artifact_sha256"] diff --git a/tools/engineering_benchmark_validate.py b/tools/engineering_benchmark_validate.py new file mode 100644 index 000000000..2fb7af8a8 --- /dev/null +++ b/tools/engineering_benchmark_validate.py @@ -0,0 +1,192 @@ +"""Reproduce CPU task integration with retained, authored controls. + +No models, hardware allocators or downloaded programs are invoked. Output is a +local validation bundle, not campaign acceptance or a skill-benefit experiment. +""" + +import argparse +import json +import shutil +import sys +import time +from pathlib import Path + +from inferrsi.harness.benchmarks import builtin_tasks, load_task, skill_verifier +from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action +from inferrsi.harness.runner import run_experiment +from inferrsi.harness.skill_evaluation import run_trial +from inferrsi.infra.artifacts import atomic_write_json + + +class AuthoredControl: + """A local fixture emits a retained artifact identity, never a claimed score.""" + + timeout_seconds = 20 + + def __init__(self, identity): + self.identity = identity + + def command(self, workspace, final, schema, prompt): + script = ( + "import json, pathlib, sys; " + "pathlib.Path(sys.argv[1]).write_text(json.dumps({'candidate_sha256':sys.argv[2]})); " + "print(json.dumps({'type':'turn.completed'}))" + ) + return sys.executable, "-B", "-c", script, str(final), self.identity + + +def replay(task, candidate, directory): + descriptor = skill_verifier.task_descriptor(task, candidate) + workspace = directory / "workspace" + workspace.mkdir(parents=True) + atomic_write_json(workspace / "task.json", descriptor) + return run_trial( + AuthoredControl(descriptor["benchmark"]["identity"]["candidate_sha256"]), + descriptor, + "N", + 1, + workspace, + directory / "trial", + directory / "unused-schema.json", + Path(skill_verifier.__file__), + "Reviewed offline control", + time.monotonic() + 40, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--repetitions", type=int, required=True) + args = parser.parse_args() + output = args.output_dir.resolve() + output.mkdir(parents=True, exist_ok=False) + records = [] + for name in builtin_tasks(): + # This historical replay utility has a NumPy-only candidate protocol. + if load_task(name).manifest["adapter"] != "numpy-axis1-v2": + continue + directory = output / name + directory.mkdir() + original = load_task(name) + shutil.copytree( + original.root, directory / "task", ignore=shutil.ignore_patterns("__pycache__") + ) + task = load_task(directory / "task") + candidate = directory / "implementation.py" + shutil.copyfile(task.root / "candidate.py", candidate) + # Freeze the plan before any payload execution. + spec = task.runspec( + candidate, + repo=args.repo, + output_dir=directory / "workbench", + repetitions=args.repetitions, + max_minutes=2, + min_improvement_pct=0, + ) + atomic_write_json(directory / "runspec.json", spec.to_dict()) + atomic_write_json(directory / "identity.json", task.identity(candidate)) + grade = grade_artifact(task, candidate, directory / "correctness", timeout_seconds=20) + profile = run_action(task, candidate, "profile", directory / "profile", timeout_seconds=20) + run = run_experiment( + spec, {"scope": "CPU development integration", "identity": task.identity(candidate)} + ) + evaluation = json.loads((run / "evaluation.json").read_text(encoding="utf-8")) + good = replay(task, candidate, directory / "skilleval-good") + bad_source = directory / "wrong.py" + bad_source.write_text( + "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n", + encoding="utf-8", + ) + bad = replay(task, bad_source, directory / "skilleval-bad") + rejected = run_action( + task, bad_source, "measure", directory / "wrong-measure", timeout_seconds=20 + ) + measured = [ + json.loads(p.read_text(encoding="utf-8")) + for side in ("baseline", "candidate") + for p in sorted((run / side).glob("trial-*/result.json")) + ] + consumers = measured + [ + json.loads((directory / path).read_text(encoding="utf-8")) + for path in ( + "correctness/result.json", + "profile/result.json", + "skilleval-good/trial/verification/artifact/result.json", + "skilleval-bad/trial/verification/artifact/result.json", + ) + ] + frozen_identity = task.identity(candidate) + record = { + "task": name, + "adapter": task.manifest["adapter"], + "identity": task.identity(candidate), + "correctness": grade["verdict"], + "profile_status": profile["status"], + "skilleval_good": good["verdict"], + "skilleval_bad": bad["verdict"], + "bad_measurement_status": rejected["status"], + "measurement_trials": len(measured), + "measured_calls": sum(len(c["samples_ms"]) for r in measured for c in r["cases"]), + "source_integrity": evaluation["source_integrity"]["verified"], + "descriptive_workbench_verdict": evaluation["verdict"], + "workbench_run": run.relative_to(output).as_posix(), + "all_consumer_identities_match": all( + all( + r["identity"][key] == frozen_identity[key] + for key in ("task_id", "task_sha256", "grader_sha256", "runtime_sha256") + ) + for r in consumers + ), + "all_workloads_match": all( + r["workloads"] == consumers[0]["workloads"] for r in consumers + ), + "input_sequence": consumers[0]["input_sequence"], + "all_input_sequences_match": all( + r["input_sequence"] == consumers[0]["input_sequence"] for r in consumers + ), + "all_valid_executed_calls_match": all( + r["executed_calls"] == consumers[0]["executed_calls"] + for r in consumers + if r["passed"] + ), + "expected_measured_calls": ( + 2 + * args.repetitions + * len(task.manifest["measurement"]["case_ids"]) + * task.manifest["measurement"]["samples"] + ), + } + records.append(record) + print(json.dumps(record), flush=True) + succeeded = all( + r["correctness"] == r["skilleval_good"] == "PASS" + and r["skilleval_bad"] == "FAIL" + and r["bad_measurement_status"] == "failed" + and r["profile_status"] == "completed" + and r["source_integrity"] + and r["all_consumer_identities_match"] + and r["all_workloads_match"] + and r["all_input_sequences_match"] + and r["all_valid_executed_calls_match"] + and r["measured_calls"] == r["expected_measured_calls"] + and r["measurement_trials"] == 2 * args.repetitions + for r in records + ) + atomic_write_json( + output / "summary.json", + { + "scope": "local CPU integration with authored controls", + "checks_succeeded": succeeded, + "campaign_acceptance": "INCONCLUSIVE", + "agent_or_skill_benefit": "not measured", + "accelerator_qualification": "not run", + "tasks": records, + }, + ) + return 0 if succeeded else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/evidence_inventory.json b/tools/evidence_inventory.json index 6ca2cb388..b0b6cb1b3 100644 --- a/tools/evidence_inventory.json +++ b/tools/evidence_inventory.json @@ -1,6 +1,13 @@ { "version": 1, "bundles": { + "moe-agent-comparison-20261104": { + "kind": "campaign", + "manifests": [{"path": "plan/integrity.json", "schema": "bundle-sha256"}], + "external": [ + "Labelled redacted extracts preserve all 54 attempt scores and available per-case timing samples; original rank logs, condition/session access audits and host-specific launch configuration remain outside this checkout. Their absence limits independent replay of the full historical audit. No device rerun or new TaskPackage acceptance is claimed." + ] + }, "h3-ref2va-ascend-293": { "kind": "campaign", "manifests": [ diff --git a/tools/moe_benchmark.py b/tools/moe_benchmark.py new file mode 100644 index 000000000..0dfbb5d2b --- /dev/null +++ b/tools/moe_benchmark.py @@ -0,0 +1,6 @@ +"""Checkout entry for the packaged two-rank MoE evaluator.""" + +from inferrsi.harness.scenarios.moe import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/npu_benchmark_validate.py b/tools/npu_benchmark_validate.py new file mode 100644 index 000000000..831de7d25 --- /dev/null +++ b/tools/npu_benchmark_validate.py @@ -0,0 +1,208 @@ +"""Qualify a trusted NPU task with controls, profiling and existing Workbench. + +Run inside a caller-owned NPU allocation. All policy values are explicit CLI +inputs, frozen before execution. This is an authored software-control exercise, +not a model-generated optimization campaign or formal NPU SkillEval sandbox. +""" + +import argparse +import json +import math +import shutil +import statistics +import subprocess +from pathlib import Path + +from inferrsi.harness.benchmarks import load_builtin_task +from inferrsi.harness.benchmarks.consumers import run_action +from inferrsi.harness.runner import run_experiment +from inferrsi.infra.artifacts import atomic_write_json + + +def read(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def validation_passed(validation): + result = validation.get("result", {}) + return (validation.get("execution_status") == "completed" + and validation.get("returncode") == 0 + and result.get("status") == "completed" and result.get("passed") is True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--repetitions", type=int, required=True) + parser.add_argument("--min-improvement-pct", type=float, required=True) + parser.add_argument("--max-aa-difference-pct", type=float, required=True) + parser.add_argument("--max-minutes", type=float, required=True) + parser.add_argument("--timeout-seconds", type=float, required=True) + args = parser.parse_args() + if args.repetitions < 2 or any(not math.isfinite(v) or v <= 0 for v in ( + args.min_improvement_pct, args.max_aa_difference_pct, + args.max_minutes, args.timeout_seconds, + )): + parser.error("Use at least two pairs and positive finite policy values") + output = args.output_dir.resolve() + output.mkdir(parents=True, exist_ok=False) + for directory in ("plan", "validator", "patch", "results"): + (output / directory).mkdir() + task = load_builtin_task("rmsnorm-npu") + baseline = task.root / "baseline.py" + candidate = task.root / "candidate.py" + controls = { + "zeros": "return torch.zeros_like(x)", + "missing-epsilon": "return x * torch.rsqrt((x*x).mean(-1,keepdim=True)) * weight", + "missing-weight": "return x * torch.rsqrt((x*x).mean(-1,keepdim=True) + eps)", + "wrong-axis": "return x * torch.rsqrt((x*x).mean(0,keepdim=True) + eps) * weight", + "input-shape": "x.squeeze_(0); " + "return x * torch.rsqrt((x*x).mean(-1,keepdim=True) + eps) * weight", + } + wrong_paths = {} + for name, body in controls.items(): + path = output / "patch" / f"{name}.py" + path.write_text(f"import torch\ndef compute(x, weight, *, eps):\n {body}\n") + wrong_paths[name] = path + # A correct precheck followed by stale output must also fail before scoring. + replay = output / "patch" / "replay.py" + replay.write_text( + "import torch\ncache = {}\n" + "def compute(x, weight, *, eps):\n" + " key = tuple(x.shape)\n" + " if key not in cache:\n" + " cache[key] = x * torch.rsqrt((x*x).mean(-1,keepdim=True) + eps) * weight\n" + " return cache[key]\n" + ) + wrong_paths["replay"] = replay + policy = {key: getattr(args, key) for key in ( + "repetitions", "min_improvement_pct", "max_aa_difference_pct", + "max_minutes", "timeout_seconds", + )} + implementation_paths = {"baseline": baseline, "reference": candidate, **wrong_paths} + frozen = { + "scope": "NPU software controls; no model/skill benefit or service claim", + "source_head": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=args.repo, text=True + ).strip(), + "policy": policy, + "implementations": {name: task.identity(path) + for name, path in implementation_paths.items()}, + } + atomic_write_json(output / "plan" / "frozen.json", frozen) + atomic_write_json(output / "validator" / "task.json", task.manifest) + for path in (baseline, candidate): + shutil.copyfile(path, output / "patch" / path.name) + actions = {} + + def action(name, path, mode): + execution = run_action(task, path, mode, output / "results" / name, + timeout_seconds=args.timeout_seconds) + result = read(execution["result_path"]) + record = {"implementation": next(key for key, value in implementation_paths.items() + if value == path), + "execution_status": execution["status"], + "returncode": execution["returncode"], "result": result} + actions[name] = record + print(json.dumps({"action": name, "status": execution["status"], + "passed": result.get("passed")}), flush=True) + return record + + for name, path in (("baseline", baseline), ("reference", candidate)): + action(name + "-check", path, "check") + action(name + "-profile", path, "profile") + for name, path in wrong_paths.items(): + action(name, path, "measure") + comparisons = {} + # Same implementation in both arms is the explicit no-op/A-A control. + for name, path in (("aa", baseline), ("reference", candidate)): + spec = task.runspec(path, repo=args.repo, output_dir=output / "results" / name, + repetitions=args.repetitions, max_minutes=args.max_minutes, + min_improvement_pct=args.min_improvement_pct) + run = run_experiment(spec, {"benchmark": task.identity(path), + "backend": "ascend", "qualification": "pending"}) + evaluation = read(run / "evaluation.json") + validation_dir = run / "validation" + execution = (read(validation_dir / "execution.json") + if (validation_dir / "execution.json").is_file() else {}) + validated = (read(validation_dir / "result.json") + if (validation_dir / "result.json").is_file() else {}) + validation = {"execution_status": execution.get("status"), + "returncode": execution.get("returncode"), "result": validated} + trials = {side: [read(p) for p in sorted((run / side).glob("trial-*/result.json"))] + for side in ("baseline", "candidate")} + comparisons[name] = {"run": run.relative_to(output).as_posix(), + "evaluation": evaluation, "trials": trials, + "validation": validation} + print(json.dumps({"comparison": name, "verdict": evaluation["verdict"]}), flush=True) + + def correct(name): + record = actions[name] + return (record["execution_status"] == "completed" and record["returncode"] == 0 + and record["result"].get("status") == "completed" + and record["result"].get("passed") is True) + + checks = {"correctness": all(correct(name + "-check") for name in ("baseline", "reference")), + "device_profiles": all(correct(name + "-profile") + for name in ("baseline", "reference")), + "reject_wrong_outputs": all(actions[name]["execution_status"] == "failed" + and actions[name]["returncode"] == 1 + and actions[name]["result"].get("status") == "completed" + and actions[name]["result"].get("passed") is False + and "metrics" not in actions[name]["result"] for name in wrong_paths)} + all_results = [record["result"] for record in actions.values()] + all_results.extend(trial for comparison in comparisons.values() + for trials in comparison["trials"].values() for trial in trials) + all_results.extend(comparison["validation"]["result"] for comparison in comparisons.values()) + environments = [result.get("environment") for result in all_results] + checks["fixed_environment"] = bool(environments) and all( + env and env == environments[0] for env in environments + ) + identity_checks = [record["result"].get("identity") + == frozen["implementations"][record["implementation"]] + for record in actions.values()] + for name, comparison in comparisons.items(): + selected = "baseline" if name == "aa" else "reference" + for side, trials in comparison["trials"].items(): + expected = frozen["implementations"]["baseline" if side == "baseline" else selected] + identity_checks.extend(trial.get("identity") == expected for trial in trials) + identity_checks.append(comparison["validation"]["result"].get("identity") + == frozen["implementations"][selected]) + identity_checks.extend(task.identity(path) == frozen["implementations"][name] + for name, path in implementation_paths.items()) + checks["frozen_identities"] = all(identity_checks) + task.assert_unchanged() + shape_comparisons = {} + for name, comparison in comparisons.items(): + by_shape = {} + valid = validation_passed(comparison["validation"]) and all(len(trials) == args.repetitions + and all(trial.get("passed") and "cases" in trial for trial in trials) + for trials in comparison["trials"].values()) + for case in task.manifest["measurement"]["case_ids"] if valid else []: + medians = {side: statistics.median( + next(c["median_ms"] for c in trial["cases"] if c["id"] == case) + for trial in trials) for side, trials in comparison["trials"].items()} + medians["improvement_pct"] = (1 - medians["candidate"] / medians["baseline"]) * 100 + by_shape[case] = medians + shape_comparisons[name] = by_shape + checks[name + "_qualified"] = valid and bool(by_shape) and ( + all(abs(values["improvement_pct"]) <= args.max_aa_difference_pct + for values in by_shape.values()) and comparison["evaluation"]["verdict"] == "FAIL" + if name == "aa" else all(values["improvement_pct"] >= args.min_improvement_pct + for values in by_shape.values()) + and comparison["evaluation"]["verdict"] == "PASS" + ) + summary = {"scope": frozen["scope"], "frozen": frozen, "checks": checks, + "software_qualification": "PASS" if all(checks.values()) else "INCONCLUSIVE", + "campaign_acceptance": "not established", "agent_benefit": "not measured", + "shape_comparisons": shape_comparisons, + "actions": actions, "comparisons": comparisons} + atomic_write_json(output / "summary.json", summary) + print(json.dumps({"checks": checks, + "software_qualification": summary["software_qualification"]})) + return 0 if all(checks.values()) else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index b5c0fb3e5..be75dfea8 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -12,8 +12,10 @@ from dataclasses import replace from pathlib import Path +from inferrsi.harness import skill_eval_benchmark as benchmark from inferrsi.harness import skill_eval_diagnostic as diagnostic from inferrsi.harness import skill_eval_engineering as engineering +from inferrsi.harness.benchmarks.contract import load_task from inferrsi.harness.skill_evaluation import ( CodexClient, export_bundle, @@ -38,6 +40,7 @@ ROOT = Path(__file__).resolve().parents[1] ROLES = ("baseline", "candidate") +ARTIFACT_ADAPTERS = {engineering.ADAPTER: engineering, benchmark.ADAPTER: benchmark} GIT_READ = ("git", "--no-replace-objects", "--no-lazy-fetch") CONTROLLERS = (GUARD_SOURCE, "tools/skill_eval_compare.py", "tools/skill_eval_pilot.py", "tools/skill_eval_artifact_guidance.py", @@ -188,9 +191,9 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> "adapter", "execution_mode"}: raise ValueError("Unknown configuration field") adapter = config.get("adapter", "audit") - if adapter not in ("audit", diagnostic.ADAPTER, engineering.ADAPTER): + if adapter not in ("audit", diagnostic.ADAPTER, *ARTIFACT_ADAPTERS): raise ValueError("Unknown experiment adapter") - if adapter in (diagnostic.ADAPTER, engineering.ADAPTER): + if adapter in (diagnostic.ADAPTER, *ARTIFACT_ADAPTERS): if config["mode"] != "update": raise ValueError("Diagnostic comparisons require paired old/new bundles (update mode)") if "grader" in config or "schema" in config: @@ -201,7 +204,7 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> if generation_only and (not isinstance(config.get("client"), dict) or config["client"].get("kind") != FULL_CONTEXT_KIND): raise ValueError("Generation-only requires the bounded full-context provider") - if adapter == engineering.ADAPTER and "client" in config and not generation_only: + if adapter in ARTIFACT_ADAPTERS and "client" in config and not generation_only: raise ValueError("Engineering artifacts require local execution, not read/submit provider") if config["mode"] not in ("addition", "update"): raise ValueError("mode must be addition or update") @@ -236,8 +239,8 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> if not tasks: raise ValueError("cases must be nonempty with unique IDs") for task in tasks: - if adapter == engineering.ADAPTER: - engineering.validate_task(task) + if adapter in ARTIFACT_ADAPTERS: + ARTIFACT_ADAPTERS[adapter].validate_task(task) elif adapter == diagnostic.ADAPTER: diagnostic.validate_task(task) else: @@ -326,7 +329,9 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: tasks = read_json(config_path.parent / config["cases"]) is_diagnostic = config.get("adapter") == diagnostic.ADAPTER is_engineering = config.get("adapter") == engineering.ADAPTER - grader = (Path(engineering.__file__) if is_engineering else + is_benchmark = config.get("adapter") == benchmark.ADAPTER + artifact_adapter = ARTIFACT_ADAPTERS.get(config.get("adapter")) + grader = (Path(artifact_adapter.__file__) if artifact_adapter else Path(diagnostic.__file__) if is_diagnostic else config_path.parent / config["grader"] if "grader" in config else ROOT / "tools/skill_eval_verifier.py") @@ -335,7 +340,7 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: require_git_version(subprocess.check_output(("git", "--version"), cwd=repo, text=True)) generation_only = config.get("execution_mode") == engineering.GENERATION_MODE schema = (engineering.GENERATION_SCHEMA if generation_only else - engineering.SCHEMA if is_engineering else + artifact_adapter.SCHEMA if artifact_adapter else diagnostic.SCHEMA if is_diagnostic else read_json(config_path.parent / config["schema"]) if "schema" in config else SCHEMA) if "client" in config: @@ -349,11 +354,19 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: if is_engineering: (output / "plan/starter").mkdir() (output / "plan/starter/parser.py").write_bytes(engineering.STARTER.encode("utf-8")) + if is_benchmark: + for task in tasks: + package = benchmark.task_package(task["benchmark"]["task_id"]) + shutil.copytree(package.root, output / "plan/benchmark-tasks" / task["id"], + ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + starter = output / "plan/starters" / task["id"] / benchmark.ARTIFACT + starter.parent.mkdir(parents=True) + starter.write_bytes(benchmark.starter(task)) arms = {} with tempfile.TemporaryDirectory(prefix="se-") as temporary: for role, short in zip(ROLES, ("b", "c")): selected = config["snapshots"][role] - if is_diagnostic or is_engineering: + if is_diagnostic or artifact_adapter: closure = selected_snapshot( repo, selected["git_revision"], config["skill_root"], config["skill"], Path(temporary) / short, bundle_revision=selected.get("bundle_revision")) @@ -393,7 +406,7 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: "grader": files["validator/verify.py"]}, "same_bundle": before is not None and before["revision"] == after["revision"], "prompt": (engineering.GENERATION_PROMPT if generation_only else - engineering.PROMPT if is_engineering else + artifact_adapter.PROMPT if artifact_adapter else diagnostic.PROMPT if is_diagnostic else PROMPT), "files": files, "controllers": {name: sha256(ROOT / name) for name in CONTROLLERS}, "retry_policy": "none", "cost": None, @@ -403,14 +416,25 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: } if is_diagnostic: plan["adapter"] = diagnostic.ADAPTER - if is_engineering: - plan["adapter"] = engineering.ADAPTER - name = "src/inferrsi/harness/skill_eval_engineering.py" - plan["controllers"][name] = sha256(ROOT / name) + if artifact_adapter: + plan["adapter"] = artifact_adapter.ADAPTER local_source = "tools/skill_eval_local.py" plan["controllers"][local_source] = sha256(ROOT / local_source) plan["isolation"] += (" Artifact replay requires bubblewrap namespaces and fails closed. " - "Public tests, not blind evaluation; reconstructed CPU fixture.") + "Public CPU development checks, not blind evaluation.") + if is_benchmark: + dependencies = [ROOT / "src/inferrsi/harness/skill_eval_benchmark.py"] + dependencies += list((ROOT / "src/inferrsi/harness/benchmarks").glob("*.py")) + for task in tasks: + package = benchmark.task_package(task["benchmark"]["task_id"]) + dependencies += [p for p in package.root.rglob("*") + if p.is_file() and "__pycache__" not in p.parts + and p.suffix != ".pyc"] + for path in dependencies: + plan["controllers"][path.relative_to(ROOT).as_posix()] = sha256(path) + plan["isolation"] += ( + " Independent oracle stays in the parent; only inputs and implementation enter " + "the NumPy artifact sandbox. Correctness only; no performance or skill-benefit claim.") if "client" in config: client = AnthropicAgentClient(config["model"]["name"], config["client"], config["limits"]["timeout_seconds"], @@ -500,6 +524,20 @@ def load_plan(output: Path) -> dict: for name, path in frozen_exam.items(): if plan["exam"].get(name) != plan["files"].get(path): raise ValueError(f"Frozen exam hash disagrees with the frozen file list: {name}") + if plan.get("adapter") == benchmark.ADAPTER: + if (plan["mode"] != "update" or "client" in plan or "execution_mode" in plan + or plan["exam"]["grader"] != sha256(Path(benchmark.__file__)) + or read_json(output / "plan/schema.json") != benchmark.SCHEMA + or plan["prompt"] != benchmark.PROMPT): + raise ValueError("Frozen benchmark adapter contract changed") + for task in tasks: + benchmark.validate_task(task) + package = load_task(output / "plan/benchmark-tasks" / task["id"]) + if package.task_sha256 != task["benchmark"]["task_sha256"]: + raise ValueError("Frozen benchmark package disagrees with task identity") + starter = output / "plan/starters" / task["id"] / benchmark.ARTIFACT + if starter.read_bytes() != benchmark.starter(task): + raise ValueError("Frozen benchmark starter changed") return plan @@ -579,7 +617,7 @@ def execute(output: Path, authorization: str, *, client=None, stop_on_unknown_usage = True if plan.get("execution_mode") == engineering.GENERATION_MODE and "client" not in plan: raise ValueError("Generation-only requires the frozen bounded provider") - if plan.get("adapter") == engineering.ADAPTER and client is None: + if plan.get("adapter") in ARTIFACT_ADAPTERS and client is None: raise ValueError("Engineering execution requires a verified confined local client; " "the default danger-full-access Codex client is not qualified") if client is None: @@ -613,9 +651,12 @@ def execute(output: Path, authorization: str, *, client=None, with tempfile.TemporaryDirectory(prefix="se-trial-") as temporary: workspace = Path(temporary) write_json(workspace / "task.json", task) - is_engineering = plan.get("adapter") == engineering.ADAPTER - if is_engineering: + artifact_adapter = ARTIFACT_ADAPTERS.get(plan.get("adapter")) + if artifact_adapter is engineering: shutil.copyfile(output / "plan/starter/parser.py", workspace / "parser.py") + elif artifact_adapter is benchmark: + shutil.copyfile(output / "plan/starters" / task["id"] / benchmark.ARTIFACT, + workspace / benchmark.ARTIFACT) prompt = plan["prompt"] + "\nTask operation: " + task["request"] if bundle is not None: folder = ".agents/skills" if task["split"] == "discovery" else "knowledge" @@ -627,7 +668,7 @@ def execute(output: Path, authorization: str, *, client=None, directory, output / "plan/schema.json", output / "validator/verify.py", prompt, deadline, timeout_seconds=plan["limits"]["timeout_seconds"], - engineering_artifact=is_engineering, + artifact_adapter=artifact_adapter, generation_only=(plan.get("execution_mode") == engineering.GENERATION_MODE)) result["evidence_sha256"] = {name: sha256(directory / name) for name in EVIDENCE @@ -656,6 +697,9 @@ def execute(output: Path, authorization: str, *, client=None, def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_mode=None) -> dict: + artifact_adapter = ARTIFACT_ADAPTERS.get(expected_adapter) + artifact_name = (getattr(artifact_adapter, "ARTIFACT", "parser.py") + if artifact_adapter else "parser.py") directory = trial_directory(output, entry) path = directory / "result.json" if not path.is_file(): @@ -664,10 +708,10 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m "evaluation_error": "interrupted_attempt" if interrupted else None, "missing": not interrupted, "checks": None, "output": None, "evidence": {}, "elapsed_seconds": None, "observations": {"usage": None}} - if expected_adapter == engineering.ADAPTER: + if artifact_adapter: # A retained tree is untrusted input too: reject special/symlink evidence # before legacy report readers open it. Producers must already be stopped. - names = ("result.json", "receipt.json", "answer.json", "parser.py", + names = ("result.json", "receipt.json", "answer.json", artifact_name, "author-execution.json", *EVIDENCE) try: for name in names: @@ -702,6 +746,9 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m # Select report fields; raw commands and private workspace paths stay in local artifacts. selected = {key: record[key] for key in ("task_id", "condition", "repeat", "split", "verdict", "checks", "elapsed_seconds", "observations", "inputs_unchanged")} + if artifact_adapter is benchmark: + selected["artifact_kind"] = record.get("artifact_kind") + selected["artifact_sha256"] = record.get("artifact_sha256") # Legacy local imports used [] for unparsed logs, not an observed zero. unparsed_author_log = (record.get("execution_state") == "author_reported" and record["observations"].get("completed_turns") is None) @@ -713,9 +760,8 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m } record_error = record.get("evaluation_error") # Artifact integrity failures take precedence over a replayable host error. - error = None if expected_adapter == engineering.ADAPTER else record_error - if (expected_adapter == engineering.ADAPTER - and record.get("artifact_kind") != engineering.ADAPTER): + error = None if artifact_adapter else record_error + if (artifact_adapter and record.get("artifact_kind") != expected_adapter): error = error or "missing_retained_artifact_identity" if execution_mode == engineering.GENERATION_MODE: if record.get("execution_mode") != execution_mode: @@ -757,14 +803,14 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m error = error or "retained_evidence_hash_mismatch" elif name == "author-execution.json": evidence[name] = artifact.relative_to(output).as_posix() - if record.get("artifact_kind") == engineering.ADAPTER and record.get("artifact_sha256") is None: + if artifact_adapter and record.get("artifact_sha256") is None: error = error or "missing_retained_artifact" if record.get("artifact_sha256") is not None: try: artifact_answer = json.loads(data) if data is not None else None except ValueError: artifact_answer = None - artifact = directory / "parser.py" + artifact = directory / artifact_name if artifact.is_symlink() or not artifact.is_file(): error = error or "missing_retained_artifact" elif sha256(artifact) != record["artifact_sha256"]: @@ -773,7 +819,7 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m or artifact_answer.get("artifact_source") != artifact.read_bytes().decode("utf-8")): error = error or "retained_artifact_disagrees_with_answer" else: - evidence["parser.py"] = artifact.relative_to(output).as_posix() + evidence[artifact_name] = artifact.relative_to(output).as_posix() error = error or record_error if (error or record["execution_state"] not in ("completed", "author_reported") or not record["inputs_unchanged"]): @@ -868,14 +914,14 @@ def read_report(output: Path, *, replay_artifacts: bool = True) -> dict: "fix", "regression", "unchanged_correct", "unchanged_incorrect", "missing_pair", "inconclusive", *( ("artifact_improved", "artifact_regressed", "artifacts_both_pass", "artifacts_both_fail") - if plan.get("adapter") == engineering.ADAPTER else ()))}, + if plan.get("adapter") in ARTIFACT_ADAPTERS else ()))}, "discovery": discovery, "discovery_conditions": condition_summary(discovery), "status": status, "claim": "Pipeline results only; fixed-output offline clients do not establish " "skill efficacy. Discovery is separate. Unknown usage and cost remain unknown.", } - if plan.get("adapter") == engineering.ADAPTER: - result["adapter"] = engineering.ADAPTER + if plan.get("adapter") in ARTIFACT_ADAPTERS: + result["adapter"] = plan["adapter"] if any(trial.get("author_reported_execution") for trial in trials): captured = sum(bool(trial.get("author_reported_execution")) for trial in trials) result["status"] = {"execution_state": "author_reported_unverified", @@ -887,6 +933,15 @@ def read_report(output: Path, *, replay_artifacts: bool = True) -> dict: "usage and costs are self-reported, not independently authenticated. Public tests " "may be overfit. Paired artifact differences do not prove skill causality, original " "engine behavior, hardware efficacy or whole-PR acceptance. Reviewer decides merge.") + if plan.get("adapter") == benchmark.ADAPTER: + result["claim"] = ( + "Shared CPU benchmark artifact correctness only. The trusted parent checks actual " + "outputs with the shared oracle; candidate source runs only in bubblewrap. " + "Public development inputs are not holdout. Authored controls and paired artifact " + "differences do not establish skill causality, reliable performance ranking, " + "hardware qualification or service gains. Author-reported provenance/usage remain " + "unverified; absent usage and cost remain unknown.") + result["benchmark_tasks"] = read_json(output / "plan/tasks.json") if plan.get("execution_mode") == engineering.GENERATION_MODE: result["execution_mode"] = engineering.GENERATION_MODE result["claim"] = ( @@ -917,6 +972,7 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: sha256(ROOT / "tools/skill_eval_verifier.py"): verify, sha256(Path(diagnostic.__file__)): diagnostic.verify, sha256(Path(engineering.__file__)): engineering.verify, + sha256(Path(benchmark.__file__)): benchmark.verify, }.get(plan["exam"]["grader"]) if trusted_verifier is None: for trial in trials: @@ -931,7 +987,7 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: trial["evaluation_error"] = "unverified_custom_grader" return for trial in trials: - manual_artifact = (trusted_verifier is engineering.verify + manual_artifact = (trusted_verifier in (engineering.verify, benchmark.verify) and trial.get("author_reported_execution") and trial.get("evaluation_error") in ( None, "reported_agent_failure", "artifact_sandbox_unavailable", @@ -950,7 +1006,8 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: if trial["verdict"] not in ("PASS", "FAIL") and not replay_artifact: continue try: - checks = trusted_verifier(tasks[trial["task_id"]], json.loads(trial["output"])) + verified = trusted_verifier(tasks[trial["task_id"]], json.loads(trial["output"])) + checks = {key: verified[key] for key in ("passed", "assertions")} except engineering.SandboxUnavailable: trial["verdict"] = "INCONCLUSIVE" trial["grader_trust"] = "unverified" @@ -972,6 +1029,8 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: # without rewriting capture failure or upgrading pair comparability. trial["artifact_checks"] = checks trial["artifact_verdict"] = "PASS" if checks["passed"] else "FAIL" + if trusted_verifier is benchmark.verify and "benchmark" in verified: + trial["benchmark"] = verified["benchmark"] trial["grader_trust"] = "reverified" diff --git a/tools/skill_eval_local.py b/tools/skill_eval_local.py index 44d85a222..0b26aa8ed 100644 --- a/tools/skill_eval_local.py +++ b/tools/skill_eval_local.py @@ -13,18 +13,27 @@ from pathlib import Path from inferrsi.harness import skill_eval_engineering as engineering -from inferrsi.harness.skill_evaluation import normalize_usage +from inferrsi.harness.skill_evaluation import normalize_usage, valid_checks from inferrsi.infra.artifacts import sha256, write_json +def _adapter(plan): + if plan.get("adapter") == engineering.ADAPTER: + return engineering + from inferrsi.harness import skill_eval_benchmark as benchmark + + if plan.get("adapter") == benchmark.ADAPTER: + return benchmark + raise ValueError("Local artifact handoff requires a supported engineering artifact adapter") + + def _selection(output, authorization, task_id, condition, repeat, current_head): from tools import skill_eval_compare as compare if authorization != sha256(output / "plan/freeze.json"): raise ValueError("Local handoff requires the exact frozen plan SHA-256") plan = compare.load_plan(output) - if plan.get("adapter") != engineering.ADAPTER: - raise ValueError("Local artifact handoff is only for the engineering adapter") + _adapter(plan) if current_head != plan["arms"]["candidate"]["git_revision"]: raise ValueError("Current PR head differs from the frozen candidate; revalidate first") selected = [entry for entry in plan["schedule"] if entry["task_id"] == task_id @@ -54,7 +63,7 @@ def _read_regular(path, bound=8 * 1024 * 1024): return data -def _hashes(workspace): +def _hashes(workspace, *, editable="parser.py"): result = {} for path in workspace.rglob("*"): if path.is_symlink(): @@ -62,7 +71,7 @@ def _hashes(workspace): if path.is_dir(): continue name = path.relative_to(workspace).as_posix() - if name != "parser.py": + if name != editable: result[name] = hashlib.sha256(_read_regular(path)).hexdigest() return result @@ -92,6 +101,8 @@ def stage(output: Path, authorization: str, task_id: str, condition: str, repeat workspace: Path, current_head: str): plan, entry, task, directory = _selection( output, authorization, task_id, condition, repeat, current_head) + adapter = _adapter(plan) + artifact_name = getattr(adapter, "ARTIFACT", "parser.py") if directory.exists() or workspace.exists(): raise ValueError("Stage requires fresh trial and workspace; never overwrite an attempt") # Keep writable work separate from retained evidence/validator and the other arm. @@ -101,7 +112,9 @@ def stage(output: Path, authorization: str, task_id: str, condition: str, repeat # Frozen inputs are LF bytes on every host; never apply platform translation. (workspace / "task.json").write_bytes( (json.dumps(task, ensure_ascii=False, indent=2, allow_nan=False) + "\n").encode("utf-8")) - shutil.copyfile(output / "plan/starter/parser.py", workspace / "parser.py") + starter = (output / "plan/starter/parser.py" if adapter is engineering else + output / "plan/starters" / task["id"] / artifact_name) + shutil.copyfile(starter, workspace / artifact_name) bundle = plan["arms"][condition]["bundle"] prompt = _prompt(plan, task, condition) if bundle: @@ -109,11 +122,13 @@ def stage(output: Path, authorization: str, task_id: str, condition: str, repeat (workspace / "INSTRUCTIONS.txt").write_bytes(prompt.encode("utf-8")) directory.mkdir(parents=True) write_json(directory / "local-stage.json", { - "plan_sha256": authorization, "entry": entry, "input_hashes": _hashes(workspace), + "plan_sha256": authorization, "entry": entry, + "input_hashes": _hashes(workspace, editable=artifact_name), "provenance": "author-run; execution not observed by controller", }) return {"entry": entry, "plan_sha256": authorization, - "next": "Run approved local agent; only parser.py is writable. Keep logs separately. " + "next": f"Run approved local agent; only {artifact_name} is writable. " + "Keep logs separately. " "Capture does not authorize or pay for that run."} @@ -121,6 +136,8 @@ def capture(output: Path, authorization: str, task_id: str, condition: str, repe workspace: Path, current_head: str, submission: Path): plan, entry, task, directory = _selection( output, authorization, task_id, condition, repeat, current_head) + adapter = _adapter(plan) + artifact_name = getattr(adapter, "ARTIFACT", "parser.py") stage_record = json.loads(_read_regular(directory / "local-stage.json")) if stage_record["plan_sha256"] != authorization or stage_record["entry"] != entry: raise ValueError("Stage receipt does not match the frozen trial") @@ -153,21 +170,22 @@ def capture(output: Path, authorization: str, task_id: str, condition: str, repe error = None try: expected = _expected_inputs(plan, task, condition) - unchanged = (_hashes(workspace) == expected + unchanged = (_hashes(workspace, editable=artifact_name) == expected and stage_record["input_hashes"] == expected) except ValueError: unchanged = False if not unchanged: error = "inputs_changed" - artifact = workspace / "parser.py" + artifact = workspace / artifact_name source = None artifact_bytes = None - if artifact.is_symlink() or not artifact.is_file() or artifact.stat().st_size > 65536: + if (artifact.is_symlink() or not artifact.is_file() + or artifact.stat().st_size > adapter.MAX_SOURCE_BYTES): error = error or "missing_or_invalid_artifact" else: try: - artifact_bytes = _read_regular(artifact, 65536) - if len(artifact_bytes) > 65536: + artifact_bytes = _read_regular(artifact, adapter.MAX_SOURCE_BYTES) + if len(artifact_bytes) > adapter.MAX_SOURCE_BYTES: raise UnicodeError("artifact grew beyond bound") source = artifact_bytes.decode("utf-8") except (UnicodeError, ValueError): @@ -180,22 +198,28 @@ def capture(output: Path, authorization: str, task_id: str, condition: str, repe (directory / "agent").mkdir() for name in ("stdout.log", "stderr.log"): (directory / "agent" / name).write_bytes(submitted[name]) - answer = {"artifact": "parser.py", "summary": metadata["summary"], + answer = {"artifact": artifact_name, "summary": metadata["summary"], "artifact_source": source} write_json(directory / "answer.json", answer) if source is not None: - (directory / "parser.py").write_bytes(artifact_bytes) - checks = None + (directory / artifact_name).write_bytes(artifact_bytes) + checks, raw_checks = None, None if error is None: try: - checks = engineering.verify(task, answer) - except engineering.SandboxUnavailable: + raw_checks = adapter.verify(task, answer) + except adapter.SandboxUnavailable: error = "artifact_sandbox_unavailable" + else: + proposed = {key: raw_checks.get(key) for key in ("passed", "assertions")} + if valid_checks(proposed): + checks = proposed + else: + error = "invalid_verifier_result" (directory / "verification").mkdir() for name in ("stdout.log", "stderr.log"): (directory / "verification" / name).write_text("", encoding="utf-8") - if checks is not None: - write_json(directory / "verification/result.json", checks) + if raw_checks is not None: + write_json(directory / "verification/result.json", raw_checks) artifact_verdict = None if checks is None else "PASS" if checks["passed"] else "FAIL" if metadata["agent_exit_code"] != 0: error = error or "reported_agent_failure" @@ -211,8 +235,8 @@ def capture(output: Path, authorization: str, task_id: str, condition: str, repe "execution_state": "author_reported", "inputs_unchanged": unchanged, "evaluation_error": error, "input_hashes": stage_record["input_hashes"], "output_sha256": sha256(directory / "answer.json"), - "artifact_kind": engineering.ADAPTER, - "artifact_sha256": sha256(directory / "parser.py") if source is not None else None, + "artifact_kind": adapter.ADAPTER, + "artifact_sha256": sha256(directory / artifact_name) if source is not None else None, "provenance": "author-supplied; agent execution and costs not independently verified", "artifact_verdict": artifact_verdict, "evidence_sha256": {name: sha256(directory / name) for name in compare.EVIDENCE