From 575cff4ab9389bf18f3b688c63b7192224218990 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 22 Oct 2026 17:41:25 +0800 Subject: [PATCH 1/7] config: role-based slow/fast model registry in a three-section layout The config named vendors, not roles: a flat "kimi" block, jev_endpoint / kev_endpoint / jev_endpoint_file / kev_endpoint_file side by side, and _comment fields pretending to be documentation. It also could not hold two brains or two cerebellum at once, so comparing "different brain x different cerebellum" meant restructuring the file by hand for every combination. infra/config.json now has three sections - harness (models, routing, budget), eval (cluster and benchmarks), log (results) - and the _comment fields are gone (explanations belong in the docs, not in the config). Both models are named API registries: harness.slow_model.apis. base_url / api_key / model + knobs harness.fast_model.apis. backend (jev|kev) + endpoint + knobs Each registry has a "default"; the run command picks one by name: --slow-model --fast-model so a comparison run only changes the command line, never the config structure. An unknown name lists the available ones. This stays inside the single-config rule: the registry lives in the one file, and the switch is a flag - no env profiles, no placeholder expansion, no second config file. The role naming continues into the harness, so swapping a vendor does not mean touching the state machine: SessionConfig: kimi_max_tokens / kimi_reasoning_effort / kimi_timeout_seconds / kimi_direct_api -> slow_* jev_timeout_seconds -> fast_timeout_seconds selector_backend -> fast_backend CLI: --selector jev|kev -> --fast-model --kimi-max-tokens -> --slow-max-tokens --kimi-reasoning -> --slow-reasoning KimiClient keeps its name - it really is the Kimi dialect - but the audit provider field is now "slow_model" and the worker-side audit file is slow_model_http.jsonl. The job spec follows the same split: secrets carries slow_model credentials, the fast_model address and the registry auth, while the new harness.models (slow name + fast name) stays outside secrets and therefore outside the masking - every run can be attributed to the model pair that produced it. The budget was hardcoded in rjob_spec (15 steps / 900s) while the batch script actually ran 30 / 1800; both now read harness.budget (30 / 1800) and the command line still overrides a single item. demo/ follows the rename: run.selector_backend -> run.fast_model (plus run.slow_model), jev_calls -> fast_calls, jev_latency_ms_total -> fast_latency_ms_total, and the lanes are "slow model plan/verify" and "fast model ranking" instead of naming vendors. New dev-side module config_schema.py owns the schema (validation + API selection) so rjob.py stops carrying it; the worker side stays at its six files. AGENTS.md loses the reference to the deleted refactor plan and its config-discipline section is rewritten for the new shape. Verification: ruff check src tests demo, pytest (225 passed) and tandem check (16 protocol samples) are green, plus a mock end-to-end smoke through infra/worker.sh (config -> job spec -> run directory). --- AGENTS.md | 33 +++-- DEVIATIONS.md | 25 ++++ README.md | 20 ++- demo/adapter.py | 12 +- demo/metrics.py | 10 +- demo/pages.py | 10 +- demo/player/player-compare.js | 8 +- demo/player/player-fail.js | 2 +- demo/player/player-task.js | 6 +- demo/render.py | 12 +- demo/timeline.py | 6 +- docs/CLUSTER_RUNTIME.md | 84 ++++++----- infra/config.example.json | 137 ++++++++++------- scripts/run_osw2_20x5.sh | 10 +- src/tandem/components.py | 6 +- src/tandem/config.py | 27 ++-- src/tandem/providers.py | 18 ++- src/tandem_eval/__main__.py | 32 ++-- src/tandem_eval/config_schema.py | 194 ++++++++++++++++++++++++ src/tandem_eval/entry.py | 31 ++-- src/tandem_eval/rjob.py | 26 +--- src/tandem_eval/rjob_spec.py | 141 ++++++++++++------ tests/demo/helpers.py | 4 +- tests/demo/test_adapter.py | 3 +- tests/demo/test_timeline.py | 8 +- tests/eval/rjob_fakes.py | 68 +++++++-- tests/eval/test_config.py | 213 ++++++++++++++++----------- tests/eval/test_recorder_contract.py | 20 +++ tests/eval/test_rjob_cleanup.py | 6 +- tests/eval/test_rjob_client.py | 60 ++++++-- tests/harness/test_selector_kev.py | 6 +- tests/harness/test_session.py | 4 +- 32 files changed, 865 insertions(+), 377 deletions(-) create mode 100644 src/tandem_eval/config_schema.py diff --git a/AGENTS.md b/AGENTS.md index 401bf5a..8d9a7f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,10 +1,12 @@ # AGENTS.md — Tandem 开发纪律 -本文件是协作与代码合入的硬约束。与重构计划（`../TANDEM_REFACTOR_PLAN_CN.md`）冲突时，先改计划再改本文件。 +本文件是协作与代码合入的硬约束。此前那份仓库外的重构总计划已删除， +改为 **human in the loop 的开发与重构**：改动前先与维护者确认形态与取舍， +范围与命名以本文件 + `docs/CLUSTER_RUNTIME.md` 为准。 ## 仓库定位 -主体是 **Tandem harness**（快慢双模 CUA，`src/tandem/`）；`src/tandem_eval/` 仅为适配集群 rjob 的测评薄适配层（worker 侧 6 文件 + 开发机侧 rjob.py 生命周期客户端）。 +主体是 **Tandem harness**（快慢双模 CUA，`src/tandem/`）；`src/tandem_eval/` 仅为适配集群 rjob 的测评薄适配层（worker 侧 6 文件 + 开发机侧 rjob.py / rjob_spec.py / rjob_status.py / config_schema.py）。 ## 五条硬纪律（CI/评审强制） @@ -24,9 +26,15 @@ ## 配置纪律 - `infra/config.json` 是**唯一**配置文件（含密钥 → gitignore；模板 `config.example.json` 提交）。 +- **三段结构**：`harness`（慢模型/快模型/路由/预算）、`eval`（集群与基准）、`log`（结果落盘）。 +- **按角色命名，不按厂商命名**：大脑 = `harness.slow_model`，小脑 = `harness.fast_model`； + 厂商（kimi / jev / kev）只作为条目内部的字段出现。 +- 慢模型与快模型都是**命名 API 登记表**（`apis`）：登记多套 API，启动命令用 + `--slow-model` / `--fast-model` 按名字挑（缺省取 `default`），用于比较"不同大脑 × 不同小脑"。 - 所有值为绝对路径；开发机与 worker 内一致（adpc 和 evobox 两卷挂载点两侧相同）。 -- 排序服务 endpoint（JEV/Kev）留空 = 自动从 `services/qwen_jev.json` / `kev.json` 发现文件读取最新地址（worker 侧建 session 时解析）。 -- LLM 凭据（kimi 三件套）和 docker registry auth 内联在 config 中，经 job spec 传入 worker（`code_version.json` 中 run_spec 的 secrets 段自动脱敏为 `***`）。 +- 快模型条目的 endpoint 留空 = 自动从同条目的 `endpoint_file`（`services/qwen_jev.json` / `kev.json`）发现文件读取最新地址（worker 侧建 session 时解析）。 +- 慢模型凭据（base_url/api_key/model）和 docker registry auth 内联在 config 中，经 job spec 传入 worker（`code_version.json` 中 run_spec 的 secrets 段自动脱敏为 `***`）。 +- 预算（max_steps / max_wall_seconds）也归 config，命令行只覆盖单项。 - **禁止**新增任何形式的环境档案/占位符展开/多配置文件。 ## 开发闭环 @@ -49,17 +57,17 @@ - 每个 run 的 `code_version.json` 必须记录 git describe、protocol_version、镜像 digest、run spec（secrets 脱敏）。 - OSW2 任务资产经 `OSWORLD_FILE_BASE_URL` 指向本地 `asset_root`（避免外网下载）；必须在任务模块 import **前**设置（见 osworld.py `_bootstrap_runtime`）。 -## LLM 调用纪律（KimiClient） +## 慢模型调用纪律（KimiClient，慢模型的当前实现） - `reasoning_effort` 默认 `low`（探测 97→22 token）；`off` 映射 `enable_thinking=false`（归零）。 - 空正文（全留在思考通道）→ 下一次重试自动升档（low→medium→high）。 - 叙述文字包裹 JSON → `extract_json_object` 自动抢救（剥围栏 + 首个平衡 `{…}`）。 - 403（TPM 配额限流）→ 指数退避（60/120/240/480s），总计 5 次尝试。 -- `finish_reason=length` → 明确报错提示调大 `kimi_max_tokens`（默认 16384）。 +- `finish_reason=length` → 明确报错提示调大 `slow_max_tokens`（默认 16384）。 -## JEV 排序服务纪律 +## 快模型排序服务纪律 -- `/rank` 要求 `image_path` 是**服务宿主机可读的文件路径**——截图暂存到 evobox 中转目录（`jev_image_stage_write/read`），payload 携带 JEV 宿主机视角路径。 +- `/rank` 要求 `image_path` 是**服务宿主机可读的文件路径**——截图暂存到 evobox 中转目录（`harness.fast_model.image_stage.write/read`），payload 携带服务宿主机视角路径。 - 暂存按内容哈希命名去重（`step_00_.png`），断点无害。 - endpoint 地址随闲时 GPU 重新部署变化 → 留空走自动发现文件（含 `ready`/`status` 就绪位检查）。 @@ -82,9 +90,11 @@ src/tandem/ harness 主体（组件、协议、CLI） src/tandem/protocol/ 消息层协议：schema/ 真源、validate/ 校验、samples/ golden src/tandem_eval/ worker 侧 6 文件（entry/env/runner/recorder/scoring/manifest） + 开发机侧 rjob.py / rjob_spec.py / rjob_status.py 生命周期客户端 + + config_schema.py（config.json 的校验与 API 选取） tests/ protocol/ harness/ eval/ demo/ 四组 infra/worker.sh rjob worker 入口（mock 即跑；真实基准走共享 env.sh+docker） -infra/config.json 唯一配置文件（含密钥 → gitignore；模板 config.example.json 提交） +infra/config.json 唯一配置文件（harness/eval/log 三段；含密钥 → gitignore； + 模板 config.example.json 提交） scripts/ 批量运行脚本（run_osw2_20x5.sh 等） tools/ render_summary.py（demo 占位；完整 studio 后续平移） demo/ 轨迹回放与对比站点（adapter/metrics/frames/render + player/） @@ -97,5 +107,6 @@ docs/ RESULT_DIR_CONTRACT.md（eval↔demo 唯一接口） - 唯一落盘位置 `rjob_logs//`；`obs_stage/`（episode 内工作区）结束即清。 - `events.jsonl` 必须包含 planner/gui(ranking)/cli/verified/terminal 全部事件（增量排水，游标推进）。 -- `llm_calls.jsonl` 只记 Kimi 调用（3 种 role）；JEV/Kev 排序的输入输出记在 `events.jsonl` 的 gui 事件 `ranking` 字段。 -- secrets 段（api_key/registry auth）在 `code_version.json` 的 run_spec 副本中全部脱敏为 `***`。 +- `llm_calls.jsonl` 只记慢模型调用（3 种 role）；快模型排序的输入输出记在 `events.jsonl` 的 gui 事件 `ranking` 字段。 +- secrets 段（api_key/registry auth）在 `code_version.json` 的 run_spec 副本中全部脱敏为 `***`； + 模型身份（`harness.models`：慢模型名 + 快模型名）留在 secrets 之外，保证 run 可归因。 diff --git a/DEVIATIONS.md b/DEVIATIONS.md index 1ef7bfd..be61381 100644 --- a/DEVIATIONS.md +++ b/DEVIATIONS.md @@ -58,3 +58,28 @@ 24. 【重构】**rjob.py 拆成三块**（原 322 行超单模块 300 行纪律）： rjob.py（生命周期客户端）/ rjob_spec.py（job spec 与 submit 参数）/ rjob_status.py（`rjob get` 状态解析）；旧的 `tandem_eval.rjob` 导入路径保持可用。 + +## 配置与命名重构（2026-10-22，human in the loop） + +25. 【重构】**配置 schema 角色化**：`infra/config.json` 从平铺键（`kimi` / `jev_endpoint` / + `kev_endpoint` / `jev_endpoint_file` / `kev_endpoint_file` / `osw1_root` …）改为三段结构 + ——`harness`（慢模型/快模型/路由/预算）、`eval`（集群与基准）、`log`（结果落盘）； + `_comment` 等说明字段删除（说明属于文档，不属于配置）。 + 慢模型与快模型都改为**命名 API 登记表**（`harness.slow_model.apis` / + `harness.fast_model.apis`），启动命令用 `--slow-model <名字>` / `--fast-model <名字>` 挑， + 缺省取各自的 `default`——为"不同大脑 × 不同小脑"的对比实验留出入口。 + CLI 旧参数 `--selector jev|kev` → `--fast-model <名字>`，`--kimi-max-tokens` / + `--kimi-reasoning` → `--slow-max-tokens` / `--slow-reasoning`。 + 默认行为不变（jev + kimi-k3）；但单题预算从 rjob_spec 硬编码的 15/900 改为 + config 里的 `harness.budget`（30/1800，与批量脚本实际用值一致）。 +26. 【重构】**SessionConfig 字段角色化**：`kimi_max_tokens` / `kimi_reasoning_effort` / + `kimi_timeout_seconds` / `kimi_direct_api` → `slow_*`；`jev_timeout_seconds` → + `fast_timeout_seconds`；`selector_backend` → `fast_backend`。 + `KimiClient` 类名保留（它确实是 Kimi 方言的具体实现），但审计日志的 provider 字段 + 由 `"kimi"` 改为 `"slow_model"`，worker 侧审计文件名 `kimi_http.jsonl` → + `slow_model_http.jsonl`。 +27. 【重构】**开发机侧新增 `config_schema.py`**：config 的校验与 API 选取独立成模块 + （worker 侧仍是 6 文件封顶；rjob.py 不再背着 schema 知识）。 + `demo/` 同步角色化：run 文档 `selector_backend` → `fast_model`（新增 `slow_model` 名）， + 排序调用计数 `jev_calls` → `fast_calls`、`jev_latency_ms_total` → `fast_latency_ms_total` + （这两个是 demo 层自算字段，不在结果目录契约里）。 diff --git a/README.md b/README.md index 6c584b2..2f46c2c 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,7 @@ from tandem.session import TandemSession components = build_default( client=kimi_client, # providers.KimiClient endpoint="http://<排序服务>/rank", - config=SessionConfig(selector_backend="jev"), + config=SessionConfig(fast_backend="jev"), ) session = TandemSession(components) session.begin("把 FYP 答辩安排逐条加进日历", None) # None → 慢模型抽取需求 @@ -193,6 +193,10 @@ tandem-eval rjob run --run-id --benchmark osworld2 --tasks \ --repo . --config infra/config.json --poll 60 --timeout 3600 ``` +换大脑 / 换小脑只改启动参数：`infra/config.json` 里登记多套 API（`harness.slow_model.apis` / +`harness.fast_model.apis`），运行时用 `--slow-model <名字> --fast-model <名字>` 挑， +用来比较"不同大脑 × 不同小脑"的性能差距。 + 跑完**自动删除 rjob**——无论成功、失败、超时还是中途异常，都不在集群上留残留； 要保留现场（事后查 `rjob logs`）加 `--keep`。 @@ -207,19 +211,19 @@ tandem-eval rjob run --run-id --benchmark osworld2 --tasks \ | `allow_cli` | `True` | 是否允许 planner 走 CLI 路由 | | `route_policy` | `"auto"` | `auto` / `gui` / `cli` | | `cli_timeout_seconds` | `90` | CLI 超时（协议上限 120） | -| `jev_timeout_seconds` | `300` | 排序服务超时（jev 与 kev 共用） | -| `kimi_max_tokens` | `16384` | 慢模型单次响应上限 | -| `kimi_timeout_seconds` | `120` | 慢模型超时 | -| `kimi_direct_api` | `False` | 直连 API（不经代理） | -| `kimi_reasoning_effort` | `"low"` | `off`（归零思考）/ `low` / `medium` / `high` | -| `selector_backend` | `"jev"` | `jev`（`/rank`，需截图）/ `kev`（System One，纯文本） | +| `fast_timeout_seconds` | `300` | 快模型排序服务超时（jev 与 kev 共用） | +| `slow_max_tokens` | `16384` | 慢模型单次响应上限 | +| `slow_timeout_seconds` | `120` | 慢模型超时 | +| `slow_direct_api` | `False` | 直连 API（不经代理） | +| `slow_reasoning_effort` | `"low"` | `off`（归零思考）/ `low` / `medium` / `high` | +| `fast_backend` | `"jev"` | `jev`（`/rank`，需截图）/ `kev`（System One，纯文本） | 慢模型的调用纪律（`providers.py`）： - 思考强度默认 `low`（实测 97 → 22 token）；空正文（全留在思考通道）→ 下一次重试自动升档； - 叙述文字包裹 JSON → 自动剥围栏取首个平衡 `{…}` 抢救； - 403（TPM 配额限流）→ 指数退避 60/120/240/480s，总计 5 次尝试； -- `finish_reason=length` → 明确报错，提示调大 `kimi_max_tokens`。 +- `finish_reason=length` → 明确报错，提示调大 `slow_max_tokens`。 ## 排序后端 diff --git a/demo/adapter.py b/demo/adapter.py index c28d404..81a32e1 100644 --- a/demo/adapter.py +++ b/demo/adapter.py @@ -7,9 +7,9 @@ 1. 只搬运**模型公开输出**，不生成任何虚构的"思考字幕"； 2. `failed_unscored` 永远显示为"未出分"，不折算成 0 分、不进均分； -3. JEV `/rank` 分数一律标注 "softmax，非校准概率"。 +3. 快模型排序分数一律标注 "softmax，非校准概率"。 -成本一律用 **token 用量 + 调用次数** 表达（成本代理指标）——本仓库没有 Kimi +成本一律用 **token 用量 + 调用次数** 表达（成本代理指标）——本仓库没有慢模型的 计价信息，demo 不臆造货币单价。 聚合口径在 `metrics.py`，HTML 渲染在 `render.py`。 @@ -211,8 +211,8 @@ def load_episode(run_dir: Path, entry: dict[str, Any]) -> dict[str, Any]: "wall_seconds": timing.get("wall_seconds"), "evaluation": evaluation, "cost": cost, - "jev_calls": len(ranking_latency), - "jev_latency_ms_total": round(sum(ranking_latency), 2), + "fast_calls": len(ranking_latency), + "fast_latency_ms_total": round(sum(ranking_latency), 2), "steps": steps, "requirements": requirements, "timeline": timeline, @@ -242,6 +242,7 @@ def load_run(run_dir: Path) -> dict[str, Any]: manifest = read_json(run_dir / "episodes" / "manifest.resolved.json") or {} run_spec = code_version.get("run_spec") or {} harness_config = run_spec.get("harness_config") or {} + models = (run_spec.get("harness") or {}).get("models") or {} entries = manifest.get("tasks") if not isinstance(entries, list): @@ -259,7 +260,8 @@ def load_run(run_dir: Path) -> dict[str, Any]: "protocol_version": code_version.get("protocol_version"), "git_describe": code_version.get("git_describe"), "images": code_version.get("images") or {}, - "selector_backend": harness_config.get("selector_backend", "jev"), + "fast_model": models.get("fast") or harness_config.get("fast_backend", "jev"), + "slow_model": models.get("slow", ""), "budget": run_spec.get("budget") or {}, "summary": { "total": summary.get("total", len(tasks)), diff --git a/demo/metrics.py b/demo/metrics.py index bd9ac06..2542912 100644 --- a/demo/metrics.py +++ b/demo/metrics.py @@ -43,11 +43,11 @@ def aggregate(tasks: list[dict[str, Any]]) -> dict[str, Any]: scored = [t for t in tasks if t["scored"] and t["score"] is not None] walls = [t["wall_seconds"] for t in tasks if isinstance(t["wall_seconds"], (int, float))] cost_total = {key: 0 for key in COST_KEYS} - jev_calls = 0 + fast_calls = 0 for task in tasks: for key in COST_KEYS: cost_total[key] += task["cost"]["total"].get(key, 0) - jev_calls += task.get("jev_calls", 0) + fast_calls += task.get("fast_calls", 0) failures: dict[str, int] = {} for task in tasks: if not task["scored"]: @@ -65,7 +65,7 @@ def aggregate(tasks: list[dict[str, Any]]) -> dict[str, Any]: round(sum(t["wall_seconds"] for t in scored) / len(scored), 2) if scored else None ), "cost": cost_total, - "jev_calls": jev_calls, + "fast_calls": fast_calls, "failures": failures, } @@ -88,7 +88,7 @@ def build_comparison(runs: list[dict[str, Any]]) -> dict[str, Any]: task["cost"]["total"]["prompt_tokens"] + task["cost"]["total"]["completion_tokens"] ), "llm_calls": task["cost"]["total"]["calls"], - "jev_calls": task["jev_calls"], + "fast_calls": task["fast_calls"], } return { "runs": [ @@ -96,7 +96,7 @@ def build_comparison(runs: list[dict[str, Any]]) -> dict[str, Any]: "run_id": run["run_id"], "benchmark": run["benchmark"], "git_describe": run["git_describe"], - "selector_backend": run["selector_backend"], + "fast_model": run["fast_model"], "budget": run["budget"], "totals": run["totals"], } diff --git a/demo/pages.py b/demo/pages.py index f834b72..a8854f0 100644 --- a/demo/pages.py +++ b/demo/pages.py @@ -22,7 +22,7 @@ def _run_card(run: dict[str, Any]) -> str: ) return f"""

{html.escape(run['run_id'])}

-

git {html.escape(str(run['git_describe']))} · selector={html.escape(run['selector_backend'])}

+

git {html.escape(str(run['git_describe']))} · 大脑={html.escape(str(run.get('slow_model', '')))} · 小脑={html.escape(run['fast_model'])}

已出分
{totals['scored']} / {totals['total']}
未出分
{totals['failed_unscored']}
@@ -33,7 +33,7 @@ def _run_card(run: dict[str, Any]) -> str:
输入 token
{totals['cost']['prompt_tokens']:,}
输出 token
{totals['cost']['completion_tokens']:,}
其中思考 token
{totals['cost']['reasoning_tokens']:,}
-
JEV 排序次数
{totals['jev_calls']}
+
快模型排序次数
{totals['fast_calls']}
失败分类
{html.escape(failures)}
""" @@ -178,7 +178,7 @@ def _task_rows(comparison: dict[str, Any]) -> str: {cell['steps']} {cell['tokens']:,} {cell['llm_calls']} - {cell['jev_calls']} + {cell['fast_calls']} """ ) return "".join(rows) @@ -222,7 +222,7 @@ def render_index(runs: list[dict[str, Any]], comparison: dict[str, Any]) -> str:

逐题对比（成本 / 耗时 / 准确率）· 点表头排序

- + {_task_rows(comparison)}
任务run分数状态停止原因耗时步数tokenLLM 调用JEV 排序
耗时步数tokenLLM 调用快模型排序
@@ -244,7 +244,7 @@ def render_compare(comparison: dict[str, Any]) -> str: } options = "".join( f'' + f'（大脑={html.escape(str(run.get("slow_model", "")))} · 小脑={html.escape(run["fast_model"])}）' for run in comparison["runs"] ) return f""" diff --git a/demo/player/player-compare.js b/demo/player/player-compare.js index c2fa197..7f09426 100644 --- a/demo/player/player-compare.js +++ b/demo/player/player-compare.js @@ -65,7 +65,7 @@ "" + delta(ca.wall_seconds, cb.wall_seconds, "s", true) + "" + "" + ca.steps + " / " + cb.steps + "" + "" + ca.llm_calls + " / " + cb.llm_calls + "" + - "" + ca.jev_calls + " / " + cb.jev_calls + "" + + "" + ca.fast_calls + " / " + cb.fast_calls + "" + ""; }); @@ -73,13 +73,13 @@ var tb = runB.totals; function card(run, totals) { return "

" + esc(run.run_id) + "

" + - "

selector=" + esc(run.selector_backend) + + "

大脑=" + esc(run.slow_model) + " · 小脑=" + esc(run.fast_model) + " · git " + esc(run.git_describe) + "

" + "
已出分
" + totals.scored + " / " + totals.total + "
" + "
均分
" + (totals.average_score === null ? "—" : totals.average_score) + "
" + "
token
" + num(totals.cost.prompt_tokens + totals.cost.completion_tokens) + "
" + "
LLM 调用
" + totals.cost.calls + "
" + - "
JEV 排序
" + totals.jev_calls + "
" + + "
快模型排序
" + totals.fast_calls + "
" + "
总耗时
" + totals.wall_seconds_total + "s
" + "
"; } @@ -90,7 +90,7 @@ "任务A 分数B 分数Δ分数" + "A tokenB tokenΔtoken" + "A 耗时B 耗时Δ耗时" + - "步数 A/BLLM A/BJEV A/B" + + "步数 A/BLLM A/B小脑排序 A/B" + "" + rows + ""; } diff --git a/demo/player/player-fail.js b/demo/player/player-fail.js index 0bc0068..6b2dbe8 100644 --- a/demo/player/player-fail.js +++ b/demo/player/player-fail.js @@ -62,7 +62,7 @@ ? String(task.score) : "未出分（不计 0 分）"], ["终止原因", esc(reason)], ["决策步", "0"], - ["JEV 排序", String(task.jev_calls === undefined ? "-" : task.jev_calls) + " 次"], + ["快模型排序", String(task.fast_calls === undefined ? "-" : task.fast_calls) + " 次"], ["模型调用", String(calls.length) + " 次" + (roles.length ? "（" + esc(roles.join("、")) + "）" : "")], ["真实用时", typeof wall === "number" ? wall.toFixed(3) + "s" : "-"] diff --git a/demo/player/player-task.js b/demo/player/player-task.js index f954fec..9985d66 100644 --- a/demo/player/player-task.js +++ b/demo/player/player-task.js @@ -165,9 +165,9 @@ "

合计 " + (total.calls || 0) + " 次调用 · " + num(total.prompt_tokens) + " 输入 token · " + num(total.completion_tokens) + " 输出 token（含 " + num(total.reasoning_tokens) + " 思考 token）· " + - "JEV 排序 " + (task.jev_calls || 0) + " 次 / " + - (typeof task.jev_latency_ms_total === "number" - ? (task.jev_latency_ms_total / 1000).toFixed(1) + "s" : "-") + + "快模型排序 " + (task.fast_calls || 0) + " 次 / " + + (typeof task.fast_latency_ms_total === "number" + ? (task.fast_latency_ms_total / 1000).toFixed(1) + "s" : "-") + " · 端到端 " + secs(task.wall_seconds) + "
成本以 token 用量与调用次数表达，未换算货币单价。

"; } diff --git a/demo/render.py b/demo/render.py index 787108e..6709082 100644 --- a/demo/render.py +++ b/demo/render.py @@ -18,7 +18,7 @@ DISCLOSURE = ( "未出分（failed_unscored）如实标注，不折算 0 分、不计入均分；" - "JEV /rank 分数为 softmax 输出，非校准正确率；成本以 token 用量与调用次数表达，" + "快模型排序分数为 softmax 输出，非校准正确率；成本以 token 用量与调用次数表达，" "未换算货币单价；截图为逐步离散帧，非连续录屏；" "时间轴为实测时刻重建（不估算未记录的时间）。" ) @@ -77,7 +77,7 @@ def display_payload(run: dict[str, Any], task: dict[str, Any], frames: dict[str, "run_id": run["run_id"], "benchmark": run["benchmark"], "git_describe": run["git_describe"], - "selector_backend": run["selector_backend"], + "fast_model": run["fast_model"], }, "task": { "task_id": task["task_id"], @@ -91,8 +91,8 @@ def display_payload(run: dict[str, Any], task: dict[str, Any], frames: dict[str, "exception_type": task["exception_type"], "wall_seconds": task["wall_seconds"], "cost": task["cost"], - "jev_calls": task["jev_calls"], - "jev_latency_ms_total": task["jev_latency_ms_total"], + "fast_calls": task["fast_calls"], + "fast_latency_ms_total": task["fast_latency_ms_total"], "evaluation": task["evaluation"], "steps": steps, "llm_calls": llm_calls, @@ -144,7 +144,7 @@ def _readout(task: dict[str, Any]) -> str:
官方评分 · 准确率{html.escape(score_big)}{html.escape(score_sub)}
token 用量 · 执行成本{html.escape(cost_big)}{html.escape(cost_sub)}
真实用时 · wall-clock{html.escape(time_big)}{html.escape(time_sub)}
-
{len(task['steps'])} 决策步JEV 排序 {task['jev_calls']} 次
+
{len(task['steps'])} 决策步快模型排序 {task['fast_calls']} 次
""" @@ -165,7 +165,7 @@ def render_task_page(run: dict[str, Any], task: dict[str, Any], frames: dict[str ← 总览

OSW2 任务 {html.escape(task['task_id'])}

-

{html.escape(run['run_id'])} · selector={html.escape(run['selector_backend'])} · git {html.escape(str(run['git_describe']))}

+

{html.escape(run['run_id'])} · 大脑={html.escape(str(run.get('slow_model', '')))} · 小脑={html.escape(run['fast_model'])} · git {html.escape(str(run['git_describe']))}

契约 {CONTRACT_VERSION} diff --git a/demo/timeline.py b/demo/timeline.py index b78471d..fce97df 100644 --- a/demo/timeline.py +++ b/demo/timeline.py @@ -8,7 +8,7 @@ | 泳道 | 区间 | 数据来源 | |------|------|----------| -| `plan` | 上一步结束 → Kimi 规划调用结束 | `llm_calls[].time`（响应落盘时刻） | +| `plan` | 上一步结束 → 慢模型规划调用结束 | `llm_calls[].time`（响应落盘时刻） | | `rank` | 规划结束 → 规划结束 + 排序延迟 | `ranking.latency_ms`（服务端实测） | | `act` | 排序结束 → 本步观察采集时刻 | `traj[].wall_time` | | `verify` | 观察时刻 → 验证调用结束 | `llm_calls[].time` | @@ -28,10 +28,10 @@ LANE_VERIFY = "verify" LANES = ( - (LANE_PLAN, "KIMI 规划"), + (LANE_PLAN, "慢模型 规划"), (LANE_RANK, "排序服务"), (LANE_ACT, "动作与观察"), - (LANE_VERIFY, "KIMI 验证"), + (LANE_VERIFY, "慢模型 验证"), ) diff --git a/docs/CLUSTER_RUNTIME.md b/docs/CLUSTER_RUNTIME.md index 5dfcbd5..23afcc7 100644 --- a/docs/CLUSTER_RUNTIME.md +++ b/docs/CLUSTER_RUNTIME.md @@ -13,7 +13,7 @@ worker 入口、共享资源与已知坑。 | 组件 | 位置 | 职责 | |---|---|---| | `src/tandem/` | 仓库主体 | harness 本体，与集群无关（见 README） | -| `src/tandem_eval/` | 薄适配层 | worker 侧 6 文件（entry/env/runner/recorder/scoring/manifest）+ 开发机侧 `rjob.py` 生命周期客户端 | +| `src/tandem_eval/` | 薄适配层 | worker 侧 6 文件（entry/env/runner/recorder/scoring/manifest）+ 开发机侧 `rjob.py` / `rjob_spec.py` / `rjob_status.py` / `config_schema.py` | | `infra/config.json` | 唯一配置 | 全绝对路径、含密钥 → **已 gitignore**，模板 `infra/config.example.json` 提交 | | `infra/worker.sh` | worker 入口 | rjob worker 内被调用，按 benchmark 分 mock / 真机两条路 | | `scripts/` | 批量脚本 | `run_osw2_20x5.sh` 等批量运行 | @@ -28,29 +28,48 @@ worker 入口、共享资源与已知坑。 ## 2. 配置文件 -必填键（`rjob.REQUIRED_CONFIG_KEYS`，缺一个直接报错）： +`infra/config.json` 分三段：**harness**（大脑/小脑/路由/预算）、**eval**（集群与基准）、 +**log**（结果落盘）。模板见 `infra/config.example.json`（真实文件含密钥，已 gitignore）。 + +### harness —— 快慢双模与运行旋钮 | 键 | 说明 | |---|---| -| `repo_root` | 本仓库路径（开发机） | -| `deploy_root` | rsync 部署副本路径（worker 读代码；两侧挂载点一致） | -| `results_root` | 唯一落盘位置，run 目录 = `//` | -| `mounts` | rjob `--mount` 参数（gpfs 源 → 容器内挂载点） | -| `rjob` | `charged_group` / `private_machine` / `cpu` / `memory` / `gpu` / `image` | -| `inner_image` | OSWorld VM 用的 inner 镜像（worker 内 dockerd 拉取） | -| `osw1_root` / `osw1_vm` | OSWorld1 基准根与 qcow2 | -| `osw2_root` / `osw2_vm` | OSWorld2 基准根与 qcow2 | -| `asset_root` | OSW2 任务资产根（`OSWORLD_FILE_BASE_URL` 指向它，避免外网下载） | -| `kimi` | LLM 凭据三件套：`base_url` / `api_key` / `model` | -| `jev_endpoint` | 排序服务地址；**留空 = 自动从发现文件读取**（见 §6） | -| `registry` | docker registry auth（内联传给 worker 写 `~/.docker/config.json`） | - -可选但常用： +| `harness.slow_model.default` | 默认慢模型（大脑）API 名 | +| `harness.slow_model.apis.<名字>` | 慢模型 API 条目：`base_url` / `api_key` / `model` + `max_tokens` / `timeout_seconds` / `reasoning_effort` / `direct_api` | +| `harness.fast_model.default` | 默认快模型（小脑）API 名 | +| `harness.fast_model.apis.<名字>` | 快模型 API 条目：`backend`（`jev` / `kev`）+ `endpoint` / `endpoint_file` + `timeout_seconds` | +| `harness.fast_model.image_stage.write` / `.read` | 截图中转目录的**两侧视角**：写入侧在 adpc，读取侧在 evobox（服务端宿主机要能读到） | +| `harness.route.policy` / `.allow_cli` / `.cli_timeout_seconds` | 路由策略与 CLI 开关 | +| `harness.budget.max_steps` / `.max_wall_seconds` | 单题预算默认值（命令行可覆盖） | + +**慢模型与快模型都是命名 API 登记表**：登记多套大脑/小脑，启动命令用 `--slow-model` / +`--fast-model` 按名字挑（缺省取各自的 `default`），用于横向比较"不同大脑 × 不同小脑"的 +性能差距。名字写错时错误信息会列出全部可选名。 + +### eval —— 集群与基准 | 键 | 说明 | |---|---| -| `jev_endpoint_file` / `kev_endpoint_file` | 排序服务活地址发现文件（部署侧维护） | -| `jev_image_stage_write` / `jev_image_stage_read` | JEV 截图中转目录的**两侧视角**：写入侧在 adpc，读取侧在 evobox（服务端宿主机要能读到） | +| `eval.repo_root` | 本仓库路径（开发机） | +| `eval.deploy_root` | rsync 部署副本路径（worker 读代码；两侧挂载点一致） | +| `eval.mounts` | rjob `--mount` 参数（gpfs 源 → 容器内挂载点） | +| `eval.rjob` | `charged_group` / `private_machine` / `cpu` / `memory` / `gpu` / `image` | +| `eval.inner_image` | OSWorld VM 用的 inner 镜像（worker 内 dockerd 拉取） | +| `eval.benchmarks.osworld1.root` / `.vm` | OSWorld1 基准根与 qcow2 | +| `eval.benchmarks.osworld2.root` / `.vm` | OSWorld2 基准根与 qcow2 | +| `eval.benchmarks.osworld2.asset_root` | OSW2 任务资产根（`OSWORLD_FILE_BASE_URL` 指向它，避免外网下载） | +| `eval.registry` | docker registry auth（内联传给 worker 写 `~/.docker/config.json`） | + +### log —— 结果落盘 + +| 键 | 说明 | +|---|---| +| `log.results_root` | 唯一落盘位置，run 目录 = `//` | +| `log.record.a11y_full` | 是否落每步完整 a11y XML（默认只落摘要） | + +缺项/非法值由 `tandem_eval.config_schema` 统一校验（一次列出全部问题）； +必填路径清单见 `config_schema.REQUIRED_PATHS`。 ## 3. 一次 run 的生命周期 @@ -62,9 +81,10 @@ sync rsync -a --delete 仓库 → deploy_root 排除 .git/ .venv/ __pycache__/ .pytest_cache/ .ruff_cache/ rjob_logs/ job_specs/ （job_specs 排除是必须的：否则并发 run 会删掉在跑任务的 spec） spec 组装 job spec → deploy_root/job_specs/.json - spec 段：env（基准根 / VM / 资产 / 启动等待）/ secrets（kimi / registry / - 排序服务地址）/ jev（截图中转目录）/ harness / budget / record / code - （git describe + 内外镜像） + spec 段：env（基准根 / VM / 资产 / 启动等待）/ secrets（slow_model 凭据 / + fast_model 地址 / registry）/ image_stage（截图中转目录）/ harness（含本次用的 + models：慢模型名 + 快模型名）/ harness_config（行为旋钮）/ budget / record / + code（git describe + 内外镜像） submit rjob submit（先 delete 同名残留做防御性清理） watch 轮询至终态（--poll 间隔，--timeout 上限） fetch 取平台日志补写进同一 run 目录（尽力而为，失败不阻断 cleanup） @@ -98,15 +118,16 @@ cleanup stop（释放资源）+ delete（永久删除 rjob 记录）——默 | 参数 | 说明 | |---|---| | `--run-id` | 必填。run 目录名 = `rjob_logs//` | -| `--benchmark` | `mock` / `osworld1` / `osworld2` / `osworld21` | +| `--benchmark` | `mock` / `osworld1` / `osworld2` | | `--tasks` | 逗号分隔 task id（覆盖 manifest） | | `--limit N` | 取清单前 N 题 | | `--shard I/N` | 把（limit 后的）清单**交错**切成 N 片，本 run 跑第 I 片（并发用） | -| `--selector jev/kev` | 排序后端（默认 jev；kev 走 System One API） | +| `--slow-model NAME` | 慢模型（大脑）API 名，取自 `harness.slow_model.apis`；缺省用 `default` | +| `--fast-model NAME` | 快模型（小脑）API 名，取自 `harness.fast_model.apis`；缺省用 `default` | | `--route-policy` / `--allow-cli` | 路由策略与 CLI 开关 | | `--max-steps` / `--max-wall` | 单题预算（步数 / 墙钟秒） | -| `--kimi-reasoning` | `off` / `low`（默认）/ `medium` / `high` | -| `--kimi-max-tokens` | 慢模型单次响应上限 | +| `--slow-reasoning` | `off` / `low`（默认）/ `medium` / `high` | +| `--slow-max-tokens` | 慢模型单次响应上限 | | `--export-paths` | 逗号分隔的 VM 内交付物路径（finish 时导出） | | `--poll` / `--timeout` | 轮询间隔与总超时 | | `--keep` | 结束后**保留** rjob（不 stop 不 delete），供事后排查；默认跑完即删 | @@ -134,9 +155,6 @@ tandem-eval rjob watch tandem- --run-id --config infra/config.j 5. 用 `$OSWORLD_PYTHON` 跑 `tandem_eval entry`（Pillow / easyocr 已在冻结 venv 内）， 跑前先做一次依赖自检。 -OSW2.1 使用 `osw21_shared_env`、`osw21_proxy` 和 `osw21_inner_image`， -直接拉取固定版本镜像，不覆盖 `latest`；冻结 venv 只检查 Pillow，不要求 easyocr。 - worker 侧 `tandem_eval entry --spec ` 是实际执行体：起 VM → 跑任务 → 录事件 → 评分 → 写 run 目录。环境生命周期、预算强制、落盘都在这一层，harness 只管决策。 @@ -165,13 +183,13 @@ worker 侧 `tandem_eval entry --spec ` 是实际执行体：起 VM → 跑 JEV / Kev 部署在闲时 GPU，**地址随重新部署变化**。因此： -- 配置里 `jev_endpoint` / `kev_endpoint` **留空** = worker 侧建 session 时从发现文件 - （`services/qwen_jev.json` / `services/kev.json`）读取最新地址； +- 配置里快模型条目的 `endpoint` **留空** = worker 侧建 session 时从同条目的 + `endpoint_file`（`services/qwen_jev.json` / `services/kev.json`）读取最新地址； - 发现文件带 `ready` / `status` 就绪位检查，并有双视角符号链接兜底； - 固定地址时直接写配置值，不走发现。 JEV 的 `/rank` 要求 `image_path` 是**服务宿主机可读的文件路径**。截图本体在 adpc， -因此由 harness 暂存到 evobox 中转目录（`jev_image_stage_write` → `jev_image_stage_read`）， +因此由 harness 暂存到 evobox 中转目录（`harness.fast_model.image_stage.write` → `.read`）， payload 带 JEV 宿主机视角路径。暂存按内容哈希命名去重（`step_00_.png`），断点无害。 ## 7. 批量运行 @@ -223,6 +241,4 @@ worker 侧各段日志： 真机 smoke： - `--benchmark mock` 走 mock 链路（不起 VM、不碰共享资源），可本地跑； -- 真实基准走 OSW1 / OSW2 / OSW2.1，按本文 §3–§7。 - -上游重构总计划：`../../TANDEM_REFACTOR_PLAN_CN.md`（在仓库外，与本仓库同级目录的上级）。 +- 真实基准走 OSW1 / OSW2，按本文 §3–§7。 diff --git a/infra/config.example.json b/infra/config.example.json index 48904bf..868fddf 100644 --- a/infra/config.example.json +++ b/infra/config.example.json @@ -1,60 +1,95 @@ { - "_comment": "模板：复制为 infra/config.json 并填入真实值（该文件已 gitignore）。", - "repo_root": "/mnt/shared-storage-user/adpc-share/chenxinquan/fastslow/tandem", - "deploy_root": "/mnt/shared-storage-user/adpc-share/chenxinquan/fastslow/tandem-deploy", - "results_root": "/mnt/shared-storage-user/adpc-share/chenxinquan/fastslow/tandem/rjob_logs", - "mounts": [ - { - "source": "gpfs://gpfs2/evobox-share-gpfs2", - "target": "/mnt/shared-storage-user/evobox-share-gpfs2" + "harness": { + "slow_model": { + "default": "kimi-k3", + "apis": { + "kimi-k3": { + "base_url": "https://your-slow-model-endpoint/v1", + "api_key": "sk-xxx", + "model": "kimi-k3", + "max_tokens": 16384, + "timeout_seconds": 120, + "reasoning_effort": "low", + "direct_api": false + } + } + }, + "fast_model": { + "default": "jev", + "apis": { + "jev": { + "backend": "jev", + "endpoint": "", + "endpoint_file": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/services/qwen_jev.json", + "timeout_seconds": 300 + }, + "kev": { + "backend": "kev", + "endpoint": "", + "endpoint_file": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/services/kev.json", + "timeout_seconds": 300 + } + }, + "image_stage": { + "write": "/mnt/shared-storage-user/evobox-share-gpfs2/chenxinquan/tandem_jev_images", + "read": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/chenxinquan/tandem_jev_images" + } }, - { - "source": "gpfs://gpfs2/adpc-share", - "target": "/mnt/shared-storage-user/adpc-share" + "route": { + "policy": "auto", + "allow_cli": true, + "cli_timeout_seconds": 90 }, - { - "source": "gpfs://gpfs2/evobox-share-gpfs2", - "target": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2" + "budget": { + "max_steps": 30, + "max_wall_seconds": 1800 } - ], - "rjob": { - "charged_group": "evobox_proxy", - "private_machine": true, - "cpu": 4, - "memory": 16384, - "gpu": 0, - "image": "registry.h.pjlab.org.cn/ailab-evobox-evobox_cpu/weavebench:osworld-v2-rjob-openclaw-kimi-tasks-uploadfix-20260701" }, - "inner_image": "registry.h.pjlab.org.cn/ailab-evobox-evobox_cpu/weavebench:osworld-v2-inner-osworld-docker-socat-20260712", - "osw1_root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/benchmarks/osworld1", - "osw1_vm": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/vm/osworld1/Ubuntu.qcow2", - "osw2_root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/benchmarks/osworld2", - "osw2_vm": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/vm/osworld2/osworld-v2-ubuntu-x86.qcow2", - "asset_root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld2_0808/assets", - "osw21_root": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/prepared/runtime", - "osw21_vm": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/vm/osworld-v2-ubuntu-x86.qcow2", - "osw21_asset_root": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/assets", - "osw21_shared_env": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/prepared/env.sh", - "_osw21_inner_image_comment": "Fill osw21_inner_image with the release-pinned image available to your registry account; no latest tag.", - "osw21_inner_image": "", - "osw21_proxy": "http://100.100.135.73:3128", - "osw21_volume_size": 64, - "kimi": { - "base_url": "https://your-kimi-endpoint/v1", - "api_key": "sk-xxx", - "model": "kimi-k3" - }, - "jev_endpoint": "", - "kev_endpoint": "", - "registry": { - "auths": { - "registry.h.pjlab.org.cn": { - "auth": "" + "eval": { + "repo_root": "/mnt/shared-storage-user/adpc-share/chenxinquan/fastslow/tandem", + "deploy_root": "/mnt/shared-storage-user/adpc-share/chenxinquan/fastslow/tandem-deploy", + "mounts": [ + { + "source": "gpfs://gpfs2/evobox-share-gpfs2", + "target": "/mnt/shared-storage-user/evobox-share-gpfs2" + }, + { + "source": "gpfs://gpfs2/adpc-share", + "target": "/mnt/shared-storage-user/adpc-share" + } + ], + "rjob": { + "charged_group": "evobox_proxy", + "private_machine": true, + "cpu": 4, + "memory": 16384, + "gpu": 0, + "image": "registry.h.pjlab.org.cn/ailab-evobox-evobox_cpu/weavebench:osworld-v2-rjob-openclaw-kimi-tasks-uploadfix-20260701" + }, + "inner_image": "registry.h.pjlab.org.cn/ailab-evobox-evobox_cpu/weavebench:osworld-v2-inner-osworld-docker-socat-20260712", + "benchmarks": { + "osworld1": { + "root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/benchmarks/osworld1", + "vm": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/vm/osworld1/Ubuntu.qcow2" + }, + "osworld2": { + "root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/benchmarks/osworld2", + "vm": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/vm/osworld2/osworld-v2-ubuntu-x86.qcow2", + "asset_root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld2_0808/assets" + } + }, + "registry": { + "auths": { + "registry.h.pjlab.org.cn": { + "auth": "" + } } } }, - "jev_image_stage_write": "/mnt/shared-storage-user/evobox-share-gpfs2/chenxinquan/tandem_jev_images", - "jev_image_stage_read": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/chenxinquan/tandem_jev_images", - "jev_endpoint_file": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/services/qwen_jev.json", - "kev_endpoint_file": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/services/kev.json" + "log": { + "results_root": "/mnt/shared-storage-user/adpc-share/chenxinquan/fastslow/tandem/rjob_logs", + "record": { + "a11y_full": false + } + } } diff --git a/scripts/run_osw2_20x5.sh b/scripts/run_osw2_20x5.sh index 8295769..093b38d 100755 --- a/scripts/run_osw2_20x5.sh +++ b/scripts/run_osw2_20x5.sh @@ -6,6 +6,9 @@ # 可用环境变量： # LIMIT（默认 20）/ SHARDS（默认 5）/ MAX_STEPS（默认 30）/ MAX_WALL（默认 1800） # TIMEOUT（整片 watch 上限，默认 7200）/ PREFIX（run-id 前缀，默认 osw2-20） +# BENCHMARK（基准，默认 osworld2）/ TASKS（逗号分隔题号，空 = 走清单）/ ROUTE_POLICY（路由，空 = 用 config） +# SLOW_MODEL（慢模型 API 名，默认用 config 的 default）/ FAST_MODEL（快模型 API 名） +# PREFIX 建议带上模型组合（如 osw2-kimi-kev），demo 按 run 对比时一眼能看出谁在跑 # OSW2.1：BENCHMARK=osworld21 LIMIT=56 SHARDS=4 PREFIX=osw21-56 bash scripts/run_osw2_20x5.sh # 单题：再设置 TASKS=004 LIMIT=1 SHARDS=1；ROUTE_POLICY=gui 可限定 GUI 路由。 set -u @@ -21,6 +24,11 @@ TIMEOUT="${TIMEOUT:-7200}" BENCHMARK="${BENCHMARK:-osworld2}" TASKS="${TASKS:-}" PREFIX="${PREFIX:-osw2-20}" +SLOW_MODEL="${SLOW_MODEL:-}" +FAST_MODEL="${FAST_MODEL:-}" +MODEL_FLAGS=() +[ -n "$SLOW_MODEL" ] && MODEL_FLAGS+=(--slow-model "$SLOW_MODEL") +[ -n "$FAST_MODEL" ] && MODEL_FLAGS+=(--fast-model "$FAST_MODEL") BATCH_TS="$(date +%Y%m%d-%H%M%S)" LOG_DIR="rjob_logs/${PREFIX}x${SHARDS}-${BATCH_TS}" mkdir -p "$LOG_DIR" @@ -48,7 +56,7 @@ for i in $(seq 1 "$SHARDS"); do --max-steps "$MAX_STEPS" --max-wall "$MAX_WALL" \ --repo . --config infra/config.json \ --poll 60 --timeout "$TIMEOUT" \ - "${EXTRA[@]}" \ + "${EXTRA[@]}" "${MODEL_FLAGS[@]}" \ > "$LOG_DIR/shard-${i}.log" 2>&1 & pids+=($!) # 错开提交，给第 1 片的 rsync 留出完成窗口 diff --git a/src/tandem/components.py b/src/tandem/components.py index 2f4b67c..3cc0ae8 100644 --- a/src/tandem/components.py +++ b/src/tandem/components.py @@ -33,12 +33,12 @@ class Components: def build_default(*, client, endpoint: str, config: SessionConfig, image_stage: tuple[str, str] | None = None) -> Components: - if getattr(config, "selector_backend", "jev") == "kev": + if config.fast_backend == "kev": from .selector_kev import KevPolicy - selector = KevPolicy(endpoint, timeout_seconds=config.jev_timeout_seconds) + selector = KevPolicy(endpoint, timeout_seconds=config.fast_timeout_seconds) else: - selector = JevOmniPolicy(endpoint, timeout_seconds=config.jev_timeout_seconds, + selector = JevOmniPolicy(endpoint, timeout_seconds=config.fast_timeout_seconds, image_stage=image_stage) return Components( planner=partial( diff --git a/src/tandem/config.py b/src/tandem/config.py index 2f441d8..6aeb453 100644 --- a/src/tandem/config.py +++ b/src/tandem/config.py @@ -1,4 +1,9 @@ -"""harness 侧会话配置。环境生命周期/预算强制/落盘配置属于 tandem_eval（M2），不在此处。""" +"""harness 侧会话配置。环境生命周期/预算强制/落盘配置属于 tandem_eval（M2），不在此处。 + +字段按**角色**命名，不按厂商命名：`slow_*` 是大脑（慢模型）的旋钮， +`fast_*` 是小脑（快模型排序后端）的旋钮。换厂商只换配置里的 API 条目， +不改这里，也不改状态机。 +""" from __future__ import annotations @@ -10,19 +15,19 @@ class SessionConfig: allow_cli: bool = True route_policy: str = "auto" cli_timeout_seconds: int = 90 - jev_timeout_seconds: float = 300 - kimi_max_tokens: int = 16384 - kimi_timeout_seconds: float = 120 - kimi_direct_api: bool = False - kimi_reasoning_effort: str = "low" # off=enable_thinking:false | low/medium/high=reasoning_effort - selector_backend: str = "jev" # "jev"（qwen-jev-choice /rank）| "kev"（Kev System One） + slow_max_tokens: int = 16384 + slow_timeout_seconds: float = 120 + slow_direct_api: bool = False + slow_reasoning_effort: str = "low" # off=enable_thinking:false | low/medium/high=reasoning_effort + fast_backend: str = "jev" # "jev"（qwen-jev-choice /rank）| "kev"（Kev System One） + fast_timeout_seconds: float = 300 def __post_init__(self): if self.route_policy not in {"auto", "gui", "cli"}: raise ValueError("route_policy must be auto, gui or cli") if not 1 <= self.cli_timeout_seconds <= 120: raise ValueError("cli_timeout_seconds must be within 1..120 (协议上限)") - if self.selector_backend not in {"jev", "kev"}: - raise ValueError("selector_backend must be jev or kev") - if self.kimi_reasoning_effort not in {"off", "low", "medium", "high"}: - raise ValueError("kimi_reasoning_effort must be off/low/medium/high") + if self.fast_backend not in {"jev", "kev"}: + raise ValueError("fast_backend must be jev or kev") + if self.slow_reasoning_effort not in {"off", "low", "medium", "high"}: + raise ValueError("slow_reasoning_effort must be off/low/medium/high") diff --git a/src/tandem/providers.py b/src/tandem/providers.py index 5c57f81..6e09cde 100644 --- a/src/tandem/providers.py +++ b/src/tandem/providers.py @@ -1,4 +1,6 @@ -"""Kimi JSON 传输客户端（含脱敏审计）。从旧仓库 providers.py 按契约移植。 +"""慢模型（大脑）的 JSON 传输客户端——OpenAI 兼容 /chat/completions 口径， +当前实现沿用 Kimi 的方言（reasoning_effort / enable_thinking）。含脱敏审计。 +从旧仓库 providers.py 按契约移植。 全量请求/响应落盘（llm_calls.jsonl）属于 M2 的 recorder，此处保持审计日志。""" from __future__ import annotations @@ -63,12 +65,12 @@ def __init__(self, credential_file, config, audit_path: Path, llm_sink=None): def json_request(self, messages, validate, *, role): credentials = self.credentials payload = { - "model": credentials.get("model", "kimi-k3"), + "model": credentials.get("model") or "unknown", "messages": messages, "stream": False, - "max_tokens": self.config.kimi_max_tokens, + "max_tokens": self.config.slow_max_tokens, } - effort = getattr(self.config, "kimi_reasoning_effort", "low") + effort = getattr(self.config, "slow_reasoning_effort", "low") if effort == "off": payload["enable_thinking"] = False else: @@ -89,14 +91,14 @@ def build_request(): opener = ( urllib.request.build_opener(urllib.request.ProxyHandler({})).open - if self.config.kimi_direct_api + if self.config.slow_direct_api else urllib.request.urlopen ) def audit(event, started, attempt, **details): record = { "event": event, - "provider": "kimi", + "provider": "slow_model", "role": role, "attempt": attempt + 1, "time": time.time(), @@ -133,7 +135,7 @@ def redact(text): request = build_request() # 每次重试重建：升档思考强度需重新序列化 audit("request_start", started, attempt) try: - with opener(request, timeout=self.config.kimi_timeout_seconds) as response: + with opener(request, timeout=self.config.slow_timeout_seconds) as response: raw = json.load(response) if self.llm_sink is not None: self.llm_sink( @@ -157,7 +159,7 @@ def redact(text): completion_tokens = (raw.get("usage") or {}).get("completion_tokens") raise ValueError( f"Kimi response truncated by max_tokens (finish_reason=length, " - f"completion_tokens={completion_tokens}); raise kimi_max_tokens" + f"completion_tokens={completion_tokens}); raise slow_max_tokens" ) if not content and raw["choices"][0]["message"].get("reasoning_content"): raise ValueError( diff --git a/src/tandem_eval/__main__.py b/src/tandem_eval/__main__.py index 92f20a9..d9e2e43 100644 --- a/src/tandem_eval/__main__.py +++ b/src/tandem_eval/__main__.py @@ -36,11 +36,14 @@ def main(argv: list[str] | None = None) -> int: run_cmd.add_argument("--allow-cli", choices=["true", "false"], default=None, help="harness 是否允许 CLI 路由") run_cmd.add_argument("--route-policy", choices=["auto", "gui", "cli"], default=None) - run_cmd.add_argument("--selector", choices=["jev", "kev"], default=None, - help="排序后端（kev 需档案提供 kev_endpoint）") - run_cmd.add_argument("--kimi-max-tokens", type=int, default=None) - run_cmd.add_argument("--kimi-reasoning", choices=["off", "low", "medium", "high"], - default=None, help="kimi 思考强度（默认 low）") + run_cmd.add_argument("--slow-model", default=None, metavar="NAME", + help="慢模型（大脑）API 名：config 的 harness.slow_model.apis 里选；默认取 default") + run_cmd.add_argument("--fast-model", default=None, metavar="NAME", + help="快模型（小脑）API 名：config 的 harness.fast_model.apis 里选；默认取 default") + run_cmd.add_argument("--slow-max-tokens", type=int, default=None, + help="慢模型单次响应上限") + run_cmd.add_argument("--slow-reasoning", choices=["off", "low", "medium", "high"], + default=None, help="慢模型思考强度（默认 low）") run_cmd.add_argument("--export-paths", default="", help="逗号分隔的 VM 内交付物路径") run_cmd.add_argument("--poll", type=float, default=60) run_cmd.add_argument("--timeout", type=float, default=None, help="watch 超时秒数") @@ -80,17 +83,22 @@ def main(argv: list[str] | None = None) -> int: manifest["shard"] = {"index": int(index_text), "total": int(total_text)} if manifest: overrides["manifest"] = manifest + models: dict = {} + if args.slow_model: + models["slow"] = args.slow_model + if args.fast_model: + models["fast"] = args.fast_model + if models: + overrides["models"] = models harness_config: dict = {} if args.allow_cli is not None: harness_config["allow_cli"] = args.allow_cli == "true" if args.route_policy: harness_config["route_policy"] = args.route_policy - if args.selector: - harness_config["selector_backend"] = args.selector - if args.kimi_max_tokens: - harness_config["kimi_max_tokens"] = args.kimi_max_tokens - if args.kimi_reasoning: - harness_config["kimi_reasoning_effort"] = args.kimi_reasoning + if args.slow_max_tokens: + harness_config["slow_max_tokens"] = args.slow_max_tokens + if args.slow_reasoning: + harness_config["slow_reasoning_effort"] = args.slow_reasoning if harness_config: overrides["harness_config"] = harness_config budget: dict = {} @@ -119,7 +127,7 @@ def main(argv: list[str] | None = None) -> int: client = RjobClient() status = watch(client, args.job, poll_seconds=args.poll, timeout_seconds=args.timeout) - run_root = Path(config["results_root"]) / args.run_id + run_root = Path(config["log"]["results_root"]) / args.run_id fetch_artifacts(client, args.job, run_root) if args.keep: print(f"已保留 rjob {args.job}（--keep）", file=sys.stderr) diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py new file mode 100644 index 0000000..d2bd676 --- /dev/null +++ b/src/tandem_eval/config_schema.py @@ -0,0 +1,194 @@ +"""infra/config.json 的 schema 解释器：校验 + 按名字选 API。 + +配置分三段，职责互不重叠： + +- `harness`：慢模型（大脑）/ 快模型（小脑）/ 路由 / 预算； +- `eval`：集群与基准（rjob、镜像、基准资产、registry 凭据）； +- `log`：结果落盘（唯一落盘位置 + 记录粒度）。 + +慢模型与快模型都是**命名 API 登记表**（`apis`）：每个条目一个 API， +启动命令用 `--slow-model` / `--fast-model` 按名字挑，默认取各自的 `default`。 +登记多个 API 便于横向比较"不同大脑 × 不同小脑"的性能差距—— +换组合只改命令行，不改配置结构。 + +本模块只管"配置长什么样、怎么选"，不碰提交与运行流程。 +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +#: 必填路径（点号路径，逐段下钻；None / 空串 / 空列表视为缺失） +REQUIRED_PATHS = ( + "harness.slow_model.default", + "harness.fast_model.default", + "harness.route.policy", + "harness.route.allow_cli", + "harness.route.cli_timeout_seconds", + "harness.budget.max_steps", + "harness.budget.max_wall_seconds", + "eval.repo_root", + "eval.deploy_root", + "eval.mounts", + "eval.rjob.image", + "eval.inner_image", + "eval.benchmarks.osworld1.root", + "eval.benchmarks.osworld1.vm", + "eval.benchmarks.osworld2.root", + "eval.benchmarks.osworld2.vm", + "eval.benchmarks.osworld2.asset_root", + "eval.registry", + "log.results_root", +) + +#: 必须是绝对路径的键（纪律：配置里所有路径都是绝对路径，开发机与 worker 内一致） +ABSOLUTE_PATH_PATHS = ( + "eval.repo_root", + "eval.deploy_root", + "log.results_root", + "eval.benchmarks.osworld1.root", + "eval.benchmarks.osworld1.vm", + "eval.benchmarks.osworld2.root", + "eval.benchmarks.osworld2.vm", + "eval.benchmarks.osworld2.asset_root", + "harness.fast_model.image_stage.write", + "harness.fast_model.image_stage.read", +) + +#: 快模型后端协议（决定用哪种排序传输：jev 的 /rank 还是 kev 的 /v1/systemone） +FAST_BACKENDS = ("jev", "kev") + +#: 慢模型每个 API 条目必填的键 +SLOW_API_KEYS = ("base_url", "api_key", "model") + + +class ConfigError(ValueError): + """配置缺项或取值非法（配置问题，不是运行问题）。""" + + +def get_path(config: dict, path: str) -> Any: + """点号路径取值；任一段不存在返回 None。""" + node: Any = config + for part in path.split("."): + if not isinstance(node, dict) or part not in node: + return None + node = node[part] + return node + + +def _is_missing(value: Any) -> bool: + if value is None: + return True + if isinstance(value, str): + return not value.strip() + if isinstance(value, (list, dict)): + return not value + return False + + +def _require(config: dict, path: str) -> list[str]: + return [path] if _is_missing(get_path(config, path)) else [] + + +def _validate_slow_models(config: dict) -> list[str]: + problems: list[str] = [] + apis = get_path(config, "harness.slow_model.apis") + if not isinstance(apis, dict) or not apis: + return ["harness.slow_model.apis"] + for name, api in apis.items(): + for key in SLOW_API_KEYS: + value = api.get(key) if isinstance(api, dict) else None + if _is_missing(value): + problems.append(f"harness.slow_model.apis.{name}.{key}") + default = get_path(config, "harness.slow_model.default") + if default not in apis: + problems.append(f"harness.slow_model.default={default!r}（必须在 apis 里）") + return problems + + +def _validate_fast_models(config: dict) -> list[str]: + problems: list[str] = [] + apis = get_path(config, "harness.fast_model.apis") + if not isinstance(apis, dict) or not apis: + return ["harness.fast_model.apis"] + for name, api in apis.items(): + if not isinstance(api, dict): + problems.append(f"harness.fast_model.apis.{name}") + continue + if api.get("backend") not in FAST_BACKENDS: + problems.append( + f"harness.fast_model.apis.{name}.backend" + f"（应为 {' 或 '.join(FAST_BACKENDS)}）" + ) + endpoint = api.get("endpoint") + endpoint_file = api.get("endpoint_file") + # endpoint 允许留空 = 自动从发现文件读取最新地址（服务闲时部署，地址会变） + if _is_missing(endpoint) and _is_missing(endpoint_file): + problems.append( + f"harness.fast_model.apis.{name}.endpoint / .endpoint_file（至少给一个）" + ) + if isinstance(endpoint, str) and endpoint.strip() and not endpoint.startswith("http"): + problems.append(f"harness.fast_model.apis.{name}.endpoint（应为 http(s) 地址）") + if isinstance(endpoint_file, str) and endpoint_file.strip(): + if not endpoint_file.startswith("/"): + problems.append(f"harness.fast_model.apis.{name}.endpoint_file（必须是绝对路径）") + default = get_path(config, "harness.fast_model.default") + if default not in apis: + problems.append(f"harness.fast_model.default={default!r}（必须在 apis 里）") + return problems + + +def _validate_absolute_paths(config: dict) -> list[str]: + problems: list[str] = [] + for path in ABSOLUTE_PATH_PATHS: + value = get_path(config, path) + if isinstance(value, str) and value.strip() and not value.startswith("/"): + problems.append(f"{path}（必须是绝对路径）") + return problems + + +def validate(config: dict) -> list[str]: + """返回全部问题（空列表 = 配置合法）。""" + problems: list[str] = [] + for path in REQUIRED_PATHS: + problems += _require(config, path) + problems += _validate_slow_models(config) + problems += _validate_fast_models(config) + problems += _validate_absolute_paths(config) + return problems + + +def load_config(path: str | Path) -> dict: + """加载唯一配置文件（infra/config.json）并校验 schema。""" + config = json.loads(Path(path).read_text(encoding="utf-8")) + problems = validate(config) + if problems: + raise ConfigError( + "配置缺少必填键或取值非法：" + + "；".join(problems) + + "（见 infra/config.json）" + ) + return config + + +def api_names(config: dict, kind: str) -> list[str]: + """某个角色（slow_model / fast_model）登记的 API 名字。""" + apis = get_path(config, f"harness.{kind}.apis") + return sorted(apis) if isinstance(apis, dict) else [] + + +def select_api(config: dict, kind: str, name: str | None = None) -> tuple[str, dict]: + """按名字挑一个 API 条目；name 为空取该角色的 default。 + + 返回 (名字, 条目)。名字不存在时列出全部可选名——切换慢/快模型时 + 最常犯的错就是记错名字。 + """ + section = get_path(config, f"harness.{kind}") or {} + apis = section.get("apis") or {} + key = name or section.get("default") + if key not in apis: + choices = "、".join(sorted(apis)) or "（无）" + raise ConfigError(f"harness.{kind}.apis 里没有 {key!r}；可选：{choices}") + return key, dict(apis[key]) diff --git a/src/tandem_eval/entry.py b/src/tandem_eval/entry.py index db89d7c..061f165 100644 --- a/src/tandem_eval/entry.py +++ b/src/tandem_eval/entry.py @@ -67,16 +67,20 @@ def read_endpoint_file(path: str) -> str: ) -def resolve_selector_endpoint(secrets: dict, backend: str) -> str: - """显式 endpoint 优先；为空则从对应发现文件读取（JEV 闲时部署，地址常变）。""" - explicit = secrets.get(backend_key := ("kev_endpoint" if backend == "kev" else "jev_endpoint")) +def resolve_fast_model_endpoint(fast: dict) -> str: + """快模型地址：显式 endpoint 优先；为空则从发现文件读取（闲时部署，地址常变）。 + + fast 段来自 job spec 的 secrets.fast_model（已由提交侧按选中的 API 条目填好）。 + """ + explicit = fast.get("endpoint") if explicit: - return explicit - file_path = secrets.get(f"{backend_key}_file") + return str(explicit).rstrip("/") + file_path = fast.get("endpoint_file") if file_path: return read_endpoint_file(file_path) raise ValueError( - f"selector_backend={backend}：config.json 的 {backend_key} 为空且未配置 {backend_key}_file" + "快模型地址为空且未配置 endpoint_file" + "（infra/config.json → harness.fast_model.apis.<名字>）" ) @@ -160,17 +164,16 @@ def build_session(spec: dict, *, llm_tap: LlmTap | None = None) -> TandemSession from tandem.providers import KimiClient secrets = spec["secrets"] - credentials = dict(secrets["kimi"]) - credentials.setdefault("model", "kimi-k3") - backend = getattr(config, "selector_backend", "jev") - endpoint = resolve_selector_endpoint(secrets, backend) + credentials = dict(secrets["slow_model"]) + endpoint = resolve_fast_model_endpoint(secrets.get("fast_model") or {}) run_root = Path(spec["output_root"]) run_root.mkdir(parents=True, exist_ok=True) - client = KimiClient(credentials, config, run_root / "kimi_http.jsonl", llm_sink=llm_tap) - jev = spec.get("jev") or {} + client = KimiClient(credentials, config, run_root / "slow_model_http.jsonl", + llm_sink=llm_tap) + stage = spec.get("image_stage") or {} image_stage = None - if jev.get("image_stage_write") and jev.get("image_stage_read"): - image_stage = (jev["image_stage_write"], jev["image_stage_read"]) + if stage.get("write") and stage.get("read"): + image_stage = (stage["write"], stage["read"]) components = build_default( client=client, endpoint=endpoint, config=config, image_stage=image_stage, diff --git a/src/tandem_eval/rjob.py b/src/tandem_eval/rjob.py index e27d905..eb4cc47 100644 --- a/src/tandem_eval/rjob.py +++ b/src/tandem_eval/rjob.py @@ -13,12 +13,12 @@ from __future__ import annotations -import json import subprocess import time from pathlib import Path # 以下仍从本模块导出（对外兼容旧导入路径） +from .config_schema import ConfigError, load_config # noqa: F401 （对外兼容旧导入路径） from .rjob_spec import build_job_spec, build_submit_args, job_name # noqa: F401 from .rjob_status import TERMINAL_STATES, parse_status # noqa: F401 @@ -71,24 +71,6 @@ def _subprocess_run(argv: list[str]) -> tuple[int, str]: return result.returncode, (result.stdout + result.stderr) -REQUIRED_CONFIG_KEYS = ( - "repo_root", "deploy_root", "results_root", "mounts", "rjob", "inner_image", - "osw1_root", "osw1_vm", "osw2_root", "osw2_vm", "asset_root", - "kimi", "jev_endpoint", "registry", -) - - -def load_config(path: str | Path) -> dict: - """加载唯一配置文件（infra/config.json）：全绝对路径，开发机与 worker 内一致。""" - config = json.loads(Path(path).read_text(encoding="utf-8")) - missing = [key for key in REQUIRED_CONFIG_KEYS if key not in config] - if missing: - raise RjobError(f"配置缺少必填键: {missing}（见 infra/config.json）") - if not config["rjob"].get("image"): - raise RjobError("配置 rjob.image 不能为空") - return config - - def sync_repo(repo_root: Path, deploy_root_dev: Path) -> None: deploy_root_dev.mkdir(parents=True, exist_ok=True) argv = ["rsync", "-a", "--delete"] @@ -175,7 +157,7 @@ def run_job(*, config: dict, repo_root: Path, run_id: str, sync: bool = True) -> dict: """一站式：sync → spec → submit → watch → fetch → cleanup。 - 唯一落盘位置 = results_root/（= tandem/rjob_logs/）： + 唯一落盘位置 = log.results_root/（= tandem/rjob_logs/）： worker 直接写 run 目录（adpc 挂载），平台日志由 fetch 补写进同一目录。 **收尾默认删 rjob**（delete=True）：submit 之后的任何结局——成功、失败、 @@ -185,7 +167,7 @@ def run_job(*, config: dict, repo_root: Path, run_id: str, spec_overrides = spec_overrides or {} client = client or RjobClient() if sync: - sync_repo(repo_root, Path(config["deploy_root"])) + sync_repo(repo_root, Path(config["eval"]["deploy_root"])) build_job_spec(config=config, run_id=run_id, repo_root=repo_root, spec_overrides=spec_overrides) name = job_name(run_id) @@ -207,7 +189,7 @@ def run_job(*, config: dict, repo_root: Path, run_id: str, result["status"], result["error"] = "error", str(error) finally: # fetch 失败不得阻断 cleanup（否则资源泄漏）；错误记入 result 继续 - run_root = Path(config["results_root"]) / run_id + run_root = Path(config["log"]["results_root"]) / run_id result["run_root"] = str(run_root) try: fetch_artifacts(client, name, run_root) diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py index 84fe476..94bfd6f 100644 --- a/src/tandem_eval/rjob_spec.py +++ b/src/tandem_eval/rjob_spec.py @@ -3,9 +3,13 @@ 把唯一配置文件 `infra/config.json` + 本次 run 的 overrides，翻译成两样东西： - **job spec**：写进部署副本 `job_specs/.json`，worker 侧 `entry` 按它启动 - （环境段 / 凭据段 / JEV 中转段 / harness 段 / 预算 / 镜像与 git describe）； + （环境段 / 凭据段 / 快模型中转段 / harness 段 / 预算 / 镜像与 git describe）； - **submit 参数**：`rjob submit` 的命令行（含 --mount 挂载共享存储）。 +角色与厂商解耦：配置里 `harness.slow_model.apis` / `harness.fast_model.apis` 是 +**命名 API 登记表**，本模块按名字挑出本次 run 用的大脑与小脑（命令行可覆盖）， +再把"凭据 + 地址"拆进 secrets 段、"行为旋钮"拆进 harness_config 段。 + 路径两侧一致、不做视角换算：spec 里的路径都是 worker 容器内可见的挂载点。 """ @@ -15,6 +19,8 @@ import subprocess from pathlib import Path +from .config_schema import select_api + def job_name(run_id: str) -> str: return f"tandem-{run_id}" @@ -31,77 +37,118 @@ def _git_describe(repo_root: Path) -> str: def _env_section(benchmark: str, config: dict) -> dict: - if benchmark == "osworld21": - return {"root": config["osw21_root"], "vm": config["osw21_vm"], - "asset_root": config["osw21_asset_root"], "shared_env": config["osw21_shared_env"], - "proxy": config["osw21_proxy"], "volume_size": config["osw21_volume_size"], - "startup_wait_seconds": 30} + benchmarks = config["eval"]["benchmarks"] if benchmark == "osworld1": - return {"root": config["osw1_root"], "vm": config["osw1_vm"], - "setup_cache_root": str(Path(config["osw1_root"]) / "cache"), + section = benchmarks["osworld1"] + return {"root": section["root"], "vm": section["vm"], + "setup_cache_root": str(Path(section["root"]) / "cache"), "startup_wait_seconds": 30} if benchmark == "osworld2": - return {"root": config["osw2_root"], "vm": config["osw2_vm"], - "asset_root": config["asset_root"], "startup_wait_seconds": 30} + section = benchmarks["osworld2"] + return {"root": section["root"], "vm": section["vm"], + "asset_root": section["asset_root"], "startup_wait_seconds": 30} return {} -def _secrets_section(benchmark: str, config: dict) -> dict: - if benchmark not in {"osworld1", "osworld2", "osworld21"}: +def _secrets_section(benchmark: str, config: dict, slow_api: dict, fast_api: dict) -> dict: + """凭据段：慢模型三件套 + 快模型地址 + registry auth（落盘时整体脱敏）。""" + if benchmark not in {"osworld1", "osworld2"}: return {} - section = { - "kimi": dict(config["kimi"]), - # jev 部署在闲时 GPU，ip/端口随重新部署变化：endpoint 为空时由 worker - # 从 jev_endpoint_file（部署后写入的 {"base_url": ...}）自动发现 - "jev_endpoint": config.get("jev_endpoint", ""), - "registry": config.get("registry"), + return { + "slow_model": {key: slow_api[key] for key in ("base_url", "api_key", "model")}, + # 快模型部署在闲时 GPU，ip/端口随重新部署变化：endpoint 为空时由 worker + # 从 endpoint_file（部署后写入的 {"base_url": ...}）自动发现 + "fast_model": { + "endpoint": fast_api.get("endpoint", ""), + "endpoint_file": fast_api.get("endpoint_file", ""), + }, + "registry": config["eval"].get("registry"), + } + + +def _harness_config_section(config: dict, slow_api: dict, fast_api: dict) -> dict: + """行为旋钮段：SessionConfig 的字段（与 harness 侧 dataclass 一一对应）。""" + route = config["harness"]["route"] + return { + "route_policy": route.get("policy", "auto"), + "allow_cli": bool(route.get("allow_cli", True)), + "cli_timeout_seconds": int(route.get("cli_timeout_seconds", 90)), + "slow_max_tokens": int(slow_api.get("max_tokens", 16384)), + "slow_timeout_seconds": float(slow_api.get("timeout_seconds", 120)), + "slow_reasoning_effort": slow_api.get("reasoning_effort", "low"), + "slow_direct_api": bool(slow_api.get("direct_api", False)), + "fast_backend": fast_api.get("backend", "jev"), + "fast_timeout_seconds": float(fast_api.get("timeout_seconds", 300)), + } + + +def _budget_section(config: dict) -> dict: + budget = config["harness"]["budget"] + return { + "max_steps": int(budget["max_steps"]), + "max_wall_seconds": float(budget["max_wall_seconds"]), } - if config.get("jev_endpoint_file"): - section["jev_endpoint_file"] = config["jev_endpoint_file"] - if config.get("kev_endpoint"): - section["kev_endpoint"] = config["kev_endpoint"] - if config.get("kev_endpoint_file"): - section["kev_endpoint_file"] = config["kev_endpoint_file"] - return section -def _jev_section(config: dict) -> dict: - """JEV 截图中转目录（服务端要求 image_path 在其宿主机可读；截图本体在 adpc， +def _record_section(config: dict) -> dict: + return {"a11y_full": bool(config["log"]["record"].get("a11y_full", False))} + + +def _image_stage_section(config: dict) -> dict: + """截图中转目录（服务端要求 image_path 在其宿主机可读；截图本体在 adpc， 排序前由 harness 暂存到 evobox 并传规范路径）。""" - write_root = config.get("jev_image_stage_write") - read_root = config.get("jev_image_stage_read") + stage = config["harness"]["fast_model"].get("image_stage") or {} + write_root = stage.get("write") + read_root = stage.get("read") if write_root and read_root: - return {"image_stage_write": write_root, "image_stage_read": read_root} + return {"write": write_root, "read": read_root} return {} def build_job_spec(*, config: dict, run_id: str, repo_root: Path, spec_overrides: dict) -> dict: - """组装 job spec 并写入部署副本 job_specs/.json（路径两侧一致，无视角换算）。""" + """组装 job spec 并写入部署副本 job_specs/.json（路径两侧一致，无视角换算）。 + + 本次用哪套大脑/小脑由 overrides["models"] 决定（缺省取配置里的 default）； + 选中的 API 名字会记进 harness.models，run 目录里据此还原"谁在跑"。 + """ benchmark = spec_overrides.get("benchmark", "mock") - if benchmark == "osworld21" and not config.get("osw21_inner_image"): - raise ValueError("osw21_inner_image must identify the release-pinned OSWorld 2.1 runtime image") + overrides = dict(spec_overrides) + requested = dict(overrides.pop("models", {}) or {}) + slow_name, slow_api = select_api(config, "slow_model", requested.get("slow")) + fast_name, fast_api = select_api(config, "fast_model", requested.get("fast")) + + harness_config = _harness_config_section(config, slow_api, fast_api) + harness_config.update(overrides.pop("harness_config", {}) or {}) + budget = _budget_section(config) + budget.update(overrides.pop("budget", {}) or {}) + spec = { "run_id": run_id, - "repo_root": config["deploy_root"], - "output_root": str(Path(config["results_root"]) / run_id), + "repo_root": config["eval"]["deploy_root"], + "output_root": str(Path(config["log"]["results_root"]) / run_id), "benchmark": benchmark, "env": _env_section(benchmark, config), - "secrets": _secrets_section(benchmark, config), - "jev": _jev_section(config), - "harness": {"name": "tandem", "mode": "planner_led"}, - "budget": {"max_steps": 15, "max_wall_seconds": 900}, - "record": {"a11y_full": False}, + "secrets": _secrets_section(benchmark, config, slow_api, fast_api), + "image_stage": _image_stage_section(config), + "harness": { + "name": "tandem", + "mode": "planner_led", + "models": {"slow": slow_name, "fast": fast_name}, + }, + "harness_config": harness_config, + "budget": budget, + "record": _record_section(config), "code": { "git_describe": _git_describe(repo_root), "images": { - "outer": config["rjob"]["image"], - "inner": config["osw21_inner_image"] if benchmark == "osworld21" else config["inner_image"], + "outer": config["eval"]["rjob"]["image"], + "inner": config["eval"]["inner_image"], }, }, - **spec_overrides, + **overrides, } - spec_dir = Path(config["deploy_root"]) / "job_specs" + spec_dir = Path(config["eval"]["deploy_root"]) / "job_specs" spec_dir.mkdir(parents=True, exist_ok=True) (spec_dir / f"{run_id}.json").write_text( json.dumps(spec, ensure_ascii=False, indent=2), encoding="utf-8" @@ -110,7 +157,7 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path, def build_submit_args(*, config: dict, run_id: str) -> list[str]: - rjob_conf = config["rjob"] + rjob_conf = config["eval"]["rjob"] args = [ f"--name={job_name(run_id)}", f"--charged-group={rjob_conf.get('charged_group', 'evobox_proxy')}", @@ -126,9 +173,9 @@ def build_submit_args(*, config: dict, run_id: str) -> list[str]: "--auto-restart", "false", "--privileged", "true", ] - for mount in config.get("mounts", []): + for mount in config["eval"].get("mounts", []): args.append(f"--mount={mount['source']}:{mount['target']}") - deploy = config["deploy_root"] + deploy = config["eval"]["deploy_root"] args += ["--", "bash", f"{deploy}/infra/worker.sh", f"{deploy}/job_specs/{run_id}.json"] return args diff --git a/tests/demo/helpers.py b/tests/demo/helpers.py index 845ac60..bb3dbb4 100644 --- a/tests/demo/helpers.py +++ b/tests/demo/helpers.py @@ -50,7 +50,9 @@ def make_run(root: Path, run_id: str, tasks: list[dict]) -> Path: "images": {"inner": "img:inner"}, "run_spec": { "budget": {"max_steps": 30, "max_wall_seconds": 1800.0}, - "harness_config": {"selector_backend": "kev"}, + "harness": {"name": "tandem", "mode": "planner_led", + "models": {"slow": "kimi-k3", "fast": "kev"}}, + "harness_config": {"fast_backend": "kev"}, }, }, ) diff --git a/tests/demo/test_adapter.py b/tests/demo/test_adapter.py index fda9a4c..adc4ec9 100644 --- a/tests/demo/test_adapter.py +++ b/tests/demo/test_adapter.py @@ -88,7 +88,8 @@ def test_load_run_reports_unscored_without_counting_as_zero(tmp_path): assert totals["failed_unscored"] == 1 assert totals["average_score"] == 0.625 # 未出分不计入均分 assert totals["failures"] == {"ValueError": 1} - assert run["selector_backend"] == "kev" + assert run["fast_model"] == "kev" + assert run["slow_model"] == "kimi-k3" def test_build_comparison_aligns_runs_by_task(tmp_path): diff --git a/tests/demo/test_timeline.py b/tests/demo/test_timeline.py index bdfea60..4f88932 100644 --- a/tests/demo/test_timeline.py +++ b/tests/demo/test_timeline.py @@ -122,7 +122,7 @@ def test_build_chapters_maps_requirements_to_steps_without_duplicates(): assert first["requirement"] == "做 A" -def _cell(score, wall, tokens, steps, calls, jev): +def _cell(score, wall, tokens, steps, calls, fast): return { "status": "scored", "score": score, @@ -133,7 +133,7 @@ def _cell(score, wall, tokens, steps, calls, jev): "steps": steps, "tokens": tokens, "llm_calls": calls, - "jev_calls": jev, + "fast_calls": fast, } @@ -144,7 +144,7 @@ def test_render_index_and_compare_contain_expected_blocks(): "run_id": "run-a", "benchmark": "osworld2", "git_describe": "abc", - "selector_backend": "jev", + "fast_model": "jev", "budget": {}, "totals": {}, }, @@ -152,7 +152,7 @@ def test_render_index_and_compare_contain_expected_blocks(): "run_id": "run-b", "benchmark": "osworld2", "git_describe": "def", - "selector_backend": "kev", + "fast_model": "kev", "budget": {}, "totals": {}, }, diff --git a/tests/eval/rjob_fakes.py b/tests/eval/rjob_fakes.py index 0e84b64..22b9724 100644 --- a/tests/eval/rjob_fakes.py +++ b/tests/eval/rjob_fakes.py @@ -5,6 +5,8 @@ from __future__ import annotations +import copy + class FakeRunner: """假命令执行器：rjob 按子命令路由到脚本化输出（列表逐个弹出，最后一个常驻）。 @@ -50,16 +52,60 @@ def __call__(self, argv): "{'active': 1, 'succeeded': 0, 'failed': 0, 'created_failed': 0, 'pending': 0}" ) +#: 最小配置（结构同 infra/config.json：harness / eval / log 三段） CONFIG = { - "deploy_root": "/dev-deploy", - "results_root": "/dev-results", - "rjob": {"charged_group": "evobox_proxy", "private_machine": True, "cpu": 4, - "memory": 16384, "gpu": 0, "image": "reg.example/outer:1"}, - "mounts": [{"source": "gpfs://gpfs2/share", "target": "/mnt/share"}], - "osw1_root": "/shared/osw1", "osw1_vm": "/shared/vm1.qcow2", - "osw2_root": "/shared/osw2", "osw2_vm": "/shared/vm2.qcow2", - "asset_root": "/shared/assets", "jev_endpoint": "http://10.1.1.1:8891", - "kimi": {"base_url": "https://k.example/v1", "api_key": "sk-x", "model": "kimi-k3"}, - "registry": {"auths": {"registry.h.pjlab.org.cn": {"auth": "xxx"}}}, - "inner_image": "reg.example/inner:1", + "harness": { + "slow_model": { + "default": "kimi-k3", + "apis": { + "kimi-k3": { + "base_url": "https://k.example/v1", "api_key": "sk-x", "model": "kimi-k3", + "max_tokens": 4096, "timeout_seconds": 60, + "reasoning_effort": "low", "direct_api": False, + }, + }, + }, + "fast_model": { + "default": "jev", + "apis": { + "jev": { + "backend": "jev", "endpoint": "http://10.1.1.1:8891", + "endpoint_file": "/shared/services/qwen_jev.json", "timeout_seconds": 30, + }, + "kev": { + "backend": "kev", "endpoint": "", + "endpoint_file": "/shared/services/kev.json", "timeout_seconds": 30, + }, + }, + "image_stage": {"write": "/stage", "read": "/stage-read"}, + }, + "route": {"policy": "auto", "allow_cli": True, "cli_timeout_seconds": 90}, + "budget": {"max_steps": 5, "max_wall_seconds": 60}, + }, + "eval": { + "repo_root": "/dev-repo", + "deploy_root": "/dev-deploy", + "mounts": [{"source": "gpfs://gpfs2/share", "target": "/mnt/share"}], + "rjob": {"charged_group": "evobox_proxy", "private_machine": True, "cpu": 4, + "memory": 16384, "gpu": 0, "image": "reg.example/outer:1"}, + "inner_image": "reg.example/inner:1", + "benchmarks": { + "osworld1": {"root": "/shared/osw1", "vm": "/shared/vm1.qcow2"}, + "osworld2": {"root": "/shared/osw2", "vm": "/shared/vm2.qcow2", + "asset_root": "/shared/assets"}, + }, + "registry": {"auths": {"registry.h.pjlab.org.cn": {"auth": "xxx"}}}, + }, + "log": {"results_root": "/dev-results", "record": {"a11y_full": False}}, } + + +def with_paths(config: dict, *, deploy_root: str | None = None, + results_root: str | None = None) -> dict: + """深拷贝配置并替换 eval.deploy_root / log.results_root（避免共享嵌套 dict）。""" + merged = copy.deepcopy(config) + if deploy_root is not None: + merged["eval"]["deploy_root"] = deploy_root + if results_root is not None: + merged["log"]["results_root"] = results_root + return merged diff --git a/tests/eval/test_config.py b/tests/eval/test_config.py index bba8bae..8138986 100644 --- a/tests/eval/test_config.py +++ b/tests/eval/test_config.py @@ -1,4 +1,10 @@ -"""infra/config.json：唯一配置文件（无占位符、绝对路径、必填键齐全）。""" +"""infra/config.json：唯一配置文件（无占位符、绝对路径、必填键齐全、角色化命名）。 + +守的是三条纪律： +1. 三段结构 harness / eval / log 齐全，且**没有** `_comment` 之类的说明字段； +2. 慢模型（大脑）与快模型（小脑）都是**命名 API 登记表**，default 必须在 apis 里； +3. 路径全绝对、凭据内联、endpoint 可留空走发现文件。 +""" from __future__ import annotations @@ -7,119 +13,166 @@ import pytest +from tandem_eval.config_schema import ConfigError, api_names, select_api from tandem_eval.rjob import load_config CONFIG_PATH = Path(__file__).resolve().parents[2] / "infra" / "config.example.json" -def test_repo_config_loads_and_is_absolute(): +def _walk_keys(node) -> list[str]: + """递归收集所有键名（查有没有混进说明性字段）。""" + if isinstance(node, dict): + return [key for key in node] + [k for value in node.values() for k in _walk_keys(value)] + if isinstance(node, list): + return [k for value in node for k in _walk_keys(value)] + return [] + + +def test_example_config_has_three_sections_and_no_comment_fields(): config = load_config(CONFIG_PATH) - PATH_KEYS = ("repo_root", "deploy_root", "results_root", "osw1_root", "osw1_vm", - "osw2_root", "osw2_vm", "asset_root", "osw21_root", "osw21_vm", - "osw21_asset_root", "osw21_shared_env") + assert set(config) == {"harness", "eval", "log"} + # 说明性字段（_comment 之类）不留在配置里：注释属于文档，不属于配置 + assert [key for key in _walk_keys(config) if key.startswith("_")] == [] + + +def test_models_are_named_api_registries(): + config = load_config(CONFIG_PATH) + slow = config["harness"]["slow_model"] + fast = config["harness"]["fast_model"] + # default 必须指向登记表里的某个条目 + assert slow["default"] in slow["apis"] + assert fast["default"] in fast["apis"] + assert api_names(config, "slow_model") == sorted(slow["apis"]) + assert api_names(config, "fast_model") == sorted(fast["apis"]) + # 慢模型条目是凭据三件套 + 旋钮；快模型条目声明自己的传输协议 + for api in slow["apis"].values(): + assert set(api) >= {"base_url", "api_key", "model"} + for api in fast["apis"].values(): + assert api["backend"] in {"jev", "kev"} + assert api.get("endpoint") or api.get("endpoint_file") + + +def test_paths_are_absolute_and_credentials_inlined(): + config = load_config(CONFIG_PATH) + PATH_KEYS = ( + "eval.repo_root", "eval.deploy_root", "log.results_root", + "eval.benchmarks.osworld1.root", "eval.benchmarks.osworld1.vm", + "eval.benchmarks.osworld2.root", "eval.benchmarks.osworld2.vm", + "eval.benchmarks.osworld2.asset_root", + "harness.fast_model.image_stage.write", "harness.fast_model.image_stage.read", + ) for key in PATH_KEYS: - assert config[key].startswith("/"), f"{key} 必须是绝对路径: {config[key]}" - assert config["inner_image"].startswith("registry.") - # 凭据内联形态：kimi 为 dict、endpoint 为 URL、registry 为 docker config 对象 - assert set(config["kimi"]) >= {"base_url", "api_key", "model"} - # endpoint 允许为空（= 从 *_endpoint_file 发现文件自动解析） - assert isinstance(config["jev_endpoint"], str) - assert config["jev_endpoint_file"].endswith("services/qwen_jev.json") - assert config["kev_endpoint_file"].endswith("services/kev.json") - assert "auths" in config["registry"] - assert config["results_root"].endswith("tandem/rjob_logs") - assert config["repo_root"].endswith("/tandem") - for key in ("osw21_root", "osw21_vm", "osw21_asset_root", "osw21_shared_env"): - assert "/shared_resources/sa_cua/" in config[key] + value = config + for part in key.split("."): + value = value[part] + assert value.startswith("/"), f"{key} 必须是绝对路径: {value}" + assert config["eval"]["inner_image"].startswith("registry.") + assert config["eval"]["rjob"]["image"].startswith("registry.") + assert "auths" in config["eval"]["registry"] + assert config["log"]["results_root"].endswith("tandem/rjob_logs") + assert config["eval"]["repo_root"].endswith("/tandem") def test_mounts_cover_both_volumes_with_user_style_targets(): config = load_config(CONFIG_PATH) - targets = {m["target"] for m in config["mounts"]} + targets = {m["target"] for m in config["eval"]["mounts"]} assert "/mnt/shared-storage-user/evobox-share-gpfs2" in targets assert "/mnt/shared-storage-user/adpc-share" in targets # evobox 卷两侧一致（user 风格），共享资源路径随之 user 风格 - assert config["osw1_root"].startswith("/mnt/shared-storage-user/evobox-share-gpfs2/") - assert config["osw1_vm"].startswith("/mnt/shared-storage-user/evobox-share-gpfs2/") + benchmarks = config["eval"]["benchmarks"] + assert benchmarks["osworld1"]["root"].startswith("/mnt/shared-storage-user/evobox-share-gpfs2/") + assert benchmarks["osworld1"]["vm"].startswith("/mnt/shared-storage-user/evobox-share-gpfs2/") -def test_load_config_rejects_missing_keys(tmp_path): +def test_budget_and_route_come_from_config(): + config = load_config(CONFIG_PATH) + assert config["harness"]["budget"]["max_steps"] > 0 + assert config["harness"]["budget"]["max_wall_seconds"] > 0 + assert config["harness"]["route"]["policy"] in {"auto", "gui", "cli"} + + +def test_load_config_rejects_missing_section(tmp_path): bad = tmp_path / "bad.json" - bad.write_text(json.dumps({"repo_root": "/x"}), encoding="utf-8") - with pytest.raises(Exception, match="必填键"): + bad.write_text(json.dumps({"eval": {}, "log": {}}), encoding="utf-8") + with pytest.raises(ConfigError, match="必填键"): load_config(bad) def test_load_config_rejects_empty_image(tmp_path): config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) - config["rjob"]["image"] = "" + config["eval"]["rjob"]["image"] = "" + bad = tmp_path / "bad.json" + bad.write_text(json.dumps(config), encoding="utf-8") + with pytest.raises(ConfigError, match="image"): + load_config(bad) + + +def test_load_config_rejects_unknown_fast_backend(tmp_path): + config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) + name = next(iter(config["harness"]["fast_model"]["apis"])) + config["harness"]["fast_model"]["apis"][name]["backend"] = "omni" bad = tmp_path / "bad.json" bad.write_text(json.dumps(config), encoding="utf-8") - with pytest.raises(Exception, match="image"): + with pytest.raises(ConfigError, match="backend"): load_config(bad) +def test_select_api_picks_named_entry_or_default(): + config = load_config(CONFIG_PATH) + slow_default, slow_api = select_api(config, "slow_model") + assert slow_default == config["harness"]["slow_model"]["default"] + assert slow_api["model"] + # 按名字挑另一套 API（换大脑/换小脑的入口） + fast_name, fast_api = select_api(config, "fast_model", "kev") + assert fast_name == "kev" + assert fast_api["backend"] == "kev" + + +def test_select_api_error_lists_available_names(): + config = load_config(CONFIG_PATH) + with pytest.raises(ConfigError, match="可选"): + select_api(config, "fast_model", "no-such-api") + + +def test_cli_model_flags_reach_spec_overrides(monkeypatch): + """启动命令按名字切大脑/小脑：--slow-model / --fast-model 进 spec 的 models 段。""" + from tandem_eval.__main__ import main + + captured: dict = {} + + def fake_run_job(**kwargs): + captured.update(kwargs) + return {"status": "succeeded"} + + monkeypatch.setattr("tandem_eval.rjob.run_job", fake_run_job) + code = main(["rjob", "run", "--run-id", "r1", "--benchmark", "mock", + "--config", str(CONFIG_PATH), + "--slow-model", "kimi-k3", "--fast-model", "kev"]) + assert code == 0 + assert captured["spec_overrides"]["models"] == {"slow": "kimi-k3", "fast": "kev"} + + def test_real_config_is_gitignored_with_example_committed(): repo = Path(__file__).resolve().parents[2] ignore = (repo / ".gitignore").read_text(encoding="utf-8") assert "infra/config.json" in ignore, "含密钥的真实配置必须 gitignore" -def test_osworld21_spec_uses_configured_environment_without_changing_osw2(tmp_path): - from tandem_eval.rjob_spec import build_job_spec - - config = load_config(CONFIG_PATH) - config.update(deploy_root=str(tmp_path / "deploy"), osw21_volume_size=80, - osw21_inner_image="registry.example/runtime:2.1") - spec = build_job_spec(config=config, run_id="new", repo_root=tmp_path, - spec_overrides={"benchmark": "osworld21"}) - for key in ("root", "vm", "asset_root", "shared_env", "proxy", "volume_size"): - assert spec["env"][key] == config[f"osw21_{key}"] - assert "setup_bundles" not in spec["env"] - assert spec["code"]["images"]["inner"] == config["osw21_inner_image"] - old = build_job_spec(config=config, run_id="old", repo_root=tmp_path, - spec_overrides={"benchmark": "osworld2"}) - assert old["env"]["root"] == config["osw2_root"] - assert old["code"]["images"]["inner"] == config["inner_image"] - config["osw21_inner_image"] = "" - with pytest.raises(ValueError, match="release-pinned"): - build_job_spec(config=config, run_id="bad", repo_root=tmp_path, - spec_overrides={"benchmark": "osworld21"}) - - -@pytest.mark.parametrize("benchmark", ["osworld1", "osworld2", "osworld21"]) -def test_entry_uses_one_osworld_adapter(tmp_path, monkeypatch, benchmark): - import sys - - from tandem_eval.entry import build_env - from tandem_eval.osworld import OsWorldEnv - - monkeypatch.setattr(sys, "path", sys.path.copy()) - monkeypatch.setenv("OSWORLD_FILE_BASE_URL", "before") - env = build_env({"benchmark": benchmark, "output_root": str(tmp_path / "run"), - "env": {"root": str(tmp_path / "runtime"), "vm": str(tmp_path / "vm"), - "volume_size": 80, "asset_root": str(tmp_path / "assets")}}) - assert type(env) is OsWorldEnv - assert env.task_root == tmp_path / "runtime" - assert env.volume_size == 80 - - -def test_endpoint_discovery_prefers_explicit_then_file(tmp_path, monkeypatch): - from tandem_eval.entry import read_endpoint_file, resolve_selector_endpoint +def test_endpoint_discovery_prefers_explicit_then_file(tmp_path): + from tandem_eval.entry import read_endpoint_file, resolve_fast_model_endpoint f = tmp_path / "endpoint.json" f.write_text(json.dumps({"base_url": "http://10.9.9.9:8891/"}), encoding="utf-8") assert read_endpoint_file(str(f)) == "http://10.9.9.9:8891" # 显式值优先于文件 - assert resolve_selector_endpoint( - {"jev_endpoint": "http://1.2.3.4:1", "jev_endpoint_file": str(f)}, "jev") == "http://1.2.3.4:1" + assert resolve_fast_model_endpoint( + {"endpoint": "http://1.2.3.4:1", "endpoint_file": str(f)}) == "http://1.2.3.4:1" # 为空 → 文件发现 - assert resolve_selector_endpoint({"jev_endpoint": "", "jev_endpoint_file": str(f)}, "jev") \ + assert resolve_fast_model_endpoint({"endpoint": "", "endpoint_file": str(f)}) \ == "http://10.9.9.9:8891" # 两者皆无 → 明确报错 - import pytest as _pytest - with _pytest.raises(ValueError, match="为空且未配置"): - resolve_selector_endpoint({"jev_endpoint": ""}, "jev") + with pytest.raises(ValueError, match="endpoint_file"): + resolve_fast_model_endpoint({"endpoint": ""}) def test_endpoint_file_symlink_view_fallback(tmp_path, monkeypatch): @@ -138,26 +191,14 @@ def test_endpoint_file_symlink_view_fallback(tmp_path, monkeypatch): assert entry_mod.read_endpoint_file(str(canonical_dir / "endpoint.json")) == "http://5.5.5.5:8891" -def test_kev_endpoint_discovery_via_shared_services_file(tmp_path): - """KEV endpoint 为空时同样从 services 目录的发现文件读取。""" - from tandem_eval.entry import resolve_selector_endpoint - - f = tmp_path / "kev_endpoint.json" - f.write_text(json.dumps({"base_url": "http://10.8.8.8:8008/"}), encoding="utf-8") - assert resolve_selector_endpoint( - {"kev_endpoint": "", "kev_endpoint_file": str(f)}, "kev") == "http://10.8.8.8:8008" - - def test_endpoint_file_respects_readiness_flag(tmp_path): """发现文件标记未就绪时快速失败，不调用半启动的服务。""" - import pytest as _pytest - from tandem_eval.entry import read_endpoint_file f = tmp_path / "svc.json" f.write_text(json.dumps({"status": "starting", "ready": False, "base_url": "http://1.1.1.1:1"}), encoding="utf-8") - with _pytest.raises(ValueError, match="未就绪"): + with pytest.raises(ValueError, match="未就绪"): read_endpoint_file(str(f)) f.write_text(json.dumps({"status": "ready", "ready": True, "base_url": "http://2.2.2.2:2"}), encoding="utf-8") diff --git a/tests/eval/test_recorder_contract.py b/tests/eval/test_recorder_contract.py index bb3b230..e0b04d6 100644 --- a/tests/eval/test_recorder_contract.py +++ b/tests/eval/test_recorder_contract.py @@ -76,3 +76,23 @@ def test_heartbeat_and_code_version(tmp_path): write_code_version(tmp_path, {"git_describe": "abc123", "resources": {"schema_version": "v1"}}) info = json.loads((tmp_path / "code_version.json").read_text()) assert info["git_describe"] == "abc123" and info["protocol_version"] == "0.1" + + +def test_run_spec_masks_credentials_but_keeps_model_identity(): + """secrets 段整体脱敏；模型身份留在 secrets 之外——结果目录要能回答"这次谁在跑"。""" + from tandem_eval.entry import _mask_secrets + + spec = { + "secrets": { + "slow_model": {"base_url": "https://x/v1", "api_key": "sk-secret", "model": "kimi-k3"}, + "fast_model": {"endpoint": "http://10.0.0.1:8891", "endpoint_file": "/x/kev.json"}, + "registry": {"auths": {"reg": {"auth": "base64"}}}, + }, + "harness": {"name": "tandem", "mode": "planner_led", + "models": {"slow": "kimi-k3", "fast": "jev"}}, + } + masked = _mask_secrets(spec) + assert masked["secrets"]["slow_model"]["api_key"] == "***" + assert masked["secrets"]["fast_model"]["endpoint"] == "***" + assert masked["secrets"]["registry"]["auths"] == "***" + assert masked["harness"]["models"] == {"slow": "kimi-k3", "fast": "jev"} diff --git a/tests/eval/test_rjob_cleanup.py b/tests/eval/test_rjob_cleanup.py index 67d9ad8..256aa13 100644 --- a/tests/eval/test_rjob_cleanup.py +++ b/tests/eval/test_rjob_cleanup.py @@ -11,7 +11,7 @@ from tandem_eval.rjob import RjobClient, cleanup, run_job -from .rjob_fakes import CONFIG, REAL_SUCCEEDED, FakeRunner +from .rjob_fakes import CONFIG, REAL_SUCCEEDED, FakeRunner, with_paths def _delete_calls(runner: FakeRunner) -> list[int]: @@ -23,8 +23,8 @@ def _submit_index(runner: FakeRunner) -> int: def _run_config(tmp_path) -> dict: - return dict(CONFIG, results_root=str(tmp_path / "rjob_logs"), - deploy_root=str(tmp_path / "deploy")) + return with_paths(CONFIG, results_root=str(tmp_path / "rjob_logs"), + deploy_root=str(tmp_path / "deploy")) def test_cleanup_deletes_rjob_by_default(): diff --git a/tests/eval/test_rjob_client.py b/tests/eval/test_rjob_client.py index 83a0c2e..e1c03d2 100644 --- a/tests/eval/test_rjob_client.py +++ b/tests/eval/test_rjob_client.py @@ -21,7 +21,7 @@ watch, ) -from .rjob_fakes import CONFIG, REAL_RUNNING, REAL_SUCCEEDED, FakeRunner +from .rjob_fakes import CONFIG, REAL_RUNNING, REAL_SUCCEEDED, FakeRunner, with_paths def test_parse_status_variants(): @@ -55,21 +55,57 @@ def test_submit_args_include_mount_and_worker_entrypoint(): def test_build_job_spec_uses_config_paths(tmp_path): - config = dict(CONFIG, deploy_root=str(tmp_path / "deploy"), - results_root=str(tmp_path / "results")) + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy"), + results_root=str(tmp_path / "results")) spec = build_job_spec(config=config, run_id="r1", repo_root=tmp_path, spec_overrides={"benchmark": "osworld1"}) assert spec["output_root"] == str(tmp_path / "results" / "r1") assert spec["repo_root"] == str(tmp_path / "deploy") assert spec["benchmark"] == "osworld1" - assert spec["env"]["root"] == CONFIG["osw1_root"] - assert spec["secrets"]["jev_endpoint"] == CONFIG["jev_endpoint"] - assert spec["secrets"]["kimi"]["api_key"] == "sk-x" + assert spec["env"]["root"] == CONFIG["eval"]["benchmarks"]["osworld1"]["root"] + assert spec["secrets"]["fast_model"]["endpoint"] == "http://10.1.1.1:8891" + assert spec["secrets"]["slow_model"]["api_key"] == "sk-x" assert spec["secrets"]["registry"]["auths"] + # 角色与厂商解耦：spec 里记的是"谁在跑"，不是"哪家的模型" + assert spec["harness"]["models"] == {"slow": "kimi-k3", "fast": "jev"} + assert spec["harness_config"]["fast_backend"] == "jev" + assert spec["harness_config"]["slow_max_tokens"] == 4096 + assert spec["budget"] == {"max_steps": 5, "max_wall_seconds": 60.0} written = json.loads((tmp_path / "deploy" / "job_specs" / "r1.json").read_text()) assert written == spec +def test_build_job_spec_selects_models_by_name(tmp_path): + """换大脑/小脑只改名字：地址与协议跟着选中的 API 条目走。""" + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) + spec = build_job_spec(config=config, run_id="r1", repo_root=tmp_path, + spec_overrides={"benchmark": "osworld2", + "models": {"slow": "kimi-k3", "fast": "kev"}}) + assert spec["harness"]["models"] == {"slow": "kimi-k3", "fast": "kev"} + assert spec["harness_config"]["fast_backend"] == "kev" + assert spec["secrets"]["fast_model"]["endpoint"] == "" # kev 留空 = 走发现文件 + assert spec["secrets"]["fast_model"]["endpoint_file"].endswith("kev.json") + assert spec["secrets"]["slow_model"]["api_key"] == "sk-x" + + +def test_build_job_spec_rejects_unknown_model_name(tmp_path): + from tandem_eval.config_schema import ConfigError + + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) + with pytest.raises(ConfigError, match="可选"): + build_job_spec(config=config, run_id="r1", repo_root=tmp_path, + spec_overrides={"benchmark": "mock", "models": {"fast": "nope"}}) + + +def test_cli_budget_override_merges_with_config(tmp_path): + """预算来自 config，命令行只覆盖给定的那一项。""" + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) + spec = build_job_spec(config=config, run_id="r1", repo_root=tmp_path, + spec_overrides={"benchmark": "mock", + "budget": {"max_steps": 3}}) + assert spec["budget"] == {"max_steps": 3, "max_wall_seconds": 60.0} + + def test_watch_until_terminal_and_timeout(): runner = FakeRunner({"get": ["running", "succeeded"]}) client = RjobClient(runner) @@ -82,8 +118,8 @@ def test_watch_until_terminal_and_timeout(): def test_run_job_full_lifecycle_submits_watches_fetches_cleans(tmp_path): run_root = tmp_path / "rjob_logs" / "r1" # 唯一落盘位置 - config = dict(CONFIG, results_root=str(tmp_path / "rjob_logs"), - deploy_root=str(tmp_path / "deploy")) + config = with_paths(CONFIG, results_root=str(tmp_path / "rjob_logs"), + deploy_root=str(tmp_path / "deploy")) script = { "submit": ["submitted"], "get": ["running", "succeeded"], @@ -110,8 +146,8 @@ def test_run_job_full_lifecycle_submits_watches_fetches_cleans(tmp_path): def test_run_job_timeout_still_fetches_and_cleans(tmp_path): - config = dict(CONFIG, results_root=str(tmp_path / "rjob_logs"), - deploy_root=str(tmp_path / "deploy")) + config = with_paths(CONFIG, results_root=str(tmp_path / "rjob_logs"), + deploy_root=str(tmp_path / "deploy")) runner = FakeRunner({"get": ["running"], "stop": ["stopped"], "submit": ["ok"], "logs": ["log"]}) result = run_job(config=config, @@ -160,8 +196,8 @@ def test_fetch_artifacts_tolerates_cli_crash(tmp_path): def test_run_job_fetch_failure_never_skips_cleanup(tmp_path, monkeypatch): """平台日志补写失败不得跳过 cleanup（资源泄漏）；错误记入 result。""" - config = dict(CONFIG, results_root=str(tmp_path / "rjob_logs"), - deploy_root=str(tmp_path / "deploy")) + config = with_paths(CONFIG, results_root=str(tmp_path / "rjob_logs"), + deploy_root=str(tmp_path / "deploy")) runner = FakeRunner({"get": [REAL_SUCCEEDED], "submit": ["ok"], "logs": ["log"], "stop": ["stopped"]}) diff --git a/tests/harness/test_selector_kev.py b/tests/harness/test_selector_kev.py index dd99635..2c4b69b 100644 --- a/tests/harness/test_selector_kev.py +++ b/tests/harness/test_selector_kev.py @@ -116,13 +116,13 @@ def test_malformed_response_raises(tmp_path): def test_config_and_build_default_backend_switch(tmp_path): - with pytest.raises(ValueError, match="selector_backend"): - SessionConfig(selector_backend="omni") + with pytest.raises(ValueError, match="fast_backend"): + SessionConfig(fast_backend="omni") class NullClient: pass kev_components = build_default(client=NullClient(), endpoint="http://kev:8008", - config=SessionConfig(selector_backend="kev")) + config=SessionConfig(fast_backend="kev")) assert isinstance(kev_components.selector, KevPolicy) jev_components = build_default(client=NullClient(), endpoint="http://jev:8891", config=SessionConfig()) diff --git a/tests/harness/test_session.py b/tests/harness/test_session.py index cf7f439..cbf7f2c 100644 --- a/tests/harness/test_session.py +++ b/tests/harness/test_session.py @@ -286,7 +286,7 @@ def read(self): original_urlopen = urllib.request.urlopen urllib.request.urlopen = opener_holder try: - with pytest.raises(ValueError, match="raise kimi_max_tokens"): + with pytest.raises(ValueError, match="raise slow_max_tokens"): client.json_request([{"role": "user", "content": "hi"}], lambda v: True, role="plan_step") finally: @@ -330,7 +330,7 @@ def fake_urlopen(request, timeout=None): client.json_request([{"role": "user", "content": "hi"}], lambda v: True, role="r") assert captured["payload"]["max_tokens"] == 16384 assert captured["payload"]["reasoning_effort"] == "low" - client_off = KimiClient({"base_url": "http://x"}, SessionConfig(kimi_reasoning_effort="off"), + client_off = KimiClient({"base_url": "http://x"}, SessionConfig(slow_reasoning_effort="off"), tmp_path / "a.jsonl") client_off.json_request([{"role": "user", "content": "hi"}], lambda v: True, role="r") assert captured["payload"]["enable_thinking"] is False From b289b3548c7fed20103e13083eaa76e451b64987 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 22 Oct 2026 19:04:17 +0800 Subject: [PATCH 2/7] feat(eval): port the OSWorld 2.1 subset onto the new config schema PR #2 added the 56-task OSWorld 2.1 subset on the old flat config (kimi / jev_endpoint / osw21_root at the top level). That layout is gone - the config is now harness / eval / log, with the benchmarks under eval.benchmarks.. This commit moves the OSW2.1 support onto the new schema and fixes what the review of PR #2 turned up. Config: eval.benchmarks.osworld21.{root,vm,asset_root,shared_env,proxy, volume_size,inner_image} eval.mounts gains the canonical view gpfs://gpfs2/evobox-share-gpfs2 -> /mnt/shared-storage-gpfs2/evobox-share-gpfs2 the prepared/runtime symlink layer only resolves there; the dev machine has no canonical mount, so those paths are worker-only. The inner image must be release-pinned (@sha256:...). The schema rejects a "latest" tag, and build_job_spec refuses to build an osworld21 run without a pinned image, so a wrong image cannot be picked up silently. Fixes from the review: - the osworld21 paths are part of the required-key validation now, so an outdated config fails with "missing keys" instead of a bare KeyError; - the 56-task subset records where it comes from (the environment's validation/coverage.json, passed tasks) and a new test checks that every id in the list still has a task file in the runtime, so the snapshot cannot rot unnoticed (skipped when the runtime is not mounted); - worker.sh reads the spec through one spec_get helper instead of three inline python one-liners; - the anchor comments PR #2 dropped from osworld.py are back: the DEVIATIONS references, the "task_001 answered 404 because the asset URL was baked in at import time" diagnosis, the setup-cache seeding rationale and the receipt discipline; - the batch script documents BENCHMARK / TASKS / ROUTE_POLICY next to the model switches. osworld.py is back to 311 lines: the comments are worth more than the line budget, and the module split is left to the discipline rework. Verification: ruff check src tests demo, pytest (235 passed) and tandem check are green; a mock run through infra/worker.sh still works, and the osworld21 job spec builds with the expected env section. --- docs/CLUSTER_RUNTIME.md | 33 ++++++++++++++++++++++++++++++-- infra/config.example.json | 13 +++++++++++++ infra/worker.sh | 23 ++++++++++++++++++---- src/tandem_eval/config_schema.py | 32 +++++++++++++++++++++++++++++++ src/tandem_eval/manifest.py | 7 +++++++ src/tandem_eval/osworld.py | 17 +++++++++++++--- src/tandem_eval/rjob_spec.py | 19 +++++++++++++++++- tests/eval/rjob_fakes.py | 6 ++++++ tests/eval/test_config.py | 27 ++++++++++++++++++++++++++ tests/eval/test_manifest.py | 23 ++++++++++++++++++++++ tests/eval/test_osworld_env.py | 4 ++-- tests/eval/test_rjob_client.py | 23 ++++++++++++++++++++++ 12 files changed, 215 insertions(+), 12 deletions(-) diff --git a/docs/CLUSTER_RUNTIME.md b/docs/CLUSTER_RUNTIME.md index 23afcc7..f2a7a38 100644 --- a/docs/CLUSTER_RUNTIME.md +++ b/docs/CLUSTER_RUNTIME.md @@ -59,6 +59,11 @@ worker 入口、共享资源与已知坑。 | `eval.benchmarks.osworld1.root` / `.vm` | OSWorld1 基准根与 qcow2 | | `eval.benchmarks.osworld2.root` / `.vm` | OSWorld2 基准根与 qcow2 | | `eval.benchmarks.osworld2.asset_root` | OSW2 任务资产根（`OSWORLD_FILE_BASE_URL` 指向它，避免外网下载） | +| `eval.benchmarks.osworld21.root` / `.vm` / `.asset_root` | OSW2.1 的 runtime（prepared 软链层）、qcow2 与资产根 | +| `eval.benchmarks.osworld21.shared_env` | OSW2.1 独立 `env.sh`（**不与旧环境 env.sh 混用**） | +| `eval.benchmarks.osworld21.inner_image` | OSW2.1 运行镜像，必须发布固定（`@sha256:…`，禁 `latest`）；留空时跑 osworld21 会明确报错 | +| `eval.benchmarks.osworld21.proxy` | OSW2.1 用的代理（覆盖 worker.sh 的默认代理） | +| `eval.benchmarks.osworld21.volume_size` | 传给官方接口的**卷参数**，不等于根盘容量（实测根盘约 49GB、可用约 21GB） | | `eval.registry` | docker registry auth（内联传给 worker 写 `~/.docker/config.json`） | ### log —— 结果落盘 @@ -118,7 +123,7 @@ cleanup stop（释放资源）+ delete（永久删除 rjob 记录）——默 | 参数 | 说明 | |---|---| | `--run-id` | 必填。run 目录名 = `rjob_logs//` | -| `--benchmark` | `mock` / `osworld1` / `osworld2` | +| `--benchmark` | `mock` / `osworld1` / `osworld2` / `osworld21` | | `--tasks` | 逗号分隔 task id（覆盖 manifest） | | `--limit N` | 取清单前 N 题 | | `--shard I/N` | 把（limit 后的）清单**交错**切成 N 片，本 run 跑第 I 片（并发用） | @@ -192,6 +197,30 @@ JEV 的 `/rank` 要求 `image_path` 是**服务宿主机可读的文件路径** 因此由 harness 暂存到 evobox 中转目录（`harness.fast_model.image_stage.write` → `.read`）， payload 带 JEV 宿主机视角路径。暂存按内容哈希命名去重（`step_00_.png`），断点无害。 +## 6.5 OSWorld 2.1 的特别约定 + +OSW2.1 走**独立环境**（独立 runtime / VM / 资产 / venv / env.sh），与 OSW1、OSW2 不共用： + +- **共享资源**：`/osworld21/`（`README_CN.md` 是环境侧的权威说明）。 + `prepared/runtime/` 是一层**指向 canonical 路径 `/mnt/shared-storage-gpfs2/...` 的软链**， + 因此这些路径**只在 worker 内可见**——开发机没有 canonical 挂载点，软链是断的。 + 开发机要用 OSW2.1 的 runtime，请走 worker（`tandem-eval rjob run --benchmark osworld21`）。 +- **挂载**：因此 `eval.mounts` 里多了一条 `gpfs://gpfs2/evobox-share-gpfs2` → + `/mnt/shared-storage-gpfs2/evobox-share-gpfs2`（canonical 视角），旧的 user 视角保留不变。 +- **env.sh**：worker 用 `eval.benchmarks.osworld21.shared_env` 指定的独立 env.sh + （它再 source canonical 的 `osworld21/env.sh`），**不与旧环境 env.sh 混用**。 +- **镜像**：直接拉发布清单固定的运行镜像（digest），**不覆盖 `latest`**， + 也不复用 `eval.inner_image`；OSW2.1 的 runtime 自己按固定版本引用。 + 固定版本见 `osworld21/runtime/benchmark_releases/osworld-v2.1.json` 与 + `desktop_env/providers/docker/provider.py`。 +- **依赖自检**：OSW2.1 的独立 venv 按官方 `uv.lock` 安装，**没有 easyocr**， + worker 只检查 Pillow；旧环境仍检查 `easyocr + PIL`。 +- **题目子集**：`tandem_eval/manifest.py` 的 `OSWORLD21_TASK_IDS`（56 题）取自 + `osworld21/validation/coverage.json` 中 `official_vm.status == "passed"` 的题； + 该覆盖表由环境侧持续重生成，**子集要定期重新核对**。 +- **卷参数**：`volume_size` 只是传给官方接口的卷参数，**不代表根盘真有这么大** + （实测根盘约 49GB、可用约 21GB）。 + ## 7. 批量运行 ```bash @@ -241,4 +270,4 @@ worker 侧各段日志： 真机 smoke： - `--benchmark mock` 走 mock 链路（不起 VM、不碰共享资源），可本地跑； -- 真实基准走 OSW1 / OSW2，按本文 §3–§7。 +- 真实基准走 OSW1 / OSW2 / OSW2.1，按本文 §3–§7。 diff --git a/infra/config.example.json b/infra/config.example.json index 868fddf..c5bf7a6 100644 --- a/infra/config.example.json +++ b/infra/config.example.json @@ -56,6 +56,10 @@ { "source": "gpfs://gpfs2/adpc-share", "target": "/mnt/shared-storage-user/adpc-share" + }, + { + "source": "gpfs://gpfs2/evobox-share-gpfs2", + "target": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2" } ], "rjob": { @@ -76,6 +80,15 @@ "root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/benchmarks/osworld2", "vm": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/vm/osworld2/osworld-v2-ubuntu-x86.qcow2", "asset_root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld2_0808/assets" + }, + "osworld21": { + "root": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/prepared/runtime", + "vm": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/vm/osworld-v2-ubuntu-x86.qcow2", + "asset_root": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/assets", + "shared_env": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/prepared/env.sh", + "inner_image": "", + "proxy": "http://100.100.135.73:3128", + "volume_size": 64 } }, "registry": { diff --git a/infra/worker.sh b/infra/worker.sh index ab1ee3a..3f40a05 100755 --- a/infra/worker.sh +++ b/infra/worker.sh @@ -14,6 +14,19 @@ fi REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" export PYTHONPATH="$REPO_ROOT/src${PYTHONPATH:+:$PYTHONPATH}" +# 从 job spec 取点号路径的值（缺失 → 空串） +spec_get() { + python3 -c ' +import json, sys +node = json.load(open(sys.argv[1])) +for part in sys.argv[2].split("."): + if not isinstance(node, dict): + node = None + break + node = node.get(part) +print("" if node is None else node)' "$SPEC" "$1" +} + read -r BENCHMARK OUTPUT_ROOT < <(python3 -c ' import json, sys spec = json.load(open(sys.argv[1])) @@ -28,7 +41,8 @@ fi # ---- 真实基准：共享资源 + docker 准备 ---- SHARED_ENV="${SA_CUA_SHARED_ENV:-/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/env.sh}" if [ "$BENCHMARK" = "osworld21" ]; then - SHARED_ENV="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["env"]["shared_env"])' "$SPEC")" + # OSW2.1 自带独立 env.sh，不与旧环境混用 + SHARED_ENV="$(spec_get env.shared_env)" fi if [ ! -f "$SHARED_ENV" ]; then echo "缺少共享资源 env.sh: $SHARED_ENV" | tee "$OUTPUT_ROOT/worker/entry.log" >&2 @@ -48,11 +62,11 @@ pathlib.Path.home().joinpath(".docker/config.json").write_text(json.dumps(regist echo "registry 凭据写入失败" | tee -a "$OUTPUT_ROOT/worker/entry.log" >&2; exit 3; } chmod 600 "$HOME/.docker/config.json" -INNER_IMAGE="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["code"]["images"]["inner"])' "$SPEC")" +INNER_IMAGE="$(spec_get code.images.inner)" # 平台可能预注入不可用的代理（真机 2026-10-20：httpproxy-headless-dev-test 无法解析）， # 一律覆盖为已知可用代理，不继承环境变量。 -PROXY="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("env", {}).get( - "proxy", "http://httpproxy-headless.kubebrain.svc.pjlab.local:3128"))' "$SPEC")" +PROXY="$(spec_get env.proxy)" +PROXY="${PROXY:-http://httpproxy-headless.kubebrain.svc.pjlab.local:3128}" export HTTP_PROXY="$PROXY" HTTPS_PROXY="$PROXY" http_proxy="$PROXY" https_proxy="$PROXY" export NO_PROXY='10.0.0.0/8,100.96.0.0/12,localhost,127.0.0.1,.pjlab.org.cn' export no_proxy="$NO_PROXY" @@ -70,6 +84,7 @@ docker info > "$OUTPUT_ROOT/worker/docker-info.txt" 2>&1 || { echo "dockerd 未就绪" | tee -a "$OUTPUT_ROOT/worker/entry.log" >&2; exit 3; } docker pull "$INNER_IMAGE" > "$OUTPUT_ROOT/worker/docker-pull.log" 2>&1 if [ "$BENCHMARK" = "osworld21" ]; then + # OSW2.1 的独立 venv 按官方 uv.lock 装，没有 easyocr；只查 Pillow DEPS_CHECK='import PIL' else docker tag "$INNER_IMAGE" happysixd/osworld-docker:latest diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py index d2bd676..c97d52d 100644 --- a/src/tandem_eval/config_schema.py +++ b/src/tandem_eval/config_schema.py @@ -39,6 +39,10 @@ "eval.benchmarks.osworld2.root", "eval.benchmarks.osworld2.vm", "eval.benchmarks.osworld2.asset_root", + "eval.benchmarks.osworld21.root", + "eval.benchmarks.osworld21.vm", + "eval.benchmarks.osworld21.asset_root", + "eval.benchmarks.osworld21.shared_env", "eval.registry", "log.results_root", ) @@ -53,6 +57,10 @@ "eval.benchmarks.osworld2.root", "eval.benchmarks.osworld2.vm", "eval.benchmarks.osworld2.asset_root", + "eval.benchmarks.osworld21.root", + "eval.benchmarks.osworld21.vm", + "eval.benchmarks.osworld21.asset_root", + "eval.benchmarks.osworld21.shared_env", "harness.fast_model.image_stage.write", "harness.fast_model.image_stage.read", ) @@ -149,6 +157,29 @@ def _validate_absolute_paths(config: dict) -> list[str]: return problems +def _validate_osworld21(config: dict) -> list[str]: + """OSW2.1 专项校验：镜像必须发布固定（禁 latest），代理与卷参数合法。 + + inner_image 允许留空——它只在真正跑 osworld21 时才需要，届时 + build_job_spec 会明确报错，而不是静默拿错镜像。 + """ + problems: list[str] = [] + section = get_path(config, "eval.benchmarks.osworld21") or {} + image = section.get("inner_image") + if isinstance(image, str) and image.strip(): + if ":latest" in image: + problems.append("eval.benchmarks.osworld21.inner_image（不能用 latest，必须发布固定版本）") + elif "@sha256:" not in image: + problems.append("eval.benchmarks.osworld21.inner_image（应固定到 digest：@sha256:…）") + proxy = section.get("proxy") + if not _is_missing(proxy) and not str(proxy).startswith("http"): + problems.append("eval.benchmarks.osworld21.proxy（应为 http(s) 地址）") + volume_size = section.get("volume_size") + if volume_size is not None and (not isinstance(volume_size, int) or volume_size <= 0): + problems.append("eval.benchmarks.osworld21.volume_size（应为正整数）") + return problems + + def validate(config: dict) -> list[str]: """返回全部问题（空列表 = 配置合法）。""" problems: list[str] = [] @@ -157,6 +188,7 @@ def validate(config: dict) -> list[str]: problems += _validate_slow_models(config) problems += _validate_fast_models(config) problems += _validate_absolute_paths(config) + problems += _validate_osworld21(config) return problems diff --git a/src/tandem_eval/manifest.py b/src/tandem_eval/manifest.py index adfbb56..57a6aae 100644 --- a/src/tandem_eval/manifest.py +++ b/src/tandem_eval/manifest.py @@ -6,6 +6,13 @@ from dataclasses import dataclass from pathlib import Path +#: OSW2.1 子集题号（56 题）。 +#: +#: 出处：共享环境 `osworld21/validation/coverage.json` 中 `official_vm.status == "passed"` +#: 的题（2026-10-22 快照）。该覆盖表由 `osworld21/setup/summarize_environment.py` 持续 +#: 重生成，本清单是它的一次快照——覆盖表更新后必须重新核对，否则会与真实环境脱节。 +#: 快照时官方 passed 共 86 道，其余 30 道的排除口径尚未记录（待补）；本清单当前是 +#: "已验证通过"的保守子集，不含 failed / not_tested 的题。 OSWORLD21_TASK_IDS = tuple(( "001 002 003 004 006 009 010 011 012 014 017 022 023 024 " "025 027 029 030 032 034 037 040 042 044 045 049 051 053 " diff --git a/src/tandem_eval/osworld.py b/src/tandem_eval/osworld.py index 092d3bc..3390a99 100644 --- a/src/tandem_eval/osworld.py +++ b/src/tandem_eval/osworld.py @@ -1,6 +1,10 @@ """OSW1/OSW2/OSW2.1 统一环境适配器：DesktopEnv ↔ 协议消息。 -无回执不重放动作；终态 DONE/FAIL 送达 VM。want_observation="none" 暂仍返回完整观察。 +- 代码生成（gui_code/cli_code）语义对齐旧仓库 gui/actions.py + execution.py（DEVIATIONS #6）； +- 回执确认纪律：包装 controller.execute_python_command，无回执不重放（旧 execution.py:30-35）； +- want_observation="none" 当前仍返回完整观察（截图+a11y 由 controller 一次读取，成本低）， + 协议字段保留，严格跳过推迟到真机 profiling 后（DEVIATIONS #20）； +- 终态动作（finish/fail）经 env.step("DONE"/"FAIL") 送达 VM（评分钩子依赖它，旧 runtime 同款）。 """ from __future__ import annotations @@ -105,7 +109,10 @@ def __init__(self, *, benchmark: str, task_root: Path, vm_path: Path, self.volume_size = volume_size self.startup_wait_seconds = startup_wait_seconds self._env_factory = env_factory or self._import_desktop_env - # 任务模块在 import 时解析资产 URL，必须先设置本地资产根路径。 + # bootstrap 必须先于 fetch_task：任务模块在 import 时（fetch_task 阶段） + # 就于顶层调用 asset() 固化资产 URL——晚设 OSWORLD_FILE_BASE_URL 会让 + # 模块级 URL 烘焙成默认 HF 地址（task_001 真机 404 的根因；函数体内调用 + # 的 task_066 恰好躲过） self._bootstrap_runtime() def _bootstrap_runtime(self) -> None: @@ -123,6 +130,7 @@ def _bootstrap_runtime(self) -> None: def name(self) -> str: return f"osworld-{self.benchmark}" + # ------------------------------------------------------------------ 装载 def _import_desktop_env(self, **kwargs): self._bootstrap_runtime() os.chdir(self.task_root) @@ -164,6 +172,7 @@ def fetch_task(self, task_id: str, domain: str = "") -> dict: ) return task + # ------------------------------------------------------------------ 观察 def _observation(self, task: dict, step_idx: int, budget: Budget, receipt: Receipt | None) -> Observation: stage = self.run_root / "obs_stage" / str(task["id"]) @@ -197,8 +206,10 @@ def _capture(self) -> None: self.last_screenshot = obs["screenshot"] self.last_a11y = obs.get("accessibility_tree") or "" + # ------------------------------------------------------------------ 协议 def _seed_setup_cache(self, task_id: str) -> None: - """把已缓存的 setup 文件复制到本次 evaluation_cache，避免外网下载。""" + """把共享盘已缓存的 setup 文件播种进本次 evaluation_cache（旧 runtime.py:532-539）， + 避免任务 setup 走外网下载（HF 下载在 worker 内经代理，慢且可能失败）。""" if self.setup_cache_root is None: return source = self.setup_cache_root / task_id diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py index 94bfd6f..2ce267e 100644 --- a/src/tandem_eval/rjob_spec.py +++ b/src/tandem_eval/rjob_spec.py @@ -47,6 +47,13 @@ def _env_section(benchmark: str, config: dict) -> dict: section = benchmarks["osworld2"] return {"root": section["root"], "vm": section["vm"], "asset_root": section["asset_root"], "startup_wait_seconds": 30} + if benchmark == "osworld21": + # OSW2.1 自带独立 runtime/VM/资产/env.sh，不与旧环境混用 + section = benchmarks["osworld21"] + return {"root": section["root"], "vm": section["vm"], + "asset_root": section["asset_root"], "shared_env": section["shared_env"], + "proxy": section.get("proxy", ""), "volume_size": section.get("volume_size"), + "startup_wait_seconds": 30} return {} @@ -118,6 +125,16 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path, slow_name, slow_api = select_api(config, "slow_model", requested.get("slow")) fast_name, fast_api = select_api(config, "fast_model", requested.get("fast")) + inner_image = config["eval"]["inner_image"] + if benchmark == "osworld21": + # OSW2.1 用发布清单固定的运行镜像（digest），不覆盖 latest，也不复用 OSW2 的镜像 + inner_image = config["eval"]["benchmarks"]["osworld21"].get("inner_image", "") + if not inner_image: + raise ValueError( + "eval.benchmarks.osworld21.inner_image 必须填发布清单固定的运行镜像" + "（@sha256:…；见 osworld21/runtime/benchmark_releases/osworld-v2.1.json）" + ) + harness_config = _harness_config_section(config, slow_api, fast_api) harness_config.update(overrides.pop("harness_config", {}) or {}) budget = _budget_section(config) @@ -143,7 +160,7 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path, "git_describe": _git_describe(repo_root), "images": { "outer": config["eval"]["rjob"]["image"], - "inner": config["eval"]["inner_image"], + "inner": inner_image, }, }, **overrides, diff --git a/tests/eval/rjob_fakes.py b/tests/eval/rjob_fakes.py index 22b9724..efa3bcd 100644 --- a/tests/eval/rjob_fakes.py +++ b/tests/eval/rjob_fakes.py @@ -93,6 +93,12 @@ def __call__(self, argv): "osworld1": {"root": "/shared/osw1", "vm": "/shared/vm1.qcow2"}, "osworld2": {"root": "/shared/osw2", "vm": "/shared/vm2.qcow2", "asset_root": "/shared/assets"}, + "osworld21": { + "root": "/shared/osw21/runtime", "vm": "/shared/osw21/vm.qcow2", + "asset_root": "/shared/osw21/assets", "shared_env": "/shared/osw21/env.sh", + "inner_image": "registry.example/osw21@sha256:deadbeef", + "proxy": "http://proxy.example:3128", "volume_size": 64, + }, }, "registry": {"auths": {"registry.h.pjlab.org.cn": {"auth": "xxx"}}}, }, diff --git a/tests/eval/test_config.py b/tests/eval/test_config.py index 8138986..4a5a79e 100644 --- a/tests/eval/test_config.py +++ b/tests/eval/test_config.py @@ -134,6 +134,33 @@ def test_select_api_error_lists_available_names(): select_api(config, "fast_model", "no-such-api") +def test_osworld21_section_is_validated(): + config = load_config(CONFIG_PATH) + section = config["eval"]["benchmarks"]["osworld21"] + for key in ("root", "vm", "asset_root", "shared_env"): + assert section[key].startswith("/"), f"{key} 必须是绝对路径" + assert section["proxy"].startswith("http") + assert isinstance(section["volume_size"], int) and section["volume_size"] > 0 + + +def test_missing_osworld21_paths_are_reported(tmp_path): + config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) + del config["eval"]["benchmarks"]["osworld21"]["shared_env"] + bad = tmp_path / "bad.json" + bad.write_text(json.dumps(config), encoding="utf-8") + with pytest.raises(ConfigError, match="osworld21.shared_env"): + load_config(bad) + + +def test_osworld21_inner_image_must_be_release_pinned(tmp_path): + config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) + config["eval"]["benchmarks"]["osworld21"]["inner_image"] = "registry.example/osw21:latest" + bad = tmp_path / "bad.json" + bad.write_text(json.dumps(config), encoding="utf-8") + with pytest.raises(ConfigError, match="latest"): + load_config(bad) + + def test_cli_model_flags_reach_spec_overrides(monkeypatch): """启动命令按名字切大脑/小脑：--slow-model / --fast-model 进 spec 的 models 段。""" from tandem_eval.__main__ import main diff --git a/tests/eval/test_manifest.py b/tests/eval/test_manifest.py index 3f9c85a..3a26654 100644 --- a/tests/eval/test_manifest.py +++ b/tests/eval/test_manifest.py @@ -3,11 +3,22 @@ from __future__ import annotations import json +from pathlib import Path import pytest from tandem_eval.manifest import OSWORLD21_TASK_IDS, TaskRef, load_index, resolve_manifest, write_resolved +# OSW2.1 的 prepared/runtime 是指向 canonical 路径的软链层：worker 内可见（canonical 视角）， +# 开发机只有 user 视角的同一份 runtime。两处都没有时跳过该检查（例如 CI）。 +_OSW21_TASK_CLASS_CANDIDATES = ( + Path("/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua" + "/osworld21/prepared/runtime/evaluation_examples/task_class"), + Path("/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua" + "/osworld21/runtime/evaluation_examples/task_class"), +) +_OSW21_TASK_CLASS = next((p for p in _OSW21_TASK_CLASS_CANDIDATES if p.is_dir()), None) + def test_load_index_domain_dict_format(): import tempfile @@ -114,3 +125,15 @@ def test_index_root_derived_from_env_section(tmp_path): spec = {"benchmark": "osworld2", "env": {"root": str(root)}, "manifest": {}} refs = resolve_manifest(spec) assert [r.task_id for r in refs] == ["001", "002", "003"] + + +@pytest.mark.skipif(_OSW21_TASK_CLASS is None, reason="OSW2.1 runtime 未挂载（worker 内才有 canonical 视角）") +def test_osworld21_subset_tasks_exist_in_runtime(): + """子集里的每个题号都必须在 runtime 的 task_class 里有对应任务文件。 + + 子集来自环境侧持续重生成的覆盖表快照（见 manifest.OSWORLD21_TASK_IDS 的说明）， + 环境更新后这道题会第一时间报出"清单里有 runtime 没有的题"。 + """ + missing = [tid for tid in OSWORLD21_TASK_IDS + if not (_OSW21_TASK_CLASS / f"task_{tid}.py").is_file()] + assert missing == [], f"子集里的题号在 runtime 中不存在: {missing}" diff --git a/tests/eval/test_osworld_env.py b/tests/eval/test_osworld_env.py index 88fb199..ec0806f 100644 --- a/tests/eval/test_osworld_env.py +++ b/tests/eval/test_osworld_env.py @@ -205,7 +205,7 @@ def test_setup_cache_seeding(tmp_path): @pytest.mark.parametrize("benchmark", ["osworld2", "osworld21"]) -def test_osw2_module_level_asset_urls_resolve_locally(tmp_path, monkeypatch, benchmark): +def test_task_module_level_asset_urls_resolve_locally(tmp_path, monkeypatch, benchmark): """OSW2 任务模块在 import 时（fetch_task 阶段）顶层调用 asset()； OSWORLD_FILE_BASE_URL 必须在那之前就绪（task_001 真机 404 根因）。""" import os @@ -252,7 +252,7 @@ def test_osw2_module_level_asset_urls_resolve_locally(tmp_path, monkeypatch, ben assert os.environ.get("OSWORLD_FILE_BASE_URL") == str(assets) -def test_native_task_setup_and_evaluation_are_preserved(tmp_path, monkeypatch): +def test_task_object_is_passed_through_to_desktop_env(tmp_path, monkeypatch): import sys monkeypatch.setattr(sys, "path", sys.path.copy()) diff --git a/tests/eval/test_rjob_client.py b/tests/eval/test_rjob_client.py index e1c03d2..213e626 100644 --- a/tests/eval/test_rjob_client.py +++ b/tests/eval/test_rjob_client.py @@ -97,6 +97,29 @@ def test_build_job_spec_rejects_unknown_model_name(tmp_path): spec_overrides={"benchmark": "mock", "models": {"fast": "nope"}}) +def test_build_job_spec_for_osworld21(tmp_path): + """OSW2.1 用自己的 runtime/env/镜像，不复用 OSW2 的。""" + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) + spec = build_job_spec(config=config, run_id="r1", repo_root=tmp_path, + spec_overrides={"benchmark": "osworld21"}) + section = CONFIG["eval"]["benchmarks"]["osworld21"] + assert spec["env"]["shared_env"] == section["shared_env"] + assert spec["env"]["proxy"] == section["proxy"] + assert spec["env"]["volume_size"] == section["volume_size"] + assert spec["code"]["images"]["inner"] == section["inner_image"] + old = build_job_spec(config=config, run_id="r2", repo_root=tmp_path, + spec_overrides={"benchmark": "osworld2"}) + assert old["code"]["images"]["inner"] == CONFIG["eval"]["inner_image"] + + +def test_build_job_spec_rejects_missing_osworld21_inner_image(tmp_path): + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) + config["eval"]["benchmarks"]["osworld21"]["inner_image"] = "" + with pytest.raises(ValueError, match="inner_image"): + build_job_spec(config=config, run_id="r1", repo_root=tmp_path, + spec_overrides={"benchmark": "osworld21"}) + + def test_cli_budget_override_merges_with_config(tmp_path): """预算来自 config，命令行只覆盖给定的那一项。""" config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) From c311ead509a1cd0ec918ca678ea0e9665be5d65f Mon Sep 17 00:00:00 2001 From: root Date: Thu, 22 Oct 2026 19:23:00 +0800 Subject: [PATCH 3/7] fix(eval): mount the evobox volume once, in the canonical view The platform refuses the same source mounted twice ("the value of source path repeats"), and the OSW2.1 env.sh plus its prepared/runtime symlink layer spell their paths in the canonical view (/mnt/shared-storage-gpfs2/...). So the evobox volume gets exactly one mount, at the canonical target, and every evobox path in the config follows that view - the old environment's env.sh is self-relative, so it is happy either way. shared_env is now config-driven for all benchmarks (eval.shared_env for OSW1/OSW2, eval.benchmarks.osworld21.shared_env for OSW2.1) instead of being hardcoded in worker.sh. The dev machine keeps its user-style view; a symlink (/mnt/shared-storage-gpfs2/evobox-share-gpfs2 -> /mnt/shared-storage-user/evobox-share-gpfs2) makes the config's paths readable locally, and the runbook documents it. Verification: ruff, pytest (235 passed) and tandem check are green; the generated submit arguments carry two distinct mount sources, and every path the config references resolves through the canonical view. --- docs/CLUSTER_RUNTIME.md | 16 ++++++++++------ infra/config.example.json | 21 +++++++++------------ infra/worker.sh | 8 +++----- src/tandem_eval/config_schema.py | 2 ++ src/tandem_eval/rjob_spec.py | 5 +++-- tests/eval/rjob_fakes.py | 1 + tests/eval/test_config.py | 26 ++++++++++++++++++++------ tests/eval/test_rjob_client.py | 1 + 8 files changed, 49 insertions(+), 31 deletions(-) diff --git a/docs/CLUSTER_RUNTIME.md b/docs/CLUSTER_RUNTIME.md index f2a7a38..580769b 100644 --- a/docs/CLUSTER_RUNTIME.md +++ b/docs/CLUSTER_RUNTIME.md @@ -53,12 +53,13 @@ worker 入口、共享资源与已知坑。 |---|---| | `eval.repo_root` | 本仓库路径（开发机） | | `eval.deploy_root` | rsync 部署副本路径（worker 读代码；两侧挂载点一致） | -| `eval.mounts` | rjob `--mount` 参数（gpfs 源 → 容器内挂载点） | +| `eval.mounts` | rjob `--mount` 参数（gpfs 源 → 容器内挂载点）。**同一 source 不能挂两次**（平台报 `the value of source path repeats`），所以 evobox 卷只挂一次 | | `eval.rjob` | `charged_group` / `private_machine` / `cpu` / `memory` / `gpu` / `image` | | `eval.inner_image` | OSWorld VM 用的 inner 镜像（worker 内 dockerd 拉取） | | `eval.benchmarks.osworld1.root` / `.vm` | OSWorld1 基准根与 qcow2 | | `eval.benchmarks.osworld2.root` / `.vm` | OSWorld2 基准根与 qcow2 | | `eval.benchmarks.osworld2.asset_root` | OSW2 任务资产根（`OSWORLD_FILE_BASE_URL` 指向它，避免外网下载） | +| `eval.shared_env` | 旧环境（OSW1/OSW2）的 `env.sh`（脚本自相对，两种视角都行） | | `eval.benchmarks.osworld21.root` / `.vm` / `.asset_root` | OSW2.1 的 runtime（prepared 软链层）、qcow2 与资产根 | | `eval.benchmarks.osworld21.shared_env` | OSW2.1 独立 `env.sh`（**不与旧环境 env.sh 混用**） | | `eval.benchmarks.osworld21.inner_image` | OSW2.1 运行镜像，必须发布固定（`@sha256:…`，禁 `latest`）；留空时跑 osworld21 会明确报错 | @@ -202,11 +203,14 @@ payload 带 JEV 宿主机视角路径。暂存按内容哈希命名去重（`ste OSW2.1 走**独立环境**（独立 runtime / VM / 资产 / venv / env.sh），与 OSW1、OSW2 不共用： - **共享资源**：`/osworld21/`（`README_CN.md` 是环境侧的权威说明）。 - `prepared/runtime/` 是一层**指向 canonical 路径 `/mnt/shared-storage-gpfs2/...` 的软链**， - 因此这些路径**只在 worker 内可见**——开发机没有 canonical 挂载点，软链是断的。 - 开发机要用 OSW2.1 的 runtime，请走 worker（`tandem-eval rjob run --benchmark osworld21`）。 -- **挂载**：因此 `eval.mounts` 里多了一条 `gpfs://gpfs2/evobox-share-gpfs2` → - `/mnt/shared-storage-gpfs2/evobox-share-gpfs2`（canonical 视角），旧的 user 视角保留不变。 +- **挂载视角只有一个**：平台不允许同一 source 挂两次，而 OSW2.1 的 `env.sh` 与 + `prepared/runtime/` 软链层写的是 **canonical 路径** `/mnt/shared-storage-gpfs2/...`， + 所以 evobox 卷在 worker 里统一挂 canonical 视角，配置里所有 evobox 路径也随之统一 + （旧环境的 `env.sh` 是自相对的，两种视角都行，不受影响）。 + - worker：`gpfs://gpfs2/evobox-share-gpfs2` → `/mnt/shared-storage-gpfs2/evobox-share-gpfs2` + - 开发机：默认只有 user 视角 `/mnt/shared-storage-user/evobox-share-gpfs2`， + 想在开发机直接读配置里的路径，建一个软链即可（只影响开发机）： + `ln -s /mnt/shared-storage-user/evobox-share-gpfs2 /mnt/shared-storage-gpfs2/evobox-share-gpfs2` - **env.sh**：worker 用 `eval.benchmarks.osworld21.shared_env` 指定的独立 env.sh （它再 source canonical 的 `osworld21/env.sh`），**不与旧环境 env.sh 混用**。 - **镜像**：直接拉发布清单固定的运行镜像（digest），**不覆盖 `latest`**， diff --git a/infra/config.example.json b/infra/config.example.json index c5bf7a6..ac6c682 100644 --- a/infra/config.example.json +++ b/infra/config.example.json @@ -20,18 +20,18 @@ "jev": { "backend": "jev", "endpoint": "", - "endpoint_file": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/services/qwen_jev.json", + "endpoint_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/services/qwen_jev.json", "timeout_seconds": 300 }, "kev": { "backend": "kev", "endpoint": "", - "endpoint_file": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/services/kev.json", + "endpoint_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/services/kev.json", "timeout_seconds": 300 } }, "image_stage": { - "write": "/mnt/shared-storage-user/evobox-share-gpfs2/chenxinquan/tandem_jev_images", + "write": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/chenxinquan/tandem_jev_images", "read": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/chenxinquan/tandem_jev_images" } }, @@ -49,10 +49,6 @@ "repo_root": "/mnt/shared-storage-user/adpc-share/chenxinquan/fastslow/tandem", "deploy_root": "/mnt/shared-storage-user/adpc-share/chenxinquan/fastslow/tandem-deploy", "mounts": [ - { - "source": "gpfs://gpfs2/evobox-share-gpfs2", - "target": "/mnt/shared-storage-user/evobox-share-gpfs2" - }, { "source": "gpfs://gpfs2/adpc-share", "target": "/mnt/shared-storage-user/adpc-share" @@ -71,15 +67,16 @@ "image": "registry.h.pjlab.org.cn/ailab-evobox-evobox_cpu/weavebench:osworld-v2-rjob-openclaw-kimi-tasks-uploadfix-20260701" }, "inner_image": "registry.h.pjlab.org.cn/ailab-evobox-evobox_cpu/weavebench:osworld-v2-inner-osworld-docker-socat-20260712", + "shared_env": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/env.sh", "benchmarks": { "osworld1": { - "root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/benchmarks/osworld1", - "vm": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/vm/osworld1/Ubuntu.qcow2" + "root": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/benchmarks/osworld1", + "vm": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/vm/osworld1/Ubuntu.qcow2" }, "osworld2": { - "root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/benchmarks/osworld2", - "vm": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/vm/osworld2/osworld-v2-ubuntu-x86.qcow2", - "asset_root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld2_0808/assets" + "root": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/benchmarks/osworld2", + "vm": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/vm/osworld2/osworld-v2-ubuntu-x86.qcow2", + "asset_root": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld2_0808/assets" }, "osworld21": { "root": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/prepared/runtime", diff --git a/infra/worker.sh b/infra/worker.sh index 3f40a05..ec20a17 100755 --- a/infra/worker.sh +++ b/infra/worker.sh @@ -39,11 +39,9 @@ if [ "$BENCHMARK" = "mock" ]; then fi # ---- 真实基准：共享资源 + docker 准备 ---- -SHARED_ENV="${SA_CUA_SHARED_ENV:-/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/env.sh}" -if [ "$BENCHMARK" = "osworld21" ]; then - # OSW2.1 自带独立 env.sh，不与旧环境混用 - SHARED_ENV="$(spec_get env.shared_env)" -fi +# shared_env 由 spec 给出（OSW1/OSW2 用旧环境的 env.sh，OSW2.1 用它自己的独立 env.sh）； +# SA_CUA_SHARED_ENV 可临时覆盖。 +SHARED_ENV="${SA_CUA_SHARED_ENV:-$(spec_get env.shared_env)}" if [ ! -f "$SHARED_ENV" ]; then echo "缺少共享资源 env.sh: $SHARED_ENV" | tee "$OUTPUT_ROOT/worker/entry.log" >&2 exit 3 diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py index c97d52d..40bec4d 100644 --- a/src/tandem_eval/config_schema.py +++ b/src/tandem_eval/config_schema.py @@ -43,6 +43,7 @@ "eval.benchmarks.osworld21.vm", "eval.benchmarks.osworld21.asset_root", "eval.benchmarks.osworld21.shared_env", + "eval.shared_env", "eval.registry", "log.results_root", ) @@ -51,6 +52,7 @@ ABSOLUTE_PATH_PATHS = ( "eval.repo_root", "eval.deploy_root", + "eval.shared_env", "log.results_root", "eval.benchmarks.osworld1.root", "eval.benchmarks.osworld1.vm", diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py index 2ce267e..b937463 100644 --- a/src/tandem_eval/rjob_spec.py +++ b/src/tandem_eval/rjob_spec.py @@ -38,14 +38,15 @@ def _git_describe(repo_root: Path) -> str: def _env_section(benchmark: str, config: dict) -> dict: benchmarks = config["eval"]["benchmarks"] + shared_env = config["eval"]["shared_env"] if benchmark == "osworld1": section = benchmarks["osworld1"] - return {"root": section["root"], "vm": section["vm"], + return {"root": section["root"], "vm": section["vm"], "shared_env": shared_env, "setup_cache_root": str(Path(section["root"]) / "cache"), "startup_wait_seconds": 30} if benchmark == "osworld2": section = benchmarks["osworld2"] - return {"root": section["root"], "vm": section["vm"], + return {"root": section["root"], "vm": section["vm"], "shared_env": shared_env, "asset_root": section["asset_root"], "startup_wait_seconds": 30} if benchmark == "osworld21": # OSW2.1 自带独立 runtime/VM/资产/env.sh，不与旧环境混用 diff --git a/tests/eval/rjob_fakes.py b/tests/eval/rjob_fakes.py index efa3bcd..9b43aa2 100644 --- a/tests/eval/rjob_fakes.py +++ b/tests/eval/rjob_fakes.py @@ -85,6 +85,7 @@ def __call__(self, argv): "eval": { "repo_root": "/dev-repo", "deploy_root": "/dev-deploy", + "shared_env": "/shared/env.sh", "mounts": [{"source": "gpfs://gpfs2/share", "target": "/mnt/share"}], "rjob": {"charged_group": "evobox_proxy", "private_machine": True, "cpu": 4, "memory": 16384, "gpu": 0, "image": "reg.example/outer:1"}, diff --git a/tests/eval/test_config.py b/tests/eval/test_config.py index 4a5a79e..836d692 100644 --- a/tests/eval/test_config.py +++ b/tests/eval/test_config.py @@ -73,15 +73,29 @@ def test_paths_are_absolute_and_credentials_inlined(): assert config["eval"]["repo_root"].endswith("/tandem") -def test_mounts_cover_both_volumes_with_user_style_targets(): +def test_mounts_cover_both_volumes_without_duplicate_sources(): + """两个卷各挂一次；同一 source 挂两次会被平台拒（"the value of source path repeats"）。 + + evobox 卷在 worker 里挂 canonical 视角——OSW2.1 的 env.sh 与 prepared 软链层写的是 + canonical 路径，只有这个视角能解析；旧环境的 env.sh 是自相对的，两种视角都行。 + adpc 卷（部署副本与结果目录）维持 user 视角。 + """ config = load_config(CONFIG_PATH) - targets = {m["target"] for m in config["eval"]["mounts"]} - assert "/mnt/shared-storage-user/evobox-share-gpfs2" in targets + mounts = config["eval"]["mounts"] + sources = [m["source"] for m in mounts] + assert len(sources) == len(set(sources)), f"同一 source 不能挂两次: {mounts}" + targets = {m["target"] for m in mounts} + assert "/mnt/shared-storage-gpfs2/evobox-share-gpfs2" in targets assert "/mnt/shared-storage-user/adpc-share" in targets - # evobox 卷两侧一致（user 风格），共享资源路径随之 user 风格 + # 配置里引用的 evobox 路径必须与 worker 的挂载视角一致 benchmarks = config["eval"]["benchmarks"] - assert benchmarks["osworld1"]["root"].startswith("/mnt/shared-storage-user/evobox-share-gpfs2/") - assert benchmarks["osworld1"]["vm"].startswith("/mnt/shared-storage-user/evobox-share-gpfs2/") + for key in ("osworld1", "osworld2", "osworld21"): + for field in ("root", "vm", "asset_root"): + value = benchmarks[key].get(field) + if value: + assert value.startswith("/mnt/shared-storage-gpfs2/evobox-share-gpfs2/"), value + assert config["harness"]["fast_model"]["image_stage"]["write"].startswith( + "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/") def test_budget_and_route_come_from_config(): diff --git a/tests/eval/test_rjob_client.py b/tests/eval/test_rjob_client.py index 213e626..725876f 100644 --- a/tests/eval/test_rjob_client.py +++ b/tests/eval/test_rjob_client.py @@ -68,6 +68,7 @@ def test_build_job_spec_uses_config_paths(tmp_path): assert spec["secrets"]["registry"]["auths"] # 角色与厂商解耦：spec 里记的是"谁在跑"，不是"哪家的模型" assert spec["harness"]["models"] == {"slow": "kimi-k3", "fast": "jev"} + assert spec["env"]["shared_env"] == CONFIG["eval"]["shared_env"] assert spec["harness_config"]["fast_backend"] == "jev" assert spec["harness_config"]["slow_max_tokens"] == 4096 assert spec["budget"] == {"max_steps": 5, "max_wall_seconds": 60.0} From d067767b90bea728c600e5ba0ac4049d1b6f9d7c Mon Sep 17 00:00:00 2001 From: root Date: Thu, 22 Oct 2026 23:13:26 +0800 Subject: [PATCH 4/7] fix(eval): give osworld21 the credentials the worker needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The osworld21 run died before it did anything: the job spec carried an empty secrets section, so worker.sh had no registry auth to write to ~/.docker/config.json and exited at the pull step ("缺少 registry 凭据"). Root cause was mine: the rebase conflict resolution took rjob_spec.py wholesale from the config-refactor side, which dropped the "osworld21" that PR #2 had added to the credentials whitelist. The env section and the inner-image guard were ported back, this whitelist was not. A regression test now checks that every real benchmark produces a spec with credentials (slow_model, fast_model, registry), so a benchmark missing from a whitelist fails in the test suite instead of on the cluster. worker.sh also stops swallowing the reason: the registry failure path ran `cat` on an entry log that does not exist yet, and errexit killed the shell before it could print the message - the platform log only showed the python error and exit status 1 instead of 3. The docker pull and dependency check failures now echo the tail of their log files. --- infra/worker.sh | 7 +++++-- src/tandem_eval/rjob_spec.py | 2 +- tests/eval/test_rjob_client.py | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/infra/worker.sh b/infra/worker.sh index ec20a17..10d627d 100755 --- a/infra/worker.sh +++ b/infra/worker.sh @@ -56,7 +56,7 @@ registry = json.load(open(sys.argv[1]))["secrets"].get("registry") if not registry: print("缺少 registry 凭据（config.json 的 registry 段为空）", file=sys.stderr); raise SystemExit(3) pathlib.Path.home().joinpath(".docker/config.json").write_text(json.dumps(registry))' "$SPEC" || { - cat "$OUTPUT_ROOT/worker/entry.log" 2>/dev/null + cat "$OUTPUT_ROOT/worker/entry.log" 2>/dev/null || true echo "registry 凭据写入失败" | tee -a "$OUTPUT_ROOT/worker/entry.log" >&2; exit 3; } chmod 600 "$HOME/.docker/config.json" @@ -80,7 +80,9 @@ for _ in $(seq 1 90); do done docker info > "$OUTPUT_ROOT/worker/docker-info.txt" 2>&1 || { echo "dockerd 未就绪" | tee -a "$OUTPUT_ROOT/worker/entry.log" >&2; exit 3; } -docker pull "$INNER_IMAGE" > "$OUTPUT_ROOT/worker/docker-pull.log" 2>&1 +docker pull "$INNER_IMAGE" > "$OUTPUT_ROOT/worker/docker-pull.log" 2>&1 || { + tail -20 "$OUTPUT_ROOT/worker/docker-pull.log" | tee -a "$OUTPUT_ROOT/worker/entry.log" >&2 + echo "inner 镜像拉取失败: $INNER_IMAGE" | tee -a "$OUTPUT_ROOT/worker/entry.log" >&2; exit 3; } if [ "$BENCHMARK" = "osworld21" ]; then # OSW2.1 的独立 venv 按官方 uv.lock 装，没有 easyocr；只查 Pillow DEPS_CHECK='import PIL' @@ -91,6 +93,7 @@ fi "$OSWORLD_PYTHON" -c "$DEPS_CHECK" \ > "$OUTPUT_ROOT/worker/deps-check.log" 2>&1 || { + tail -20 "$OUTPUT_ROOT/worker/deps-check.log" | tee -a "$OUTPUT_ROOT/worker/entry.log" >&2 echo "共享 venv 依赖检查失败" | tee -a "$OUTPUT_ROOT/worker/entry.log" >&2; exit 3; } "$OSWORLD_PYTHON" -m tandem_eval entry --spec "$SPEC" 2>&1 | tee "$OUTPUT_ROOT/worker/entry.log" diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py index b937463..59b29a0 100644 --- a/src/tandem_eval/rjob_spec.py +++ b/src/tandem_eval/rjob_spec.py @@ -60,7 +60,7 @@ def _env_section(benchmark: str, config: dict) -> dict: def _secrets_section(benchmark: str, config: dict, slow_api: dict, fast_api: dict) -> dict: """凭据段：慢模型三件套 + 快模型地址 + registry auth（落盘时整体脱敏）。""" - if benchmark not in {"osworld1", "osworld2"}: + if benchmark not in {"osworld1", "osworld2", "osworld21"}: return {} return { "slow_model": {key: slow_api[key] for key in ("base_url", "api_key", "model")}, diff --git a/tests/eval/test_rjob_client.py b/tests/eval/test_rjob_client.py index 725876f..53dd21d 100644 --- a/tests/eval/test_rjob_client.py +++ b/tests/eval/test_rjob_client.py @@ -98,6 +98,22 @@ def test_build_job_spec_rejects_unknown_model_name(tmp_path): spec_overrides={"benchmark": "mock", "models": {"fast": "nope"}}) +@pytest.mark.parametrize("benchmark", ["osworld1", "osworld2", "osworld21"]) +def test_real_benchmarks_carry_credentials(tmp_path, benchmark): + """每个真实基准的 spec 都必须带凭据段。 + + 基准名在白名单里漏一个，secrets 就会是空的——worker 在拉 inner 镜像前 + 因没有 registry 凭据直接退出（2026-10-22 跑 osworld21 时踩过）。 + """ + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) + spec = build_job_spec(config=config, run_id="r1", repo_root=tmp_path, + spec_overrides={"benchmark": benchmark}) + secrets = spec["secrets"] + assert secrets["slow_model"]["api_key"] + assert secrets["fast_model"]["endpoint_file"] + assert secrets["registry"]["auths"] + + def test_build_job_spec_for_osworld21(tmp_path): """OSW2.1 用自己的 runtime/env/镜像，不复用 OSW2 的。""" config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) From d4f8f9cfd483c73122b7ac43d26f28ceaaf3e7eb Mon Sep 17 00:00:00 2001 From: root Date: Fri, 23 Oct 2026 10:03:58 +0800 Subject: [PATCH 5/7] config: register the MiniMax M3.1 Flash slow model The Kimi key behind harness.slow_model got blocked at the provider (401 "API-key is blocked"), so the brain moves to the M plan API. harness.slow_model.apis gains minimax-m3.1-flash (base_url https://api.minimaxi.com/v1, model MiniMax-M3.1-Flash) and becomes the default; kimi-k3 stays registered so it can come back with a fresh key. The harness speaks the OpenAI-compatible dialect and sends only model, messages, stream, max_tokens plus one thinking knob - MiniMax accepts that as is (verified end to end: content, reasoning_content, finish_reason and usage all come back in the expected shape). Two dialect differences are worth knowing (measured 2026-10-23): - reasoning_effort is honoured (low / medium / high); - enable_thinking=false is ignored, so the harness's "off" effort does not actually disable thinking on MiniMax - lower reasoning_effort instead. Documented in the README and the runbook. The schema now rejects what the harness would refuse at runtime anyway: base_url must be http(s) and reasoning_effort must be one of the efforts SessionConfig accepts, so a typo fails at load time. Verification: ruff, pytest (240 passed) and tandem check are green. --- README.md | 14 +++++++++++--- docs/CLUSTER_RUNTIME.md | 2 +- infra/config.example.json | 13 +++++++++++-- src/tandem_eval/config_schema.py | 15 +++++++++++++++ tests/eval/test_config.py | 19 +++++++++++++++++++ 5 files changed, 57 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 2f46c2c..a5a263e 100644 --- a/README.md +++ b/README.md @@ -52,10 +52,11 @@ ## 快慢双模：分工与边界 -| | 慢模型（Kimi） | 快模型（System One） | +| | 慢模型（大脑） | 快模型（小脑） | |---|---|---| | 角色 | 需求抽取 / 一步规划 / 独立验证 | 在候选空间里排序选择 | -| 实现 | `planner.py`、`verifier.py`、`providers.py`（KimiClient） | `selector.py`（JEV）、`selector_kev.py`（Kev） | +| 实现 | `planner.py`、`verifier.py`、`providers.py`（`KimiClient`，OpenAI 兼容方言） | `selector.py`（JEV）、`selector_kev.py`（Kev） | +| 当前默认 | `minimax-m3.1-flash`（可切 `kimi-k3`） | `jev`（可切 `kev`） | | 输入 | 指令 + 截图 + a11y 树 + 历史动作 + 需求清单 | 当前目标 + 候选动作（自带上下文描述）+ 截图（仅 JEV 需要） | | 输出 | 结构化 plan：当前目标 / 预期观察 / 路由 / 动作提案 | 排序结果：winner + operation/action 两级分数 + margin + 延迟 | | 代价 | 每步一次 LLM 调用，慢且贵 | 单次 HTTP 前向，快且便宜 | @@ -177,7 +178,7 @@ from tandem.config import SessionConfig from tandem.session import TandemSession components = build_default( - client=kimi_client, # providers.KimiClient + client=slow_client, # providers.KimiClient（慢模型客户端） endpoint="http://<排序服务>/rank", config=SessionConfig(fast_backend="jev"), ) @@ -218,6 +219,13 @@ tandem-eval rjob run --run-id --benchmark osworld2 --tasks \ | `slow_reasoning_effort` | `"low"` | `off`（归零思考）/ `low` / `medium` / `high` | | `fast_backend` | `"jev"` | `jev`（`/rank`，需截图）/ `kev`（System One，纯文本） | +**换慢模型厂商**：在 `infra/config.json` 的 `harness.slow_model.apis` 里登记多套 API， +启动时用 `--slow-model <名字>` 挑（缺省取 `default`），不改代码。 + +> 厂商方言有差异（2026-10-23 实测）：Kimi 认 `enable_thinking=false` 关思考； +> **MiniMax 忽略该字段**（仍返回 `reasoning_content`），只有 `reasoning_effort` 生效 +> （`low`/`medium`/`high`）——想少思考就把 `reasoning_effort` 调低，别指望 `off`。 + 慢模型的调用纪律（`providers.py`）： - 思考强度默认 `low`（实测 97 → 22 token）；空正文（全留在思考通道）→ 下一次重试自动升档； diff --git a/docs/CLUSTER_RUNTIME.md b/docs/CLUSTER_RUNTIME.md index 580769b..92c683a 100644 --- a/docs/CLUSTER_RUNTIME.md +++ b/docs/CLUSTER_RUNTIME.md @@ -36,7 +36,7 @@ worker 入口、共享资源与已知坑。 | 键 | 说明 | |---|---| | `harness.slow_model.default` | 默认慢模型（大脑）API 名 | -| `harness.slow_model.apis.<名字>` | 慢模型 API 条目：`base_url` / `api_key` / `model` + `max_tokens` / `timeout_seconds` / `reasoning_effort` / `direct_api` | +| `harness.slow_model.apis.<名字>` | 慢模型 API 条目：`base_url` / `api_key` / `model` + `max_tokens` / `timeout_seconds` / `reasoning_effort` / `direct_api`。当前登记 `minimax-m3.1-flash`（默认）与 `kimi-k3`（备选） | | `harness.fast_model.default` | 默认快模型（小脑）API 名 | | `harness.fast_model.apis.<名字>` | 快模型 API 条目：`backend`（`jev` / `kev`）+ `endpoint` / `endpoint_file` + `timeout_seconds` | | `harness.fast_model.image_stage.write` / `.read` | 截图中转目录的**两侧视角**：写入侧在 adpc，读取侧在 evobox（服务端宿主机要能读到） | diff --git a/infra/config.example.json b/infra/config.example.json index ac6c682..085da89 100644 --- a/infra/config.example.json +++ b/infra/config.example.json @@ -1,10 +1,19 @@ { "harness": { "slow_model": { - "default": "kimi-k3", + "default": "minimax-m3.1-flash", "apis": { + "minimax-m3.1-flash": { + "base_url": "https://api.minimaxi.com/v1", + "api_key": "sk-xxx", + "model": "MiniMax-M3.1-Flash", + "max_tokens": 16384, + "timeout_seconds": 180, + "reasoning_effort": "low", + "direct_api": false + }, "kimi-k3": { - "base_url": "https://your-slow-model-endpoint/v1", + "base_url": "https://your-kimi-endpoint/v1", "api_key": "sk-xxx", "model": "kimi-k3", "max_tokens": 16384, diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py index 40bec4d..852d340 100644 --- a/src/tandem_eval/config_schema.py +++ b/src/tandem_eval/config_schema.py @@ -73,6 +73,9 @@ #: 慢模型每个 API 条目必填的键 SLOW_API_KEYS = ("base_url", "api_key", "model") +#: 思考强度档位（与 harness 侧 SessionConfig.slow_reasoning_effort 保持一致） +REASONING_EFFORTS = ("off", "low", "medium", "high") + class ConfigError(ValueError): """配置缺项或取值非法（配置问题，不是运行问题）。""" @@ -112,6 +115,18 @@ def _validate_slow_models(config: dict) -> list[str]: value = api.get(key) if isinstance(api, dict) else None if _is_missing(value): problems.append(f"harness.slow_model.apis.{name}.{key}") + if not isinstance(api, dict): + continue + base_url = api.get("base_url") + if isinstance(base_url, str) and base_url.strip() and not base_url.startswith("http"): + problems.append(f"harness.slow_model.apis.{name}.base_url（应为 http(s) 地址）") + # 思考强度必须落在 harness 认识的档位里，否则 SessionConfig 会在运行期才报错 + effort = api.get("reasoning_effort") + if effort is not None and effort not in REASONING_EFFORTS: + problems.append( + f"harness.slow_model.apis.{name}.reasoning_effort" + f"（应为 {' / '.join(REASONING_EFFORTS)}）" + ) default = get_path(config, "harness.slow_model.default") if default not in apis: problems.append(f"harness.slow_model.default={default!r}（必须在 apis 里）") diff --git a/tests/eval/test_config.py b/tests/eval/test_config.py index 836d692..e8b3d9d 100644 --- a/tests/eval/test_config.py +++ b/tests/eval/test_config.py @@ -35,6 +35,25 @@ def test_example_config_has_three_sections_and_no_comment_fields(): assert [key for key in _walk_keys(config) if key.startswith("_")] == [] +def test_slow_model_entries_use_known_knobs(): + """配置里的旋钮必须落在 harness 认识的档位内，否则运行期才炸。""" + config = load_config(CONFIG_PATH) + for name, api in config["harness"]["slow_model"]["apis"].items(): + assert api["base_url"].startswith("http"), f"{name}: base_url" + assert api["reasoning_effort"] in {"off", "low", "medium", "high"}, f"{name}: reasoning_effort" + assert api["max_tokens"] > 0 and api["timeout_seconds"] > 0, f"{name}: 旋钮必须为正" + + +def test_slow_model_rejects_unknown_reasoning_effort(tmp_path): + config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) + name = next(iter(config["harness"]["slow_model"]["apis"])) + config["harness"]["slow_model"]["apis"][name]["reasoning_effort"] = "xhigh" + bad = tmp_path / "bad.json" + bad.write_text(json.dumps(config), encoding="utf-8") + with pytest.raises(ConfigError, match="reasoning_effort"): + load_config(bad) + + def test_models_are_named_api_registries(): config = load_config(CONFIG_PATH) slow = config["harness"]["slow_model"] From cda10e7fc2a3bd8ae976b461c8000de34026c719 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 23 Oct 2026 10:20:15 +0800 Subject: [PATCH 6/7] refactor: name the models by role, and accept any evidence shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The log said "Kimi response is missing required fields" while the brain was MiniMax - the wording pinned the harness to one vendor. Everything user-facing now speaks in roles: KimiClient -> SlowModelClient "Kimi response ..." -> "slow model response ..." "JEV health check failed" -> "fast model health check failed" "JEV returned a score count" -> "fast model returned a score count" "Kev 响应缺少 ..." -> "fast model 响应缺少 ..." event jev_service_recovery_wait -> fast_model_service_recovery_wait score_type jev_omni_choice_softmax -> fast_model_choice_softmax The two backend ids (jev / kev) and their classes keep their names: those name the sorting protocols, the way "openai" names a provider. The role is what the harness talks about; the protocol is what it speaks. This also fixes what the 2026-10-23 run actually died on: the verifier required evidence to be a string, MiniMax answers with a list (sometimes an object), so all three attempts failed validation and the episode ended in a ValueError. The validator now accepts str/list/dict and the result is normalised to text - the shape of an answer should not decide whether a run succeeds. Verification: ruff, pytest (243 passed) and tandem check are green; the demo site is rebuilt for the renamed fields. --- AGENTS.md | 2 +- DEVIATIONS.md | 2 +- README.md | 10 +++--- demo/README.md | 16 +++++----- docs/CLUSTER_RUNTIME.md | 2 +- docs/RESULT_DIR_CONTRACT.md | 2 +- src/tandem/providers.py | 12 +++---- src/tandem/selector.py | 14 ++++---- src/tandem/selector_kev.py | 2 +- src/tandem/verifier.py | 25 +++++++++++++-- src/tandem_eval/entry.py | 6 ++-- src/tandem_eval/recorder.py | 2 +- src/tandem_eval/runner.py | 2 +- tests/demo/helpers.py | 2 +- tests/harness/test_planner_verifier.py | 44 ++++++++++++++++++++++++++ tests/harness/test_selector_compile.py | 2 +- tests/harness/test_selector_kev.py | 2 +- tests/harness/test_session.py | 18 +++++------ 18 files changed, 115 insertions(+), 50 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8d9a7f2..699791e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,7 +57,7 @@ - 每个 run 的 `code_version.json` 必须记录 git describe、protocol_version、镜像 digest、run spec（secrets 脱敏）。 - OSW2 任务资产经 `OSWORLD_FILE_BASE_URL` 指向本地 `asset_root`（避免外网下载）；必须在任务模块 import **前**设置（见 osworld.py `_bootstrap_runtime`）。 -## 慢模型调用纪律（KimiClient，慢模型的当前实现） +## 慢模型调用纪律（SlowModelClient，慢模型客户端） - `reasoning_effort` 默认 `low`（探测 97→22 token）；`off` 映射 `enable_thinking=false`（归零）。 - 空正文（全留在思考通道）→ 下一次重试自动升档（low→medium→high）。 diff --git a/DEVIATIONS.md b/DEVIATIONS.md index be61381..c7e8200 100644 --- a/DEVIATIONS.md +++ b/DEVIATIONS.md @@ -75,7 +75,7 @@ 26. 【重构】**SessionConfig 字段角色化**：`kimi_max_tokens` / `kimi_reasoning_effort` / `kimi_timeout_seconds` / `kimi_direct_api` → `slow_*`；`jev_timeout_seconds` → `fast_timeout_seconds`；`selector_backend` → `fast_backend`。 - `KimiClient` 类名保留（它确实是 Kimi 方言的具体实现），但审计日志的 provider 字段 + `KimiClient` 类名当时保留（后改名为 `SlowModelClient`，慢模型客户端），审计日志的 provider 字段 由 `"kimi"` 改为 `"slow_model"`，worker 侧审计文件名 `kimi_http.jsonl` → `slow_model_http.jsonl`。 27. 【重构】**开发机侧新增 `config_schema.py`**：config 的校验与 API 选取独立成模块 diff --git a/README.md b/README.md index a5a263e..2ecb20d 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ │ ▼ ┌─────────────────────────────────────────────┐ -│慢模型 planner（Kimi） ← 理解与规划 │ +│慢模型 planner（大脑） ← 理解与规划 │ │ current_goal / expected_observation │ │ route / action_proposals / done │ └─────────────────────────────────────────────┘ @@ -39,7 +39,7 @@ │ Receipt：returncode / stdout / stderr ▼ ┌─────────────────────────────────────────────┐ -│慢模型 verifier（Kimi） ← 独立验证 │ +│慢模型 verifier（大脑） ← 独立验证 │ │ outcome · state_changed · evidence │ │ 逐条需求核验 requirement_checks │ └─────────────────────────────────────────────┘ @@ -55,7 +55,7 @@ | | 慢模型（大脑） | 快模型（小脑） | |---|---|---| | 角色 | 需求抽取 / 一步规划 / 独立验证 | 在候选空间里排序选择 | -| 实现 | `planner.py`、`verifier.py`、`providers.py`（`KimiClient`，OpenAI 兼容方言） | `selector.py`（JEV）、`selector_kev.py`（Kev） | +| 实现 | `planner.py`、`verifier.py`、`providers.py`（`SlowModelClient`，OpenAI 兼容方言） | `selector.py`（JEV）、`selector_kev.py`（Kev） | | 当前默认 | `minimax-m3.1-flash`（可切 `kimi-k3`） | `jev`（可切 `kev`） | | 输入 | 指令 + 截图 + a11y 树 + 历史动作 + 需求清单 | 当前目标 + 候选动作（自带上下文描述）+ 截图（仅 JEV 需要） | | 输出 | 结构化 plan：当前目标 / 预期观察 / 路由 / 动作提案 | 排序结果：winner + operation/action 两级分数 + margin + 延迟 | @@ -178,7 +178,7 @@ from tandem.config import SessionConfig from tandem.session import TandemSession components = build_default( - client=slow_client, # providers.KimiClient（慢模型客户端） + client=slow_client, # providers.SlowModelClient（慢模型客户端） endpoint="http://<排序服务>/rank", config=SessionConfig(fast_backend="jev"), ) @@ -279,7 +279,7 @@ python demo/build_site.py --runs-root rjob_logs --out demo/site --benchmark oswo python -m http.server 8899 --directory demo/site # 本地预览 ``` -披露口径：未出分如实标注、不折算 0 分、不计入均分；JEV 分数标注 softmax 非校准概率； +披露口径：未出分如实标注、不折算 0 分、不计入均分；排序分数标注 softmax 非校准概率； 成本只用 token 与调用次数表达，不臆造货币单价；截图标明是离散帧非录屏。 详见 [demo/README.md](demo/README.md)。 diff --git a/demo/README.md b/demo/README.md index 354e889..a0d83bb 100644 --- a/demo/README.md +++ b/demo/README.md @@ -42,7 +42,7 @@ site/ ### 总览页 - 每个 run 一张卡（已出分 / 未出分 / 均分 / 总耗时 / LLM 调用数 / 输入输出与思考 - token / JEV 排序次数 / 失败分类）； + token / 快模型排序次数 / 失败分类）； - **成本 × 准确率散点**：横轴 token 用量（平方根刻度），纵轴官方评分，气泡半径 ∝ 耗时；绿=有分、红=零分，未出分的题单独排在底部一排（不计入均分）； - **任务缩略图网格**（无帧的题显示"未进入决策步"占位）； @@ -50,19 +50,19 @@ site/ ### 对比页（`compare.html`） -任选两个 run 并排：逐题的分数 / token / 耗时 / 步数 / LLM 调用 / JEV 排序，以及 +任选两个 run 并排：逐题的分数 / token / 耗时 / 步数 / LLM 调用 / 快模型排序，以及 Δ差值（绿=更优、红=更差）。当前数据里 jev 与 kev 在任务 001 上可直接对照。 ### 任务页 -- **读数带**：官方评分、token 用量、真实用时、步数与 JEV 排序次数； +- **读数带**：官方评分、token 用量、真实用时、步数与快模型排序次数； - 左侧执行前 / 执行后双帧对照； - **需求清单 · 逐条核验**：从结构化字段（`verification.requirement_checks` + `task_requirements` 抽取结果）得到每条需求的终态结论与"在第几步被核验"， 点条目直接跳到那一步； - 右侧是这一步的完整模型操作链： 1. **规划**：当前目标、预期观察、路由、任务状态、动作提案； - 2. **候选与排序**：JEV 两阶段（操作级 → 目标级）softmax 分数与相对强度条、 + 2. **候选与排序**：快模型两阶段（操作级 → 目标级）softmax 分数与相对强度条、 margin、排序延迟； 3. **编译后动作**：kind / action_type / 参数 / 目标 id / 描述；CLI 路由附命令全文； 4. **执行回执与独立验证**：verifier 裁决、状态变化、证据、逐条需求核验、 @@ -84,7 +84,7 @@ site/ 终止前的模型调用次数与 role 分布； - **调用数也是 0 ⇒ 失败发生在模型被调用之前**（环境 / 依赖 / 配置）， 与「模型跑了但没做对」是两回事； -- 右侧「执行摘要」列出状态、评分口径、决策步、JEV 次数、调用分布、用时。 +- 右侧「执行摘要」列出状态、评分口径、决策步、快模型次数、调用分布、用时。 例：任务 017 的模型调用是 `task_requirements × 1 + plan_step × 3`（规划连续三次 非法 JSON 后抛 ValueError）；任务 020 是 0 次调用（环境问题）。 @@ -116,7 +116,7 @@ site/ | 泳道 | 区间 | 数据来源 | |------|------|----------| -| `plan` | 上一步结束 → Kimi 规划调用结束 | `llm_calls[].time`（响应落盘时刻） | +| `plan` | 上一步结束 → 慢模型规划调用结束 | `llm_calls[].time`（响应落盘时刻） | | `rank` | 规划结束 → +排序延迟 | `ranking.latency_ms`（服务端实测） | | `act` | 排序结束 → 本步观察采集时刻 | `traj[].wall_time` | | `verify` | 观察时刻 → 验证调用结束 | `llm_calls[].time` | @@ -128,8 +128,8 @@ LLM 调用按事件顺序归属步骤（实测序列恒为 `plan* → 观察 → 1. 只搬运**模型公开输出**，不生成任何虚构的"思考字幕"； 2. `failed_unscored` 一律显示「未出分」，**不折算 0 分、不计入均分**； -3. JEV `/rank` 分数始终标注 **softmax，非校准概率**； -4. 成本以 **token 用量 + 调用次数** 表达（成本代理指标），本仓库没有 Kimi 计价信息， +3. 排序服务分数始终标注 **softmax，非校准概率**； +4. 成本以 **token 用量 + 调用次数** 表达（成本代理指标），本仓库没有慢模型计价信息， 不臆造货币单价； 5. 截图为**逐步离散帧**，非连续录屏；初始观察帧与终态动作后的帧不随结果目录保留， 页面如实标注缺失原因； diff --git a/docs/CLUSTER_RUNTIME.md b/docs/CLUSTER_RUNTIME.md index 92c683a..d33cccc 100644 --- a/docs/CLUSTER_RUNTIME.md +++ b/docs/CLUSTER_RUNTIME.md @@ -196,7 +196,7 @@ JEV / Kev 部署在闲时 GPU，**地址随重新部署变化**。因此： JEV 的 `/rank` 要求 `image_path` 是**服务宿主机可读的文件路径**。截图本体在 adpc， 因此由 harness 暂存到 evobox 中转目录（`harness.fast_model.image_stage.write` → `.read`）， -payload 带 JEV 宿主机视角路径。暂存按内容哈希命名去重（`step_00_.png`），断点无害。 +payload 带排序服务宿主机视角路径。暂存按内容哈希命名去重（`step_00_.png`），断点无害。 ## 6.5 OSWorld 2.1 的特别约定 diff --git a/docs/RESULT_DIR_CONTRACT.md b/docs/RESULT_DIR_CONTRACT.md index d0f5674..c9b40af 100644 --- a/docs/RESULT_DIR_CONTRACT.md +++ b/docs/RESULT_DIR_CONTRACT.md @@ -24,7 +24,7 @@ |------|------|---------| | `task.json` | 任务定义完整副本 | episode 开始 | | `traj.jsonl` | 每执行步一行：`{step_idx, wall_time, protocol_version, observation{screenshot(sha256/path/w/h), accessibility_tree(present/source/digest/node_count), budget(steps_remaining, wall_seconds_remaining), last_receipt}, action{kind, …}}` | 每步 env.execute 后 | -| `llm_calls.jsonl` | 全量 LLM 请求/响应，每行 `{role, time, request{payload, messages}, response, attempt}`；Bearer/api_key 脱敏 | KimiClient 成功响应时（llm_sink → LlmTap → 本文件） | +| `llm_calls.jsonl` | 全量 LLM 请求/响应，每行 `{role, time, request{payload, messages}, response, attempt}`；Bearer/api_key 脱敏 | 慢模型客户端成功响应时（llm_sink → LlmTap → 本文件） | | `events.jsonl` | harness 决策事件（route 字段区分）：`planner`（发给 LLM 的计划全文 + 有效响应）/ `gui`（候选列表 + ranking{operation_request, action_request, operation_ranked, action_ranked, winner, margins, latency_ms, used_image} + 编译后动作）/ `cli`（路由决策 + 命令）/ `gui_verified` / `cli_verified`（验证裁决 + 回执）/ `terminal`（终态双确认）/ `action`（runner 写的执行记录，与 session 事件穿插） | session.act() 后增量排水 | | `a11y_digests.jsonl` | 每步 a11y sha256 + 节点数（`{step_idx, sha256, node_count}`） | 每步（全量 xml 默认关闭，record.a11y_full=true 时另写 `a11y/step_XX.xml`） | | `screenshots/step_XX.png` | 逐步截图（**契约正本**） | 每步 env.execute 后 | diff --git a/src/tandem/providers.py b/src/tandem/providers.py index 6e09cde..e38fb9c 100644 --- a/src/tandem/providers.py +++ b/src/tandem/providers.py @@ -1,5 +1,5 @@ """慢模型（大脑）的 JSON 传输客户端——OpenAI 兼容 /chat/completions 口径， -当前实现沿用 Kimi 的方言（reasoning_effort / enable_thinking）。含脱敏审计。 +当前实现用 OpenAI 兼容方言（reasoning_effort / enable_thinking）——Kimi 与 MiniMax 都吃这套。含脱敏审计。 从旧仓库 providers.py 按契约移植。 全量请求/响应落盘（llm_calls.jsonl）属于 M2 的 recorder，此处保持审计日志。""" @@ -50,7 +50,7 @@ def extract_json_object(text: str): return None -class KimiClient: +class SlowModelClient: def __init__(self, credential_file, config, audit_path: Path, llm_sink=None): if isinstance(credential_file, dict): self.credentials = dict(credential_file) @@ -158,19 +158,19 @@ def redact(text): if finish_reason == "length": completion_tokens = (raw.get("usage") or {}).get("completion_tokens") raise ValueError( - f"Kimi response truncated by max_tokens (finish_reason=length, " + f"slow model response truncated by max_tokens (finish_reason=length, " f"completion_tokens={completion_tokens}); raise slow_max_tokens" ) if not content and raw["choices"][0]["message"].get("reasoning_content"): raise ValueError( - "Kimi returned empty content (all completion tokens stayed in the " + "slow model returned empty content (all completion tokens stayed in the " "reasoning channel); escalating reasoning_effort for the next attempt" ) value = extract_json_object(content) if value is None: - raise ValueError("Kimi response contains no JSON object") + raise ValueError("slow model response contains no JSON object") if not validate(value): - raise ValueError("Kimi response is missing required fields") + raise ValueError("slow model response is missing required fields") return value except urllib.error.HTTPError as error: body = redact(error.read().decode("utf-8", errors="replace")) diff --git a/src/tandem/selector.py b/src/tandem/selector.py index e5b03fc..8947bf1 100644 --- a/src/tandem/selector.py +++ b/src/tandem/selector.py @@ -1,4 +1,4 @@ -"""selector：JEV omni 选择策略（HTTP 排序服务）。从旧仓库 gui/jev_omni_policy.py 按契约移植； +"""selector：快模型 omni 选择策略（HTTP 排序服务）。从旧仓库 gui/jev_omni_policy.py 按契约移植； 唯一接口差异：opener 可注入（测试），默认行为不变。""" from __future__ import annotations @@ -98,12 +98,12 @@ def __init__(self, endpoint: str, timeout_seconds: float = 300, opener=None, self.endpoint = str(endpoint).rstrip("/") self.timeout_seconds = timeout_seconds self.opener = opener or urllib.request.build_opener(urllib.request.ProxyHandler({})) - # (写视角目录, JEV 宿主机读视角目录)：/rank 要求 image_path 在服务端可读， + # (写视角目录, 排序服务宿主机读视角目录)：/rank 要求 image_path 在服务端可读， # 截图本体在 adpc 时先暂存到 evobox 中转目录（按内容哈希命名去重）。 self.image_stage = image_stage def _stage_screenshot(self, screenshot: Path | None): - """返回传给服务的 image_path：暂存开启时为 JEV 宿主机读视角路径，否则原样。""" + """返回传给服务的 image_path：暂存开启时为排序服务宿主机读视角路径，否则原样。""" if not self.image_stage or screenshot is None: return screenshot source = Path(screenshot) @@ -146,7 +146,7 @@ def predict(choice_request: dict[str, Any]) -> tuple[list[float], bool]: try: with self.opener.open(self.endpoint + "/", timeout=5) as response: if json.load(response).get("status") != "ok": - raise OSError("JEV health check failed") + raise OSError("fast model health check failed") request = urllib.request.Request( self.endpoint + "/rank", data=json.dumps( @@ -168,7 +168,7 @@ def predict(choice_request: dict[str, Any]) -> tuple[list[float], bool]: print( json.dumps( { - "event": "jev_service_recovery_wait", + "event": "fast_model_service_recovery_wait", "error": str(error), "remaining_seconds": round(deadline - time.monotonic()), }, @@ -179,7 +179,7 @@ def predict(choice_request: dict[str, Any]) -> tuple[list[float], bool]: time.sleep(15) scores = [float(score) for score in payload["scores"]] if len(scores) != len(choice_request["options"]): - raise ValueError("JEV returned a score count different from the candidate count") + raise ValueError("fast model returned a score count different from the candidate count") return scores, bool(payload.get("used_image")) operation_scores, operation_used_image = predict(operation_request) @@ -236,7 +236,7 @@ def ranking_dict(value: Ranking) -> dict[str, Any]: "target_margin": value.target_margin, "latency_ms": value.latency_ms, "used_image": value.used_image, - "score_type": "jev_omni_choice_softmax", + "score_type": "fast_model_choice_softmax", "score_is_calibrated_correctness_probability": False, "operation_request": value.operation_request, "action_request": value.action_request, diff --git a/src/tandem/selector_kev.py b/src/tandem/selector_kev.py index 271205d..fafae5c 100644 --- a/src/tandem/selector_kev.py +++ b/src/tandem/selector_kev.py @@ -78,7 +78,7 @@ def _predict(self, choice_request: dict[str, Any]) -> tuple[list[float], bool]: try: probabilities = response["answers"]["q"]["probabilities"] except (KeyError, TypeError) as error: - raise ValueError(f"Kev 响应缺少 answers.q.probabilities: {response}") from error + raise ValueError(f"fast model 响应缺少 answers.q.probabilities: {response}") from error scores = [float(probabilities.get(option_id, 0.0)) for option_id in choice_request["option_ids"]] return scores, False # 纯文本后端，used_image 恒 false diff --git a/src/tandem/verifier.py b/src/tandem/verifier.py index 0b1ddda..b21aea9 100644 --- a/src/tandem/verifier.py +++ b/src/tandem/verifier.py @@ -7,6 +7,22 @@ from pathlib import Path +def as_text(value) -> str: + """把模型可能给出的多种 evidence 形状压成一段文本。 + + 同一句 "evidence"，不同厂商给的形状不一样：字符串、字符串列表、字典都出现过 + （MiniMax 2026-10-23 实测给了列表与字典）。校验器只认字符串，所以这里统一压平， + 免得因为形状差异把整题判失败——大脑是可换的，形状不该是失败原因。 + """ + if isinstance(value, str): + return value + if isinstance(value, list): + return "\n".join(f"- {as_text(item)}" for item in value) + if isinstance(value, dict): + return "\n".join(f"{key}: {as_text(item)}" for key, item in value.items()) + return "" if value is None else str(value) + + def verify_observation( client, instruction, @@ -97,7 +113,7 @@ def verify_observation( value.get("outcome") in {"achieved", "not_achieved", "uncertain"} and value.get("task_status") in {"completed", "incomplete", "infeasible"} and isinstance(value.get("path_input_focused"), bool) - and isinstance(value.get("evidence"), str) + and isinstance(value.get("evidence"), (str, list, dict)) and isinstance(value.get("state_changed"), bool) and ( not terminal_claim @@ -105,13 +121,18 @@ def verify_observation( and all( isinstance(item, dict) and item.get("status") in {"met", "not_met", "uncertain"} - and isinstance(item.get("evidence"), str) + and isinstance(item.get("evidence"), (str, list, dict)) for item in value["requirement_checks"] ) ) ), role="verify_observation", ) + # 形状归一：evidence 一律压成字符串（下游 control_state_check 要拼字符串） + result["evidence"] = as_text(result.get("evidence")) + for item in result.get("requirement_checks", []) or []: + if isinstance(item, dict): + item["evidence"] = as_text(item.get("evidence")) if control_state_check is not None: result["control_state_check"] = control_state_check if not control_state_check["changed"]: diff --git a/src/tandem_eval/entry.py b/src/tandem_eval/entry.py index 061f165..7577c0b 100644 --- a/src/tandem_eval/entry.py +++ b/src/tandem_eval/entry.py @@ -1,7 +1,7 @@ """entry：job spec → 环境/harness/runner 组装（worker 侧唯一入口，infra/worker.sh 调用）。 benchmark=mock：MockEnv + 确定性脚本组件（全链路自检）。 -真实基准：环境适配器（共享目录 runtime/VM）+ KimiClient + build_default 组件， +真实基准：环境适配器（共享目录 runtime/VM）+ SlowModelClient + build_default 组件， 凭据与端点路径由提交侧从 infra/config.json 注入 spec（secrets 段）。 """ @@ -161,14 +161,14 @@ def build_session(spec: dict, *, llm_tap: LlmTap | None = None) -> TandemSession return TandemSession(mock_components(), config=config) if benchmark in {"osworld1", "osworld2", "osworld21"}: from tandem.components import build_default - from tandem.providers import KimiClient + from tandem.providers import SlowModelClient secrets = spec["secrets"] credentials = dict(secrets["slow_model"]) endpoint = resolve_fast_model_endpoint(secrets.get("fast_model") or {}) run_root = Path(spec["output_root"]) run_root.mkdir(parents=True, exist_ok=True) - client = KimiClient(credentials, config, run_root / "slow_model_http.jsonl", + client = SlowModelClient(credentials, config, run_root / "slow_model_http.jsonl", llm_sink=llm_tap) stage = spec.get("image_stage") or {} image_stage = None diff --git a/src/tandem_eval/recorder.py b/src/tandem_eval/recorder.py index df8a11a..384b8a6 100644 --- a/src/tandem_eval/recorder.py +++ b/src/tandem_eval/recorder.py @@ -69,7 +69,7 @@ def event(self, event: dict) -> None: self._events.flush() def record_llm_call(self, *, role: str, request: Any, response: Any, **meta: Any) -> None: - """KimiClient 的 llm_sink；全量请求/响应落盘，密钥脱敏。""" + """慢模型客户端的 llm_sink；全量请求/响应落盘，密钥脱敏。""" self._llm.write( json.dumps( _redact( diff --git a/src/tandem_eval/runner.py b/src/tandem_eval/runner.py index 024f06e..9ddf512 100644 --- a/src/tandem_eval/runner.py +++ b/src/tandem_eval/runner.py @@ -27,7 +27,7 @@ def run_episode(env, session: TandemSession, recorder: EpisodeRecorder, task: di stop_reason = "max_steps" observation: Observation = env.reset(task) outputs: list[dict] = [] - events_start = len(session.events) # 本题事件起点：act 后增量落盘（planner/JEV/验证） + events_start = len(session.events) # 本题事件起点：act 后增量落盘（planner/快模型/验证） def drain_events() -> None: nonlocal events_start diff --git a/tests/demo/helpers.py b/tests/demo/helpers.py index bb3dbb4..1c0b555 100644 --- a/tests/demo/helpers.py +++ b/tests/demo/helpers.py @@ -107,7 +107,7 @@ def make_run(root: Path, run_id: str, tasks: list[dict]) -> Path: "action_ranked": [{"id": "click:a11y:t0:0", "score": 0.8}], "operation_margin": 0.5, "latency_ms": 120.0, - "score_type": "jev_omni_choice_softmax", + "score_type": "fast_model_choice_softmax", "score_is_calibrated_correctness_probability": False, }, "action": {"kind": "gui", "action_type": "click", "parameters": {"x": 1, "y": 2}}, diff --git a/tests/harness/test_planner_verifier.py b/tests/harness/test_planner_verifier.py index 44edeb8..2c335b9 100644 --- a/tests/harness/test_planner_verifier.py +++ b/tests/harness/test_planner_verifier.py @@ -158,3 +158,47 @@ def test_task_requirements_output_path_sanitization(instruction, raw_path, expec ) result = task_requirements(client, instruction) assert result[0]["output_path"] == expected + + +@pytest.mark.parametrize("evidence", [ + ["要点一", "要点二"], # MiniMax 2026-10-23 实测形状 + {"可见现象": "工具栏没有变化", "结论": "未完成"}, +]) +def test_verify_observation_accepts_list_and_dict_evidence(tmp_path, evidence): + """evidence 的形状不该成为失败原因：不同厂商给字符串/列表/字典都出现过。 + + 2026-10-23 实跑：MiniMax 给的是列表（偶尔字典），Kimi 给的是字符串； + 原来的校验只认字符串 → 整题被判 ValueError 失败。 + """ + screenshot = tmp_path / "s.png" + screenshot.write_bytes(b"img") + value = { + "outcome": "not_achieved", + "evidence": evidence, + "task_status": "incomplete", + "path_input_focused": False, + "state_changed": False, + } + client = FakeClient(value) + result = verify_observation(client, "改一下样式", screenshot) + assert isinstance(result["evidence"], str) and result["evidence"].strip() + + +def test_verify_observation_normalizes_requirement_check_evidence(tmp_path): + screenshot = tmp_path / "s.png" + screenshot.write_bytes(b"img") + value = { + "outcome": "not_achieved", + "evidence": "未达成", + "task_status": "incomplete", + "path_input_focused": False, + "state_changed": False, + "requirement_checks": [{"id": "R1", "status": "not_met", "evidence": ["只列了文件"]}], + } + client = FakeClient(value) + result = verify_observation( + client, "改一下样式", screenshot, + terminal_claim=True, requirements=[{"id": "R1", "requirement": "改样式"}], + ) + check = result["requirement_checks"][0] + assert isinstance(check["evidence"], str) and "只列了文件" in check["evidence"] diff --git a/tests/harness/test_selector_compile.py b/tests/harness/test_selector_compile.py index b81a3bd..6b616dc 100644 --- a/tests/harness/test_selector_compile.py +++ b/tests/harness/test_selector_compile.py @@ -67,7 +67,7 @@ def test_rank_picks_highest_scored_candidate(tmp_path): assert ranking.action_ranked[0]["score"] == 0.8 assert ranking.target_margin == pytest.approx(0.8 - 0.2) assert ranking.used_image is True - assert ranking_dict(ranking)["score_type"] == "jev_omni_choice_softmax" + assert ranking_dict(ranking)["score_type"] == "fast_model_choice_softmax" def test_rank_single_candidate_skips_action_request(tmp_path): diff --git a/tests/harness/test_selector_kev.py b/tests/harness/test_selector_kev.py index 2c4b69b..d4d731d 100644 --- a/tests/harness/test_selector_kev.py +++ b/tests/harness/test_selector_kev.py @@ -69,7 +69,7 @@ def test_rank_two_stage_with_probabilities(tmp_path): assert ranking.winner.id == second_id assert ranking.target_margin == pytest.approx(0.8 - 0.2) assert ranking.used_image is False - assert ranking_dict(ranking)["score_type"] == "jev_omni_choice_softmax" + assert ranking_dict(ranking)["score_type"] == "fast_model_choice_softmax" # 请求体符合 System One：state 为对象、criteria 为选项→描述 first = opener.requests[0] assert first["model"] == "kev-latest" diff --git a/tests/harness/test_session.py b/tests/harness/test_session.py index cbf7f2c..4d30d70 100644 --- a/tests/harness/test_session.py +++ b/tests/harness/test_session.py @@ -255,7 +255,7 @@ def test_truncated_reasoning_raises_actionable_error(tmp_path, monkeypatch): """finish_reason=length（推理吃满 max_tokens、正文为空）必须报可定位的错误。""" import tandem.providers as providers_module from tandem.config import SessionConfig - from tandem.providers import KimiClient + from tandem.providers import SlowModelClient monkeypatch.setattr(providers_module.time, "sleep", lambda seconds: None) @@ -279,7 +279,7 @@ def read(self): "usage": {"completion_tokens": 4099}, }).encode() - client = KimiClient({"base_url": "http://x"}, SessionConfig(), + client = SlowModelClient({"base_url": "http://x"}, SessionConfig(), tmp_path / "audit.jsonl") import urllib.request opener_holder = TruncatingOpener() @@ -299,7 +299,7 @@ def test_kimi_payload_controls_reasoning_and_tokens(tmp_path): import urllib.request from tandem.config import SessionConfig - from tandem.providers import KimiClient + from tandem.providers import SlowModelClient captured = {} @@ -326,11 +326,11 @@ def fake_urlopen(request, timeout=None): urllib.request.urlopen = fake_urlopen try: config = SessionConfig() - client = KimiClient({"base_url": "http://x"}, config, tmp_path / "a.jsonl") + client = SlowModelClient({"base_url": "http://x"}, config, tmp_path / "a.jsonl") client.json_request([{"role": "user", "content": "hi"}], lambda v: True, role="r") assert captured["payload"]["max_tokens"] == 16384 assert captured["payload"]["reasoning_effort"] == "low" - client_off = KimiClient({"base_url": "http://x"}, SessionConfig(slow_reasoning_effort="off"), + client_off = SlowModelClient({"base_url": "http://x"}, SessionConfig(slow_reasoning_effort="off"), tmp_path / "a.jsonl") client_off.json_request([{"role": "user", "content": "hi"}], lambda v: True, role="r") assert captured["payload"]["enable_thinking"] is False @@ -367,7 +367,7 @@ def test_kimi_retries_escalate_reasoning_effort_on_empty_content(tmp_path): import urllib.request from tandem.config import SessionConfig - from tandem.providers import KimiClient + from tandem.providers import SlowModelClient payloads = [] responses = iter([ @@ -401,7 +401,7 @@ def fake_urlopen(request, timeout=None): original = urllib.request.urlopen urllib.request.urlopen = fake_urlopen try: - client = KimiClient({"base_url": "http://x"}, SessionConfig(), tmp_path / "a.jsonl") + client = SlowModelClient({"base_url": "http://x"}, SessionConfig(), tmp_path / "a.jsonl") value = client.json_request([{"role": "user", "content": "hi"}], lambda v: v == {"done": True}, role="plan_step") assert value == {"done": True} @@ -426,7 +426,7 @@ def test_kimi_accepts_prose_wrapped_json(tmp_path): import urllib.request from tandem.config import SessionConfig - from tandem.providers import KimiClient + from tandem.providers import SlowModelClient class _Resp: status, headers = 200, {} @@ -446,7 +446,7 @@ def read(self): original = urllib.request.urlopen urllib.request.urlopen = lambda request, timeout=None: _Resp() try: - client = KimiClient({"base_url": "http://x"}, SessionConfig(), tmp_path / "a.jsonl") + client = SlowModelClient({"base_url": "http://x"}, SessionConfig(), tmp_path / "a.jsonl") assert client.json_request([{"role": "user", "content": "hi"}], lambda v: v == {"done": True}, role="r") == {"done": True} finally: From 5852e9f1f9687332cd8ad8fb99d9c36b2180e404 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 23 Oct 2026 12:47:59 +0800 Subject: [PATCH 7/7] fix(eval): record the exception message and traceback in the episode record A failed episode only stored exception_type, so a run that died in the middle left nothing to work with: the 2026-10-23 osw2-kev run had a RuntimeError after 623s of CLI work and a ValueError 1ms into the next task, and both were indistinguishable from each other without the message. episode.json now also carries the error message and the tail of the traceback (redacted, truncated). The result-dir contract documents the two new optional fields, and the demo shows the message next to the failure reason so a failed task explains itself in the player. Verification: ruff, pytest (245 passed) and tandem check are green. --- demo/adapter.py | 1 + demo/render.py | 2 ++ docs/RESULT_DIR_CONTRACT.md | 2 +- src/tandem_eval/recorder.py | 18 ++++++++++++++++ tests/eval/test_recorder_contract.py | 31 ++++++++++++++++++++++++++++ 5 files changed, 53 insertions(+), 1 deletion(-) diff --git a/demo/adapter.py b/demo/adapter.py index 81a32e1..fb79741 100644 --- a/demo/adapter.py +++ b/demo/adapter.py @@ -207,6 +207,7 @@ def load_episode(run_dir: Path, entry: dict[str, Any]) -> dict[str, Any]: "score": score if isinstance(score, (int, float)) else None, "stop_reason": episode.get("stop_reason"), "exception_type": episode.get("exception_type"), + "error": episode.get("error"), "steps_reported": episode.get("steps"), "wall_seconds": timing.get("wall_seconds"), "evaluation": evaluation, diff --git a/demo/render.py b/demo/render.py index 6709082..563aea4 100644 --- a/demo/render.py +++ b/demo/render.py @@ -129,6 +129,8 @@ def _readout(task: dict[str, Any]) -> str: score_big = "未出分" score_cls = "na" score_sub = f"{task['exception_type'] or task['stop_reason'] or '异常'} · 如实标注，不计 0 分" + if task.get("error"): + score_sub += f" · {str(task['error'])[:200]}" total = task["cost"]["total"] cost_big = f"{total['prompt_tokens'] + total['completion_tokens']:,}" cost_sub = ( diff --git a/docs/RESULT_DIR_CONTRACT.md b/docs/RESULT_DIR_CONTRACT.md index c9b40af..40b01a5 100644 --- a/docs/RESULT_DIR_CONTRACT.md +++ b/docs/RESULT_DIR_CONTRACT.md @@ -32,7 +32,7 @@ | `artifacts.json` | deliverables 导出记录（`[{vm_path, exported, local_path, bytes}]`） | episode 终态 | | `evaluation.json` | 官方评分（`{source: "official_benchmark_evaluator", raw, score}`） | episode 终态（评分成功时） | | `result.txt` | 分数（`score\n`） | 同上 | -| `episode.json` | `{task_id, harness, stop_reason, steps, status(scored\|failed_unscored), score, exception_type}` | episode 终态（必写） | +| `episode.json` | `{task_id, harness, stop_reason, steps, status(scored\|failed_unscored), score, exception_type, error?, error_traceback?}` | episode 终态（必写）。`error`/`error_traceback` 仅异常题有：异常消息与栈尾几行（脱敏）——只记类型事后没法定位 | | `timing.json` | `{wall_seconds, status}` | episode 终态 | ## 步号语义（demo 侧对齐依据） diff --git a/src/tandem_eval/recorder.py b/src/tandem_eval/recorder.py index 384b8a6..62fe82b 100644 --- a/src/tandem_eval/recorder.py +++ b/src/tandem_eval/recorder.py @@ -11,6 +11,7 @@ import json import re import time +import traceback from pathlib import Path from typing import Any @@ -31,6 +32,20 @@ def _redact(value: Any) -> Any: return value +_SECRET_PATTERN = re.compile(r"sk-[A-Za-z0-9._-]{6,}") + + +def _redact_text(text: str, limit: int = 500) -> str: + """异常消息可能带出凭据，落盘前脱敏并截断。""" + return _SECRET_PATTERN.sub("[REDACTED]", text)[:limit] + + +def _traceback_tail(exception: BaseException, limit: int = 8) -> list[str]: + """异常栈尾几行——只记类型不够，事后没法判断是哪一步炸的。""" + lines = traceback.format_exception(type(exception), exception, exception.__traceback__) + return [_redact_text(line.rstrip(), 300) for line in lines[-limit:]] + + def _write_json(path: Path, payload: Any) -> None: path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") @@ -131,6 +146,9 @@ def finalize( "score": score, "exception_type": type(exception).__name__ if exception else None, } + if exception is not None: + episode["error"] = _redact_text(str(exception)) + episode["error_traceback"] = _traceback_tail(exception) _write_json(self.dir / "episode.json", episode) _write_json( self.dir / "timing.json", diff --git a/tests/eval/test_recorder_contract.py b/tests/eval/test_recorder_contract.py index e0b04d6..b66fa66 100644 --- a/tests/eval/test_recorder_contract.py +++ b/tests/eval/test_recorder_contract.py @@ -96,3 +96,34 @@ def test_run_spec_masks_credentials_but_keeps_model_identity(): assert masked["secrets"]["fast_model"]["endpoint"] == "***" assert masked["secrets"]["registry"]["auths"] == "***" assert masked["harness"]["models"] == {"slow": "kimi-k3", "fast": "jev"} + + +def test_episode_records_error_message_and_traceback(tmp_path): + """异常题必须记下消息与栈尾：只记 exception_type 的话，事后无法判断炸在哪一步。""" + from tandem_eval.recorder import EpisodeRecorder + + recorder = EpisodeRecorder(tmp_path / "ep", {"id": "016"}) + try: + raise ValueError("asset() requires a non-empty relative path") + except ValueError as exc: + episode = recorder.finalize( + stop_reason="exception", outputs=[], raw_score=None, score=None, + status="failed_unscored", exception=exc, + ) + assert episode["exception_type"] == "ValueError" + assert "non-empty relative path" in episode["error"] + assert any("ValueError" in line for line in episode["error_traceback"]) + stored = json.loads((tmp_path / "ep" / "episode.json").read_text()) + assert stored["error"].startswith("asset()") + + +def test_episode_error_redacts_secrets(tmp_path): + from tandem_eval.recorder import EpisodeRecorder + + recorder = EpisodeRecorder(tmp_path / "ep", {"id": "x"}) + episode = recorder.finalize( + stop_reason="exception", outputs=[], raw_score=None, score=None, + status="failed_unscored", exception=RuntimeError("boom sk-secretkey123456"), + ) + assert "sk-secretkey" not in episode["error"] + assert "[REDACTED]" in episode["error"]