From 7dfa7969557818c9201ce023939140662bb85dbe Mon Sep 17 00:00:00 2001 From: xxxyxun Date: Sat, 24 Oct 2026 04:45:06 +0800 Subject: [PATCH 1/4] feat(harness): add stage execution with rule verification and replanning --- DEVIATIONS.md | 10 ++ README.md | 6 + docs/CLUSTER_RUNTIME.md | 2 + docs/RESULT_DIR_CONTRACT.md | 10 ++ docs/STAGE_EXECUTION.md | 196 +++++++++++++++++++++++ infra/config.example.json | 4 + src/tandem/components.py | 3 +- src/tandem/config.py | 6 + src/tandem/episode.py | 19 ++- src/tandem/session.py | 34 ++-- src/tandem/stage_planner.py | 168 ++++++++++++++++++++ src/tandem/stage_rules.py | 75 +++++++++ src/tandem/stages.py | 93 +++++++++++ src/tandem_eval/__main__.py | 7 + src/tandem_eval/config_schema.py | 9 ++ src/tandem_eval/entry.py | 15 +- src/tandem_eval/rjob_spec.py | 5 +- tests/eval/test_mock_chain.py | 7 +- tests/eval/test_stage_config.py | 60 +++++++ tests/harness/test_stages.py | 265 +++++++++++++++++++++++++++++++ 20 files changed, 966 insertions(+), 28 deletions(-) create mode 100644 docs/STAGE_EXECUTION.md create mode 100644 src/tandem/stage_planner.py create mode 100644 src/tandem/stage_rules.py create mode 100644 src/tandem/stages.py create mode 100644 tests/eval/test_stage_config.py create mode 100644 tests/harness/test_stages.py diff --git a/DEVIATIONS.md b/DEVIATIONS.md index c7e8200..40a0823 100644 --- a/DEVIATIONS.md +++ b/DEVIATIONS.md @@ -83,3 +83,13 @@ `demo/` 同步角色化：run 文档 `selector_backend` → `fast_model`（新增 `slow_model` 名）， 排序调用计数 `jev_calls` → `fast_calls`、`jev_latency_ms_total` → `fast_latency_ms_total` （这两个是 demo 层自算字段，不在结果目录契约里）。 + +## 可选阶段执行（2026-10-24） + +28. 【行为，可选】`execution_mode=stage` 时，慢模型一次规划当前阶段的多个里程碑， + 快模型依据每帧新观察重新选择动作；规则后置条件优先，证据不足才调用模型 Verify。 + 阶段完成、目标无法绑定、执行失败或阶段动作上限触发后返回慢模型。 + 不采用严格只规划一次的模式；阶段完成不等于任务完成，原有独立终态检查保留。 + 默认 `step` 路径不变，动作/观察/回执协议不变。 + 此为旧阶段执行实验的组件化移植，不包含其传输重试、部署或环境准备逻辑。 + 详细实现边界见 [STAGE_EXECUTION.md](docs/STAGE_EXECUTION.md)。 diff --git a/README.md b/README.md index 2ecb20d..9ba793e 100644 --- a/README.md +++ b/README.md @@ -293,3 +293,9 @@ python -m http.server 8899 --directory demo/site # 本地预览 | [DEVIATIONS.md](DEVIATIONS.md) | 与旧仓库 SA-CUAharness 的已知行为差异清单 | | [AGENTS.md](AGENTS.md) | 开发纪律与目录地图 | | [demo/README.md](demo/README.md) | demo 站点说明 | + +## 阶段执行（可选） + +默认保持逐步规划；启用 `--execution-mode stage` 后，由慢模型规划阶段、快模型连续选择动作， +规则验证优先，仅在必要时调用模型验证或重规划。 +配置、执行边界、开发入口和验收方法见 [阶段执行说明](docs/STAGE_EXECUTION.md)。 diff --git a/docs/CLUSTER_RUNTIME.md b/docs/CLUSTER_RUNTIME.md index d33cccc..5da581e 100644 --- a/docs/CLUSTER_RUNTIME.md +++ b/docs/CLUSTER_RUNTIME.md @@ -41,6 +41,7 @@ worker 入口、共享资源与已知坑。 | `harness.fast_model.apis.<名字>` | 快模型 API 条目：`backend`（`jev` / `kev`）+ `endpoint` / `endpoint_file` + `timeout_seconds` | | `harness.fast_model.image_stage.write` / `.read` | 截图中转目录的**两侧视角**：写入侧在 adpc，读取侧在 evobox（服务端宿主机要能读到） | | `harness.route.policy` / `.allow_cli` / `.cli_timeout_seconds` | 路由策略与 CLI 开关 | +| `harness.execution.mode` / `.stage_max_actions` | 可选：step（默认）或 stage；阶段内动作上限默认 24。见 [阶段执行说明](STAGE_EXECUTION.md) | | `harness.budget.max_steps` / `.max_wall_seconds` | 单题预算默认值（命令行可覆盖） | **慢模型与快模型都是命名 API 登记表**：登记多套大脑/小脑，启动命令用 `--slow-model` / @@ -131,6 +132,7 @@ cleanup stop（释放资源）+ delete（永久删除 rjob 记录）——默 | `--slow-model NAME` | 慢模型（大脑）API 名，取自 `harness.slow_model.apis`；缺省用 `default` | | `--fast-model NAME` | 快模型（小脑）API 名，取自 `harness.fast_model.apis`；缺省用 `default` | | `--route-policy` / `--allow-cli` | 路由策略与 CLI 开关 | +| `--execution-mode` / `--stage-max-actions` | step/stage 模式与阶段动作上限 | | `--max-steps` / `--max-wall` | 单题预算（步数 / 墙钟秒） | | `--slow-reasoning` | `off` / `low`（默认）/ `medium` / `high` | | `--slow-max-tokens` | 慢模型单次响应上限 | diff --git a/docs/RESULT_DIR_CONTRACT.md b/docs/RESULT_DIR_CONTRACT.md index 40b01a5..49e1110 100644 --- a/docs/RESULT_DIR_CONTRACT.md +++ b/docs/RESULT_DIR_CONTRACT.md @@ -58,6 +58,7 @@ |------|--------|---------| | `task_requirements` | verifier.task_requirements | episode 开始时抽取任务需求（1-12 条） | | `plan_step` | planner.plan_step | 每步规划（截图+可见控件+需求+近期动作→下一步决策） | +| `plan_stage` | stage_planner.plan_stage | 仅 stage 模式：初始阶段、阶段完成或请求修复时生成阶段计划 | | `verify_observation` | verifier.verify_observation | 每步动作执行后独立验证（BEFORE/AFTER 截图→achieved/not_achieved/uncertain）| `verify_observation` 不信任 planner 的自我宣称：它拿前后截图+预期变化独立裁决，驱动 recent 历史、no_action 连击计数、toggle 检查、以及终态双确认（terminal_claim 模式下逐条核验 requirements + 输出文件存在性）。 @@ -71,8 +72,17 @@ | `cli` | planner 的 route=cli 决策 + 命令 | | `gui_verified` / `cli_verified` | 验证裁决（outcome/task_status/evidence/state_changed）+ 回执 | | `terminal` | 终态裁决（planner 声称完成后的 verifier 双确认全文） | +| `stage_local` | 消费已生成的阶段计划；含 stage_goal、milestone 和当帧重绑定后的 plan，不代表新的模型调用 | +| `stage_progress` | 里程碑推进，含 stage_goal、milestone、验证 source | +| `stage_replan` | 阶段修复请求，含原因及当前里程碑；不等于调用已经完成 | | `action`（kind 字段） | runner 写的执行记录 `{step, kind: "action", action: {…协议动作…}}` | +stage 模式的 planner 事件只在真实规划调用时写入，增加 mode、reason、seconds； +gui/cli 中的 plan 带 planner_called，表示本轮是否刚调用规划模型。 +gui_verified/cli_verified 的 verification 增加 source（rule_receipt / rule_a11y / model） +与 seconds。规则检查不写 llm_calls，不能计为一次慢模型 Verify。 +其他文件和动作/观察/回执协议不变；详细触发规则见 [STAGE_EXECUTION.md](STAGE_EXECUTION.md)。 + ## failed_unscored 纪律 API/VM/评分异常 ≠ 0 分——异常题不写 `result.txt`，`episode.json.status=failed_unscored`，run 继续下一题。`summary.json.failed_unscored` 单独计数，不计入均分。 diff --git a/docs/STAGE_EXECUTION.md b/docs/STAGE_EXECUTION.md new file mode 100644 index 0000000..4439def --- /dev/null +++ b/docs/STAGE_EXECUTION.md @@ -0,0 +1,196 @@ +# 阶段执行、规则验证与按需重规划 + +## 目的与边界 + +这是对 Tandem 决策策略的可选扩展，不替换环境、排序模型或动作协议。 +默认 `step` 模式保持原有逐步 Planner → 执行 → 模型 Verify 的行为。 +`stage` 模式一次规划一个有边界的阶段，阶段内按新观察持续执行，阶段结束再规划后续工作。 +不强制整任务只规划一次，也不保证固定数量动作内可以完成任意阶段。 + +```text +原始任务 → 需求抽取 → 慢模型规划当前阶段（1–8 个里程碑） + → 当前截图与 A11y → 当前里程碑目标重绑定 → 候选构造 → JEV/Kev 选择 → 真实执行 + → 回执与后续观察 → 规则检查优先，证据不足才用模型 Verify + → 已达成：推进下一里程碑；阶段完成后规划下一阶段 + → 仍可安全滚动：留在当前里程碑，继续观察和选择 + → 失败／无法绑定／证据不足未获确认：请求剩余工作的重规划 + → Planner 声称完成 → 原有输出文件探针与独立终态 Verify → Finish 或返回规划 +``` + +CLI 里程碑执行慢模型明确给出的命令，stdout/stderr 和真实退出码进入验证。 +退出码为零、命令打印“成功”、文件名或界面变化都不能单独证明任务完成。 +本版没有“任意自然语言目标自动转换成完整 GUI 程序”的能力；不确定的新界面必须返回规划。 + +## 配置和启动 + +唯一用户配置仍是 `infra/config.json`。增加可选段，省略时等同下列默认值： + +```json +"execution": { + "mode": "step", + "stage_max_actions": 24 +} +``` + +该段放在 `harness` 下。`stage_max_actions` 是单个阶段内实际执行动作的上限， +不是强制每隔 24 步调用模型；提前完成、不确定或失败可以更早返回规划。 +全任务步数、墙钟限制仍由现有 budget 和 runner 管理。 + +完成正常集群配置后，在仓库根执行： + +```bash +PYTHONPATH=src python -m tandem_eval rjob run \ + --config infra/config.json --repo . --run-id stage-smoke \ + --benchmark osworld1 --tasks bb5e4c0d-f964-439c-97b6-bdb9747de3f4 \ + --execution-mode stage --stage-max-actions 24 \ + --allow-cli true --route-policy auto --max-steps 50 --max-wall 1800 +``` + +模型仍用现有 `--slow-model NAME`、`--fast-model NAME` 选择。 +恢复单步模式只需使用 `--execution-mode step`。 +集群权限、服务发现、共享环境与 rjob 生命周期约定见 `CLUSTER_RUNTIME.md`。 +当前主线 rjob 客户端存在实际任务名可能追加后缀的已知监控问题；本 PR 不修改该客户端。 +若平台追加后缀，必须按 submit 返回的完整名称检查、归档及清理，不能误判 worker 未启动。 + +## 内部计划格式 + +阶段计划是组件内部数据，不是 Action/Observation/Receipt 协议的新字段。 +计划由 `stage_planner.valid_stage` 校验。示例中的控件名只是说明，不能当真实任务预制动作： +校验失败时向既有客户端的下一次尝试提供格式纠正信息，不增加重试次数、不截断计划或放宽类型。 + +```json +{ + "done": false, + "stage_goal": "打开设置并启用选项", + "milestones": [ + { + "route": "gui", + "current_goal": "打开设置对话框", + "expected_observation": "出现设置中的 Enabled 控件", + "action_proposals": [ + {"operation": "click", "target_name": "Settings", "target_role": "push_button"} + ], + "checks": [ + {"target": {"name": "Enabled", "role": "check_box"}, "property": "visible", "expected": true} + ] + }, + { + "route": "gui", + "current_goal": "启用选项", + "expected_observation": "Enabled 处于选中状态", + "action_proposals": [ + {"operation": "click", "target_name": "Enabled", "target_role": "check_box"} + ], + "checks": [ + {"target": {"name": "Enabled", "role": "check_box"}, "property": "checked", "expected": true} + ] + } + ] +} +``` + +- 每个里程碑给出一条具体状态转换；GUI 可以给 1–4 个动作提案供候选构造。 +- 不缓存 `target_id` 或坐标。按每帧的精确名称、role、可选 `parent_contains` 重绑。 +- 多个同名控件不能随意选第一个；无法唯一绑定会请求重规划。 +- CLI 里程碑使用 `route: cli`、`command`、目标和预期结果；通常 `checks: []`。 +- GUI 操作沿用现有 click、double_click、type_text、keypress、scroll、slider、wait 路径。 +- 条件支持 `visible: boolean`、精确 `value`、checked/selected/focused/expanded 状态。 + 缺失、歧义和未知状态返回 uncertain；不把“未找到”直接当作“不存在”的证明。 +- 所有后置条件满足且至少一个条件是动作后新出现的证据，才规则确认 achieved。 + 点击前已经可见的按钮本身不能证明点击成功。 +- 无规则、证据未知或所有条件动作前已满足时，交给原有独立模型 Verify。 +- 明确不满足的状态请求修复；仅在界面确有变化的滚动动作中允许局部继续，仍受阶段上限约束。 +- CLI 失败或回执未确认直接记录失败，不自动重放该命令。 + +## 代码地图与开发入口 + +| 文件 | 职责 | +|---|---| +| `src/tandem/stage_planner.py` | 阶段 prompt 和内部计划校验；慢模型请求 role 为 plan_stage | +| `src/tandem/stages.py` | 每个 session 的阶段进度、语义重绑定、推进和重规划决策 | +| `src/tandem/stage_rules.py` | A11y 可观察条件与回执检查，不调用模型、不执行环境动作 | +| `src/tandem/components.py` | 根据执行模式选用规划组件，排序后端不变 | +| `src/tandem/session.py` | 保留单步路径，接入阶段组件；动作仍逐个交给环境 | +| `src/tandem/episode.py` | 动作验证分流；终态独立验证沿用原实现 | +| `src/tandem/config.py` | execution_mode、stage_max_actions 的会话配置 | +| `src/tandem_eval/config_schema.py`、`rjob_spec.py`、`__main__.py` | 配置校验、spec 和 CLI 接线 | +| `src/tandem_eval/entry.py` | mock 模式也识别阶段计划，仅用于明确标识的开发测试 | +| `tests/harness/test_stages.py`、`tests/eval/test_stage_config.py` | 执行与配置回归 | +| `tests/eval/test_mock_chain.py` | 明确标识的 step/stage 开发链路与结果目录回归 | +| `infra/config.example.json` | 可提交的执行模式配置模板，不含真实凭据 | +| `README.md`、`DEVIATIONS.md` | 功能入口及相对旧实现的行为差异 | +| `docs/CLUSTER_RUNTIME.md`、`docs/RESULT_DIR_CONTRACT.md`、本文 | 启动参数、事件字段与开发说明 | + +增加规则条件时先修改 `stage_rules.py` 和计划校验，再补对应证据不足/矛盾单测。 +更换阶段规划方法走 `components.py`，不要把特定厂商、任务 ID 或 benchmark 答案写进状态机。 +本 PR 不增加依赖，不修改官方任务、评分器、环境安装或模型服务部署。 +阶段 prompt 明确保留用户原始范围，不能自行增加匹配模式、选择过滤或例外来缩小要求。 + +## 记录与验收 + +沿用 `rjob_logs//episodes//`，不另设结果系统： + +- `events.jsonl`：planner 只在真实调用时记录；stage_local 表示消费阶段计划，不是新模型调用。 + stage_progress 记录里程碑证据；stage_replan 记录修复原因。 +- gui/cli 与 gui_verified/cli_verified 保留现有事件，验证增加 source 与 seconds。 + source 为 rule_receipt、rule_a11y 或 model。planner 事件增加 seconds。 +- `llm_calls.jsonl`：沿用原始请求/响应和 usage；新增 plan_stage role。 + 统计模型次数以该文件为准，不把阶段执行器每次运行计为 Planner 调用。 +- `timing.json`、`episode.json`、`evaluation.json`：继续使用原有 recorder 和官方评分。 + 本 PR 不改变 API 异常耗时的统计口径，不声称已经实现完整的异常开销剔除。 + +开发测试必须验证：阶段内至少两个动作不重复规划、每帧目标重新绑定、失败和未知证据不误报、 +单步路径保持通过、最终完成仍独立检查。真实 smoke 必须明确使用真实模型/JEV/VM， +单题通过不等于固定 50 题效果无退化；大规模同题对比仍需另跑。 + +## 2026-10-24 实际检查与真实试跑 + +本次检查：`ruff check src tests demo` 通过，`PYTHONPATH=src pytest -q` 为 274 passed， +`PYTHONPATH=src python -m tandem check` 为 16 个样例通过；`git diff --check` 通过。 +没有修改 Action/Observation/Receipt schema；没有训练或替换模型，没有添加运行时依赖。 + +真实试跑使用 MiniMax-M3.1-Flash（命名 API 为 minimax-shared）、reasoning_effort=low、 +max_tokens=32768，以及已有训练好的 Qwen-JEV。不是 Kimi/max 强度的历史成绩复现。 +每个 VM worker 为 evobox_proxy/private-machine=group、4 CPU、16384 MiB、0 GPU， +JEV 复用已有 GPU 服务。每次试跑是独立 VM，保留官方任务和原生评分。 +以下全部是模型实时执行，不是 mock 或固定动作回放。 + +结果根目录： + +```text +/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/tandem_stage_execution_pr_20261024/rjob_logs +``` + +| run_id | 题目与模式 | 原生分数 | 结论 | +|---|---|---:|---| +| stage-real-gui-20261024 | Chrome 默认搜索引擎，GUI-only 阶段模式 | 1.0 | 5 个真实 GUI 动作；轨迹有同一计划连续两次执行；一次 JEV 超时后自行恢复 | +| stage-real-cli-gui-20261024 | VS Code 替换，初版 CLI+GUI 阶段模式 | 0.0 | Planner 擅自收窄为完整单词匹配，Verify 也接受了收窄解释；不是成功案例 | +| stage-real-scope-v2-20261024 | 同题，第一次约束修订 | 0.0 | 仍错误收窄范围；还暴露过长计划和条件字段格式问题，记录保留 | +| step-real-scope-v2-20261024 | 同题，原有单步模式 | 1.0 | 6 个真实 GUI 动作；允许 CLI，但模型本次选择 GUI | +| stage-real-scope-v3-20261024 | 同题，原文优先、明确字段、紧凑控件与格式纠正后的阶段模式 | 1.0 | 1 CLI + 1 JEV GUI，2 次有效 Planner 调用（含最终完成声明） | +| stage-real-final-20261024 | 同题，最终代码快照 | 1.0 | 8 JEV GUI + 1 CLI；6 次规则检查、3 次模型动作 Verify；4 次 Planner（含完成声明），另有独立终态 Verify | + +初次失败后只补通用约束：原文优先于抽取笔记、不得引入未请求的匹配范围、阶段不要覆盖整条 +未知任务、条件 property 使用明确枚举、格式失败在既有尝试内纠正。没有写入题目 ID、答案、 +目标文件内容或按评分结果修改任务。动作提案是同一前态下的备选，不是有序动作列表。 +修复请求携带本阶段已验证的里程碑，避免重做已确认工作。 +最终记录中 9 次动作验证有 6 次来自 rule_a11y，3 次来自 model，证明真实任务已走规则分流， +但这一个任务的覆盖率不能外推为整个 benchmark 的覆盖率。 + +每次部署保留独立源码目录（deploy、deploy-cli、deploy-scope-v2、deploy-scope-v3、 +deploy-final），不在运行中覆盖 worker 源码。对应私有 spec 在各目录的 job_specs 下，权限 600； +不进入 Git 或 PR。最终任务在 worker 内的实际入口为： + +```bash +bash /mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/tandem_stage_execution_pr_20261024/deploy-final/infra/worker.sh \ + /mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/tandem_stage_execution_pr_20261024/deploy-final/job_specs/stage-real-final-20261024.json +``` + +该命令只应在已申请的 worker 中执行，不能在开发机直接启动 VM。 +真实任务轨迹、截图、原始 LLM usage、timing 和 evaluation 在上述结果根的各 run 内。 +完成任务的平台记录先归档到同级 `rjob_archive/<完整任务名>/`，再删除 rjob；不删除轨迹。 +worker 创建的 run 目录由容器 root 所有，开发机归档使用独立的用户可写目录，未放宽共享目录权限。 + +尚未完成的验收：固定 50 题同题性能回归、OSWorld2/2.1 阶段模式回归，以及真实任务中规则 +验证覆盖率的充分验证。这些小样本不能证明准确率无退化或平均时间/token 的因果改善。 +原有单步模式仍为默认，新模式尚不应直接作为默认生产配置。 diff --git a/infra/config.example.json b/infra/config.example.json index 085da89..bc1edbe 100644 --- a/infra/config.example.json +++ b/infra/config.example.json @@ -49,6 +49,10 @@ "allow_cli": true, "cli_timeout_seconds": 90 }, + "execution": { + "mode": "step", + "stage_max_actions": 24 + }, "budget": { "max_steps": 30, "max_wall_seconds": 1800 diff --git a/src/tandem/components.py b/src/tandem/components.py index 3cc0ae8..92b4c0e 100644 --- a/src/tandem/components.py +++ b/src/tandem/components.py @@ -14,6 +14,7 @@ from .planner import plan_step from .proposals import focused_candidates from .selector import JevOmniPolicy +from .stage_planner import plan_stage from .targets import A11yTargetBuilder from .verifier import task_requirements, verify_observation @@ -42,7 +43,7 @@ def build_default(*, client, endpoint: str, config: SessionConfig, image_stage=image_stage) return Components( planner=partial( - plan_step, + plan_stage if config.execution_mode == "stage" else plan_step, client, allow_cli=config.allow_cli and config.route_policy != "gui", route_policy=config.route_policy, diff --git a/src/tandem/config.py b/src/tandem/config.py index 6aeb453..7a43bb6 100644 --- a/src/tandem/config.py +++ b/src/tandem/config.py @@ -15,6 +15,8 @@ class SessionConfig: allow_cli: bool = True route_policy: str = "auto" cli_timeout_seconds: int = 90 + execution_mode: str = "step" + stage_max_actions: int = 24 slow_max_tokens: int = 16384 slow_timeout_seconds: float = 120 slow_direct_api: bool = False @@ -23,6 +25,10 @@ class SessionConfig: fast_timeout_seconds: float = 300 def __post_init__(self): + if self.execution_mode not in {"step", "stage"}: + raise ValueError("execution_mode must be step or stage") + if type(self.stage_max_actions) is not int or self.stage_max_actions < 1: + raise ValueError("stage_max_actions must be a positive integer") if self.route_policy not in {"auto", "gui", "cli"}: raise ValueError("route_policy must be auto, gui or cli") if not 1 <= self.cli_timeout_seconds <= 120: diff --git a/src/tandem/episode.py b/src/tandem/episode.py index dc87a6f..b0643e0 100644 --- a/src/tandem/episode.py +++ b/src/tandem/episode.py @@ -117,12 +117,10 @@ def verify_pending(session, observation: Observation, tree: str | None) -> None: receipt = observation.last_receipt if receipt is None: raise RuntimeError("环境未返回上一步执行回执（协议违例）") - screenshot_after = Path(observation.screenshot.path) screenshot_before = Path(pending["before_screenshot"]) if pending["kind"] == "cli": - verification = session.components.verifier( - session.instruction, - screenshot_after, + verification = _verify_action( + session, observation, tree, pending, previous=screenshot_before, expected=pending["plan"]["expected_observation"], action=pending["action"], @@ -166,9 +164,8 @@ def verify_pending(session, observation: Observation, tree: str | None) -> None: and tree is not None else None ) - verification = session.components.verifier( - session.instruction, - screenshot_after, + verification = _verify_action( + session, observation, tree, pending, previous=screenshot_before, expected=pending["plan"]["expected_observation"], action=winner.compact(), @@ -217,3 +214,11 @@ def verify_pending(session, observation: Observation, tree: str | None) -> None: "control_state_check": control_check, } ) + + +def _verify_action(session, observation, tree, pending, **kwargs): + if session.stage is not None: + return session.stage.verify(session, observation, tree, pending, **kwargs) + return session.components.verifier( + session.instruction, Path(observation.screenshot.path), **kwargs, + ) diff --git a/src/tandem/session.py b/src/tandem/session.py index 78d40f2..15dc41c 100644 --- a/src/tandem/session.py +++ b/src/tandem/session.py @@ -17,6 +17,7 @@ from .focus import prune_stalled from .protocol.models import Action, CliAction, FailAction, GuiAction, Observation, WaitAction from .selector import ranking_dict +from .stages import StageExecution from .stopping import NO_ACTION_LIMIT, budget_exhausted @@ -40,6 +41,9 @@ def begin(self, instruction: str = "", requirements: list | None = None) -> None self.failed_toggles: dict[str, tuple[str, object]] = {} self.pending: dict | None = None self.done_probe: dict | None = None + self.stage = ( + StageExecution(self.config.stage_max_actions) if self.config.execution_mode == "stage" else None + ) # ------------------------------------------------------------------ 主入口 def act(self, observation: Observation) -> Action: @@ -64,17 +68,20 @@ def act(self, observation: Observation) -> Action: # 同一观察内最多重规划 NO_ACTION_LIMIT+3 次：完成被拒/无候选会递增 no_action 并重规划， # 与旧循环 continue 语义一致（旧版重取 obs，本帧无动作时内容等价）。 for _ in range(NO_ACTION_LIMIT + 3): - plan = self.components.planner( - self.instruction, - screenshot, - targets, - self.recent, - tree if tree is not None else "", - self.verified_focus, - self.requirements, - observation.budget.steps_remaining, - ) - self.events.append({"step": observation.step_idx, "route": "planner", "plan": plan}) + if self.stage is not None: + plan = self.stage.plan(self, observation, targets, tree, screenshot) + if plan is None: + self.consecutive_no_action += 1 + if self.consecutive_no_action >= NO_ACTION_LIMIT: + return FailAction(reason="no_action_limit") + continue + else: + plan = self.components.planner( + self.instruction, screenshot, targets, self.recent, + tree if tree is not None else "", self.verified_focus, + self.requirements, observation.budget.steps_remaining, + ) + self.events.append({"step": observation.step_idx, "route": "planner", "plan": plan}) if plan["done"]: terminal = handle_done(self, plan, observation, tree) if terminal is not None: @@ -99,6 +106,7 @@ def _emit_cli(self, plan: dict, observation: Observation) -> CliAction: "action": action.to_dict(), "plan": plan, "before_screenshot": observation.screenshot.path, + "before_tree": self._read_tree(observation), } self.consecutive_no_action = 0 self.events.append( @@ -146,7 +154,7 @@ def _plan_gui( for service_retry in range(3): try: ranking = self.components.selector.rank( - stage_goal=self.instruction, + stage_goal=plan.get("stage_goal", self.instruction), intent=local_goal, active_window="", targets=targets, @@ -175,6 +183,8 @@ def _plan_gui( } ) if compiled is None: + if self.stage is not None: + self.stage.invalidate(self, "no_executable_candidate", observation.step_idx) self.consecutive_no_action += 1 self.recent.append( { diff --git a/src/tandem/stage_planner.py b/src/tandem/stage_planner.py new file mode 100644 index 0000000..b04d6aa --- /dev/null +++ b/src/tandem/stage_planner.py @@ -0,0 +1,168 @@ +"""One bounded stage per slow-model call; plans are internal, not wire-protocol actions.""" + +from __future__ import annotations + +import base64 +import json + +from .focus import input_and_menu_context + +OPERATIONS = {"click", "double_click", "type_text", "keypress", "scroll_up", "scroll_down", + "set_slider", "wait"} + + +def valid_stage(value, *, allow_cli=True, route_policy="auto"): + if not isinstance(value, dict) or type(value.get("done")) is not bool: + return False + if value["done"]: + return (value.get("task_status") in {"completed", "infeasible"} + and isinstance(value.get("current_goal"), str) and bool(value["current_goal"].strip())) + milestones = value.get("milestones") + if (not isinstance(value.get("stage_goal"), str) or not value["stage_goal"].strip() + or not isinstance(milestones, list) or not 1 <= len(milestones) <= 8): + return False + for item in milestones: + if not isinstance(item, dict) or not all( + isinstance(item.get(key), str) and item[key].strip() + for key in ("current_goal", "expected_observation") + ): + return False + route = item.get("route") + if route == "cli": + if (not allow_cli or route_policy == "gui" or not isinstance(item.get("command"), str) + or not item["command"].strip()): + return False + elif route == "gui" and route_policy != "cli": + proposals = item.get("action_proposals") + if not isinstance(proposals, list) or not 1 <= len(proposals) <= 4: + return False + for proposal in proposals: + if not isinstance(proposal, dict) or proposal.get("operation") not in OPERATIONS: + return False + if "parent_contains" in proposal and not isinstance(proposal["parent_contains"], str): + return False + if "target_name" in proposal and not all( + isinstance(proposal.get(key), str) for key in ("target_name", "target_role") + ): + return False + operation = proposal["operation"] + if operation in {"click", "double_click", "scroll_up", "scroll_down", "set_slider"}: + if not all(isinstance(proposal.get(key), str) and proposal[key] + for key in ("target_name", "target_role")): + return False + if operation == "type_text" and ( + not isinstance(proposal.get("text"), str) + or proposal.get("input_mode") not in {"insert", "replace"} + ): + return False + if operation == "keypress": + sequence = proposal.get("key_sequence") + if (not isinstance(sequence, list) or not 1 <= len(sequence) <= 8 + or not all(isinstance(keys, list) and 1 <= len(keys) <= 4 + and all(isinstance(key, str) for key in keys) for keys in sequence)): + return False + if operation == "set_slider" and ( + type(proposal.get("value")) not in (int, float) or not 0 <= proposal["value"] <= 1 + ): + return False + else: + return False + checks = item.get("checks", []) + if not isinstance(checks, list) or len(checks) > 4: + return False + for check in checks: + if not isinstance(check, dict) or not isinstance(check.get("target"), dict): + return False + if not all(isinstance(check["target"].get(key), str) for key in ("name", "role")): + return False + if ("parent_contains" in check["target"] + and not isinstance(check["target"]["parent_contains"], str)): + return False + prop = check.get("property") + if prop not in {"visible", "value", "checked", "selected", "focused", "expanded"}: + return False + if prop == "value": + if not isinstance(check.get("expected"), str): + return False + elif type(check.get("expected")) is not bool: + return False + return True + + +def plan_stage(client, instruction, screenshot, targets, recent, accessibility_tree, + verified_focus=None, requirements=None, remaining_steps=None, *, + allow_cli=True, route_policy="auto"): + inputs, menus = input_and_menu_context(accessibility_tree) + context = { + "task": instruction, "requirements": requirements or [], "remaining_actions": remaining_steps, + "visible_controls": [ + {"name": target.name, "role": target.role, "value": target.value, + "parents": target.parent_context, "focused": target.focused, + "checked": target.checked, "selected": target.selected} for target in targets + ], + "focused_inputs": [verified_focus] if verified_focus else inputs, + "menu_paths": menus, "recent_actions_and_evidence": recent[-6:], + "allow_cli": allow_cli, "route_policy": route_policy, + } + prompt = """Plan the next bounded desktop stage, not just one action and not the entire unseen task. +Return only JSON. For a running stage use: +{"done":false,"stage_goal":"...","milestones":[{"route":"gui", +"current_goal":"one concrete transition","expected_observation":"specific resulting state", +"action_proposals":[{"operation":"click","target_name":"exact name","target_role":"push_button"}], +"checks":[{"target":{"name":"result control","role":"entry"},"property":"focused","expected":true}]}]}. +Prefer 2-4 milestones for the CURRENT stage, with a hard maximum of eight. If the whole task needs +more, return only its first bounded stage; subsequent calls will plan the rest from fresh evidence. +Do not append the rest of the task or extra audits beyond that limit. Stop at an uncertain boundary instead of +inventing unseen UI. Each milestone is one action, except a bounded scroll-until-visible transition. +action_proposals are alternatives executable from the SAME before-state, never an ordered sequence. +Opening an input, typing into it, and submitting are separate milestones, not three proposals in one. +Future targets need exact semantic names/roles, optional parent_contains for duplicate names, never +cached target_id or coordinates. Targets are rebound from each new observation. GUI operations: +click, double_click, scroll_up, scroll_down, type_text, keypress, set_slider, wait. +type_text requires exact text and input_mode insert/replace. Omit target fields only for a confirmed +keyboard-focused input. keypress requires key_sequence, e.g. [["ctrl","l"]]. set_slider needs value 0..1. +checks are optional; use [] when independent model verification is needed. The property field must +be exactly one of ["visible","value","checked","selected","focused","expanded"], never "exact value". +Use property="value" with expected:string for exact text, otherwise expected:boolean, on an exact target +{name,role,parent_contains?}. Check the RESULT, not the pre-existing button being clicked. +Unknown or duplicate controls cannot prove success. Do not invent conditions to bypass verification. +For CLI use route=cli, command (bounded Bash script in /home/user), current_goal, expected_observation, +checks:[]. Actual stdout/stderr will be verified; exit=0 or printed success alone proves nothing. +When CLI is allowed under auto routing, prefer it for deterministic public-file/data edits or audits; +use GUI for application state and inspect the real application after CLI document edits. +Honor allow_cli/route_policy. Never inspect evaluators, reference answers, rewards, unrelated secrets, +or modify scoring. Do not install packages or broadly delete files unless the task requires it. +After CLI document edits, reopen/reload and inspect the real application before completion. +On a repair request, use the fresh observation and recorded failure, preserve verified work, and +replace only remaining work. Do not repeat a failed method unchanged. Missing candidates do not +make a task infeasible. An unexpected UI requires a revised grounded plan, not invented controls. +Preserve the user's exact scope and literal values. Do not introduce unrequested matching modes, +word boundaries, selection filters, case restrictions, or exceptions. A narrower self-chosen goal +cannot satisfy a broader original requirement. Check commands against the original instruction, +not just their own success messages; all occurrences means all literal occurrences unless specified. +The original task is authoritative; extracted requirements are derived notes, not permission to +reinterpret the task or add restrictions. Resolve any mismatch in favor of the original wording. +Use {"done":true,"task_status":"completed" or "infeasible","current_goal":"observed reason"} +only when the original task is fully satisfied (including saving) or demonstrably infeasible. +Completion is independently verified; a stage finishing is NOT proof of task completion.""" + messages = [{"role": "system", "content": prompt}, {"role": "user", "content": [ + {"type": "text", "text": json.dumps(context, ensure_ascii=False)}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + + base64.b64encode(screenshot.read_bytes()).decode("ascii")}}, + ]}] + def validate(value): + valid = valid_stage(value, allow_cli=allow_cli, route_policy=route_policy) + if not valid: + # Use the client's existing attempts; do not truncate, coerce or execute invalid plans. + messages.extend([ + {"role": "assistant", "content": json.dumps(value, ensure_ascii=False)}, + {"role": "user", "content": ( + "Invalid stage schema. Return only the current stage with 1-8 milestones, respecting " + "allow_cli/route_policy. Each check property must be exactly visible, value, checked, " + "selected, focused or expanded (not 'exact value'). Preserve the ORIGINAL task scope. " + "Correct the JSON using the field names and types specified above." + )}, + ]) + return valid + + return client.json_request(messages, validate, role="plan_stage") diff --git a/src/tandem/stage_rules.py b/src/tandem/stage_rules.py new file mode 100644 index 0000000..6249129 --- /dev/null +++ b/src/tandem/stage_rules.py @@ -0,0 +1,75 @@ +"""Observable stage postconditions. Unknown or ambiguous evidence never proves success.""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET + + +def controls(tree: str | None) -> list[dict]: + """Read active, visible controls, including non-actionable labels and paragraphs.""" + if tree is None: + return [] + result = [] + + def visit(node, parents=(), active=False, hidden=False): + attrs = {key.rsplit("}", 1)[-1]: value for key, value in node.attrib.items()} + active = active or attrs.get("active") == "true" + hidden = hidden or attrs.get("showing") == "false" or attrs.get("visible") == "false" + name = attrs.get("name", "") + if active and not hidden: + result.append({ + "name": name, "role": node.tag.rsplit("}", 1)[-1].replace("-", "_"), + "parents": parents, "value": attrs.get("value", node.text or ""), + **{key: attrs.get(key) == "true" if key in attrs else None + for key in ("focused", "selected", "checked", "expanded")}, + }) + for child in node: + visit(child, parents + ((name,) if name else ()), active, hidden) + + visit(ET.fromstring(tree)) + return result + + +def matches(items, query): + return [item for item in items + if item["role"] == query["role"] and item["name"] == query["name"] + and (not query.get("parent_contains") + or any(query["parent_contains"] in parent for parent in item["parents"]))] + + +def check_state(items, check) -> str: + found = matches(items, check["target"]) + if len(found) != 1: + return "uncertain" + observed = True if check["property"] == "visible" else found[0].get(check["property"]) + if observed is None: + return "uncertain" + return "met" if observed == check["expected"] else "not_met" + + +def rule_verification(pending, receipt, tree): + """Return None for semantic model verification; never use exit=0 as success proof.""" + base = {"task_status": "incomplete", "path_input_focused": False, "state_changed": False} + if not receipt.acknowledged or receipt.returncode != 0: + return {**base, "outcome": "not_achieved", "evidence": "Execution receipt failed.", + "source": "rule_receipt"} + checks = pending["plan"].get("checks", []) + if not checks: + return None + before, after = controls(pending.get("before_tree")), controls(tree) + states = [check_state(after, check) for check in checks] + if "uncertain" in states: + return None + changed = before != after + if "not_met" in states: + # Only scrolling may repeat locally; never blindly repeat clicks, typing or commands. + repeat = pending["kind"] == "gui" and pending["winner"].operation == "scroll" and changed + return {**base, "outcome": "uncertain" if repeat else "not_achieved", + "state_changed": changed, "continue_local": repeat, + "evidence": f"Observable postconditions: {states}.", "source": "rule_a11y"} + # A condition already true before the action is not evidence of its effect. + if not any(check_state(before, check) != "met" for check in checks): + return None + return {**base, "outcome": "achieved", "state_changed": changed, + "evidence": "All postconditions met, including a newly observed condition.", + "source": "rule_a11y"} diff --git a/src/tandem/stages.py b/src/tandem/stages.py new file mode 100644 index 0000000..85973b0 --- /dev/null +++ b/src/tandem/stages.py @@ -0,0 +1,93 @@ +"""Session-local stage progress; every action still uses current observations and receipts.""" + +from __future__ import annotations + +import time +from pathlib import Path + +from .stage_rules import rule_verification + + +class StageExecution: + def __init__(self, max_actions): + self.max_actions = max_actions + self.milestones = [] + self.index = 0 + self.actions = 0 + self.goal = "" + self.reason = "initial_stage" + + def invalidate(self, session, reason, step): + session.events.append({"route": "stage_replan", "step": step, "reason": reason, + "stage_goal": self.goal, "milestone": self.index}) + session.recent.append({"operation": "stage_replan", "outcome": "no_action", + "goal": self.goal, "reason": reason, + "verified_milestones": [item["current_goal"] + for item in self.milestones[:self.index]]}) + self.milestones = [] + self.reason = reason + + def plan(self, session, observation, targets, tree, screenshot): + called = False + if self.milestones and self.actions >= self.max_actions: + self.invalidate(session, "stage_action_limit", observation.step_idx) + if not self.milestones: + started = time.perf_counter() + plan = session.components.planner( + session.instruction, screenshot, targets, session.recent, tree or "", + session.verified_focus, session.requirements, observation.budget.steps_remaining, + ) + session.events.append({"route": "planner", "step": observation.step_idx, + "plan": plan, "reason": self.reason, "mode": "stage", + "seconds": time.perf_counter() - started}) + called = True + if plan["done"]: + return plan + self.goal = plan["stage_goal"] + self.milestones, self.index, self.actions = plan["milestones"], 0, 0 + milestone = self.milestones[self.index] + plan = {**milestone, "done": False, "task_status": "running", + "stage_goal": self.goal, "milestone": self.index, "planner_called": called} + if plan["route"] == "gui": + bound = [] + for proposal in plan["action_proposals"]: + proposal = {key: value for key, value in proposal.items() if key != "target_id"} + if proposal.get("target_name") is not None and proposal["operation"] != "wait": + found = [target for target in targets + if target.name == proposal["target_name"] + and target.role == proposal.get("target_role") + and (not proposal.get("parent_contains") + or any(proposal["parent_contains"] in p for p in target.parent_context))] + if len(found) != 1: + continue + proposal["target_id"] = found[0].id + bound.append(proposal) + if not bound: + self.invalidate(session, "target_unbound_or_ambiguous", observation.step_idx) + return None + plan["action_proposals"] = bound + session.events.append({"route": "stage_local", "step": observation.step_idx, + "stage_goal": self.goal, "milestone": self.index, "plan": plan}) + return plan + + def verify(self, session, observation, tree, pending, **kwargs): + started = time.perf_counter() + result = rule_verification(pending, observation.last_receipt, tree) + if result is None: + result = session.components.verifier( + session.instruction, Path(observation.screenshot.path), **kwargs, + ) + result = {**result, "source": "model"} + result = {**result, "seconds": time.perf_counter() - started} + self.actions += 1 + if result["outcome"] == "achieved": + session.events.append({"route": "stage_progress", "step": observation.step_idx, + "stage_goal": self.goal, "milestone": self.index, + "source": result["source"]}) + self.index += 1 + if self.index == len(self.milestones): + self.milestones = [] + self.reason = "stage_completed" + elif not result.get("continue_local"): + self.invalidate(session, "execution_failed_or_unverified", observation.step_idx) + return result diff --git a/src/tandem_eval/__main__.py b/src/tandem_eval/__main__.py index d9e2e43..e3ffc95 100644 --- a/src/tandem_eval/__main__.py +++ b/src/tandem_eval/__main__.py @@ -36,6 +36,9 @@ def main(argv: list[str] | None = None) -> int: run_cmd.add_argument("--allow-cli", choices=["true", "false"], default=None, help="harness 是否允许 CLI 路由") run_cmd.add_argument("--route-policy", choices=["auto", "gui", "cli"], default=None) + run_cmd.add_argument("--execution-mode", choices=["step", "stage"], default=None) + run_cmd.add_argument("--stage-max-actions", type=int, default=None, + help="阶段内动作数上限，达到后请求剩余工作的重规划") run_cmd.add_argument("--slow-model", default=None, metavar="NAME", help="慢模型（大脑）API 名：config 的 harness.slow_model.apis 里选；默认取 default") run_cmd.add_argument("--fast-model", default=None, metavar="NAME", @@ -91,6 +94,10 @@ def main(argv: list[str] | None = None) -> int: if models: overrides["models"] = models harness_config: dict = {} + if args.execution_mode is not None: + harness_config["execution_mode"] = args.execution_mode + if args.stage_max_actions is not None: + harness_config["stage_max_actions"] = args.stage_max_actions if args.allow_cli is not None: harness_config["allow_cli"] = args.allow_cli == "true" if args.route_policy: diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py index 852d340..acd4320 100644 --- a/src/tandem_eval/config_schema.py +++ b/src/tandem_eval/config_schema.py @@ -206,6 +206,15 @@ def validate(config: dict) -> list[str]: problems += _validate_fast_models(config) problems += _validate_absolute_paths(config) problems += _validate_osworld21(config) + execution = get_path(config, "harness.execution") or {} + if not isinstance(execution, dict): + problems.append("harness.execution（应为对象）") + else: + if execution.get("mode", "step") not in {"step", "stage"}: + problems.append("harness.execution.mode（应为 step / stage）") + limit = execution.get("stage_max_actions", 24) + if type(limit) is not int or limit < 1: + problems.append("harness.execution.stage_max_actions（应为正整数）") return problems diff --git a/src/tandem_eval/entry.py b/src/tandem_eval/entry.py index 7577c0b..8d5d2d5 100644 --- a/src/tandem_eval/entry.py +++ b/src/tandem_eval/entry.py @@ -97,15 +97,16 @@ def git_describe(repo_root: Path) -> str: class _ScriptedPlanner: """mock 模式的确定性 planner：先点第一个目标，再声称完成。""" - def __init__(self): + def __init__(self, *, stage_mode=False): self.calls = 0 + self.stage_mode = stage_mode def __call__(self, instruction, screenshot, targets, recent, tree, verified_focus, requirements, remaining_steps): self.calls += 1 if self.calls == 1 and targets: target = targets[0] - return { + plan = { "done": False, "task_status": "running", "current_goal": "操作第一个控件", "expected_observation": "界面变化", "action_proposals": [{ @@ -113,6 +114,10 @@ def __call__(self, instruction, screenshot, targets, recent, tree, "target_name": target.name, "target_role": target.role, }], } + if self.stage_mode: + plan.update(route="gui", checks=[]) + return {"done": False, "stage_goal": "mock stage", "milestones": [plan]} + return plan return {"done": True, "task_status": "completed", "current_goal": "完成", "expected_observation": ""} @@ -140,14 +145,14 @@ def rank(self, **kwargs): ) -def mock_components() -> Components: +def mock_components(*, stage_mode=False) -> Components: from tandem.candidates import ActionCandidateBuilder from tandem.compile import gui_action from tandem.proposals import focused_candidates from tandem.targets import A11yTargetBuilder return Components( - planner=_ScriptedPlanner(), verifier=_ScriptedVerifier(), + planner=_ScriptedPlanner(stage_mode=stage_mode), verifier=_ScriptedVerifier(), requirements=lambda instruction: [], selector=_FirstCandidateSelector(), target_builder=A11yTargetBuilder(), candidate_builder=ActionCandidateBuilder(), focus_candidates=focused_candidates, grounder=None, compile_gui=gui_action, @@ -158,7 +163,7 @@ def build_session(spec: dict, *, llm_tap: LlmTap | None = None) -> TandemSession config = SessionConfig(**spec.get("harness_config", {})) benchmark = spec.get("benchmark") if benchmark == "mock": - return TandemSession(mock_components(), config=config) + return TandemSession(mock_components(stage_mode=config.execution_mode == "stage"), config=config) if benchmark in {"osworld1", "osworld2", "osworld21"}: from tandem.components import build_default from tandem.providers import SlowModelClient diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py index 59b29a0..2c463de 100644 --- a/src/tandem_eval/rjob_spec.py +++ b/src/tandem_eval/rjob_spec.py @@ -77,7 +77,10 @@ def _secrets_section(benchmark: str, config: dict, slow_api: dict, fast_api: dic def _harness_config_section(config: dict, slow_api: dict, fast_api: dict) -> dict: """行为旋钮段：SessionConfig 的字段（与 harness 侧 dataclass 一一对应）。""" route = config["harness"]["route"] + execution = config["harness"].get("execution", {}) return { + "execution_mode": execution.get("mode", "step"), + "stage_max_actions": execution.get("stage_max_actions", 24), "route_policy": route.get("policy", "auto"), "allow_cli": bool(route.get("allow_cli", True)), "cli_timeout_seconds": int(route.get("cli_timeout_seconds", 90)), @@ -151,7 +154,7 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path, "image_stage": _image_stage_section(config), "harness": { "name": "tandem", - "mode": "planner_led", + "mode": "stage" if harness_config["execution_mode"] == "stage" else "planner_led", "models": {"slow": slow_name, "fast": fast_name}, }, "harness_config": harness_config, diff --git a/tests/eval/test_mock_chain.py b/tests/eval/test_mock_chain.py index 529c119..a158723 100644 --- a/tests/eval/test_mock_chain.py +++ b/tests/eval/test_mock_chain.py @@ -7,6 +7,8 @@ import sys from pathlib import Path +import pytest + from tandem_eval.entry import entry_main, run_from_spec from tandem_eval.env import MockEnv from tandem_eval.manifest import TaskRef @@ -39,8 +41,9 @@ def _spec(tmp_path: Path, **overrides) -> dict: return base -def test_full_mock_chain_produces_renderable_result_dir(tmp_path): - episodes = run_from_spec(_spec(tmp_path)) +@pytest.mark.parametrize("mode", ["step", "stage"]) +def test_full_mock_chain_produces_renderable_result_dir(tmp_path, mode): + episodes = run_from_spec(_spec(tmp_path, harness_config={"execution_mode": mode})) assert episodes and episodes[0]["status"] == "scored" run_root = tmp_path / "run" episode_dir = run_root / "episodes" / "mock-mock-000" diff --git a/tests/eval/test_stage_config.py b/tests/eval/test_stage_config.py new file mode 100644 index 0000000..40cbd54 --- /dev/null +++ b/tests/eval/test_stage_config.py @@ -0,0 +1,60 @@ +"""Stage-mode knobs reach the same worker session through config, spec and CLI.""" + +from __future__ import annotations + +import pytest + +from tandem.config import SessionConfig +from tandem_eval.__main__ import main +from tandem_eval.config_schema import validate +from tandem_eval.entry import build_session +from tandem_eval.rjob_spec import build_job_spec + +from .rjob_fakes import CONFIG, with_paths + + +def test_stage_config_reaches_worker(tmp_path): + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy"), results_root=str(tmp_path)) + config["harness"]["execution"] = {"mode": "stage", "stage_max_actions": 12} + assert validate(config) == [] + spec = build_job_spec(config=config, run_id="stage", repo_root=tmp_path, + spec_overrides={"benchmark": "mock"}) + assert spec["harness"]["mode"] == "stage" + session = build_session(spec) + assert session.config.execution_mode == "stage" and session.stage.max_actions == 12 + + +def test_old_config_keeps_single_step_default(tmp_path): + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy"), results_root=str(tmp_path)) + spec = build_job_spec(config=config, run_id="step", repo_root=tmp_path, spec_overrides={}) + assert build_session(spec).stage is None + + +@pytest.mark.parametrize("execution", [{"mode": "invalid"}, {"stage_max_actions": False}, + {"stage_max_actions": 0}, {"stage_max_actions": 1.5}]) +def test_invalid_execution_config_is_reported(execution): + config = with_paths(CONFIG) + config["harness"]["execution"] = execution + assert any("harness.execution" in problem for problem in validate(config)) + + +def test_session_rejects_invalid_execution_knobs(): + with pytest.raises(ValueError, match="execution_mode"): + SessionConfig(execution_mode="unknown") + with pytest.raises(ValueError, match="stage_max_actions"): + SessionConfig(stage_max_actions=False) + + +def test_cli_execution_flags_reach_overrides(monkeypatch): + from pathlib import Path + + captured = {} + monkeypatch.setattr("tandem_eval.rjob.run_job", lambda **kwargs: ( + captured.update(kwargs) or {"status": "succeeded"} + )) + config_path = Path(__file__).resolve().parents[2] / "infra/config.example.json" + assert main(["rjob", "run", "--run-id", "stage", "--config", str(config_path), + "--execution-mode", "stage", "--stage-max-actions", "12"]) == 0 + assert captured["spec_overrides"]["harness_config"] == { + "execution_mode": "stage", "stage_max_actions": 12, + } diff --git a/tests/harness/test_stages.py b/tests/harness/test_stages.py new file mode 100644 index 0000000..8ca1c8c --- /dev/null +++ b/tests/harness/test_stages.py @@ -0,0 +1,265 @@ +"""Development fixtures, not benchmark evidence: real session/candidate/compiler paths.""" + +from __future__ import annotations + +from copy import deepcopy + +import pytest + +from tandem.config import SessionConfig +from tandem.protocol.models import CliAction, FailAction, FinishAction, GuiAction +from tandem.session import TandemSession +from tandem.stage_planner import valid_stage + +from .conftest import ( + FixedSelector, + ScriptedPlanner, + ScriptedVerifier, + cli_receipt, + gui_receipt, + make_components, + make_observation, + valid_verifier_response, +) + + +def tree(*buttons, checked=False): + return ('' + + ''.join(f'' + for name, x in buttons) + + f'') + + +def milestone(name, result): + return {"route": "gui", "current_goal": f"Open {name}", "expected_observation": result, + "action_proposals": [{"operation": "click", "target_name": name, + "target_role": "push_button"}], + "checks": [{"target": {"name": result, "role": "push_button"}, + "property": "visible", "expected": True}]} + + +def stage(*milestones): + return {"done": False, "stage_goal": "Configure and save", "milestones": list(milestones)} + + +DONE = {"done": True, "task_status": "completed", "current_goal": "Original task complete"} + + +def session_for(plans, responses=(), **config): + planner = ScriptedPlanner(list(plans)) + verifier, selector = ScriptedVerifier(list(responses)), FixedSelector() + session = TandemSession(make_components(planner, verifier, selector), + config=SessionConfig(execution_mode="stage", **config)) + session.begin("Configure and save", []) + return session, planner, verifier, selector + + +def test_two_actions_use_one_plan_and_fresh_target_ids_then_independent_terminal_check(tmp_path): + session, planner, verifier, selector = session_for( + [stage(milestone("Open", "Save"), milestone("Save", "Saved")), DONE], + [valid_verifier_response(task_status="completed")], + ) + first = session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) + assert isinstance(first, GuiAction) and first.target_id == "a11y:t0:0" + second = session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), + tree_xml=tree(("Save", 500)))) + assert isinstance(second, GuiAction) and second.target_id == "a11y:t1:0" + assert second.parameters["x"] == 540 # Never reuse the old target or coordinates. + assert len(planner.calls) == 1 and verifier.calls == [] and len(selector.calls) == 2 + terminal = session.act(make_observation(tmp_path, 2, receipt=gui_receipt(), + tree_xml=tree(("Saved", 50)))) + assert isinstance(terminal, FinishAction) + assert len(verifier.calls) == 1 and verifier.calls[0]["terminal_claim"] == DONE + verified = [e["verification"] for e in session.events if e["route"] == "gui_verified"] + assert [v["source"] for v in verified] == ["rule_a11y", "rule_a11y"] + assert all(v["task_status"] == "incomplete" for v in verified) + assert len([e for e in session.events if e["route"] == "planner"]) == 2 + + +def test_unbound_target_replans_without_executing_a_near_match(tmp_path): + session, planner, _, selector = session_for([ + stage(milestone("Missing", "Saved")), stage(milestone("Actual", "Saved")), + ]) + action = session.act(make_observation(tmp_path, 0, tree_xml=tree(("Actual", 10)))) + assert isinstance(action, GuiAction) + assert len(planner.calls) == 2 and len(selector.calls) == 1 + assert any(e.get("reason") == "target_unbound_or_ambiguous" for e in session.events) + + +def test_duplicate_names_require_parent_disambiguation(tmp_path): + session, _, _, selector = session_for([stage(milestone("Open", "Saved"))] * 3) + result = session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10), ("Open", 200)))) + assert isinstance(result, FailAction) and result.reason == "no_action_limit" + assert selector.calls == [] + + +def test_failed_cli_receipt_skips_model_verify_and_repairs_remaining_stage(tmp_path): + cli = {"route": "cli", "command": "false", "current_goal": "Read public file", + "expected_observation": "Actual file contents", "checks": []} + session, planner, verifier, _ = session_for([stage(cli), stage(milestone("Open", "Saved"))]) + assert isinstance(session.act(make_observation(tmp_path, 0)), CliAction) + result = session.act(make_observation(tmp_path, 1, receipt=cli_receipt(returncode=1), + tree_xml=tree(("Open", 10)))) + assert isinstance(result, GuiAction) and len(planner.calls) == 2 + assert verifier.calls == [] + assert next(e for e in session.events if e["route"] == "cli_verified")["verification"]["source"] == ( + "rule_receipt" + ) + + +def test_exit_zero_and_printed_success_are_not_rule_proof(tmp_path): + cli = {"route": "cli", "command": "echo success", "current_goal": "Inspect file", + "expected_observation": "Contents satisfy the task", "checks": []} + session, _, verifier, _ = session_for([stage(cli), DONE], [ + valid_verifier_response(), valid_verifier_response(task_status="completed"), + ]) + session.act(make_observation(tmp_path, 0)) + session.act(make_observation(tmp_path, 1, receipt=cli_receipt(stdout="success"))) + assert verifier.calls[0]["execution_result"]["stdout"] == "success" + assert len(verifier.calls) == 2 # Action + independent terminal verification. + + +def test_unknown_a11y_uses_model_not_automatic_success(tmp_path): + session, _, verifier, _ = session_for([stage(milestone("Open", "Saved")), DONE], [ + valid_verifier_response(), valid_verifier_response(task_status="completed"), + ]) + session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) + session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), tree_xml=None)) + assert len(verifier.calls) == 2 + verification = next(e for e in session.events if e["route"] == "gui_verified")["verification"] + assert verification["source"] == "model" + + +def test_preexisting_condition_does_not_prove_action_succeeded(tmp_path): + value = tree(("Open", 10), ("Saved", 100)) + session, _, verifier, _ = session_for([stage(milestone("Open", "Saved")), DONE], [ + valid_verifier_response(), valid_verifier_response(task_status="completed"), + ]) + session.act(make_observation(tmp_path, 0, tree_xml=value)) + session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), tree_xml=value)) + assert len(verifier.calls) == 2 + + +def test_stage_action_limit_requests_replan_before_next_action(tmp_path): + session, planner, _, _ = session_for([ + stage(milestone("Open", "Save"), milestone("Save", "Saved")), + stage(milestone("Save", "Saved")), + ], stage_max_actions=1) + session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) + session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), tree_xml=tree(("Save", 200)))) + assert len(planner.calls) == 2 + assert any(e.get("reason") == "stage_action_limit" for e in session.events) + assert planner.calls[1]["recent_tail"][0]["verified_milestones"] == ["Open Open"] + + +def test_begin_discards_prior_stage(tmp_path): + session, planner, _, _ = session_for([stage(milestone("Open", "Save"))] * 2) + session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) + session.begin("New task", []) + session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) + assert len(planner.calls) == 2 and session.stage.actions == 0 + + +@pytest.mark.parametrize("mutate", [ + lambda p: p.update(done=0), + lambda p: p.update(milestones=[]), + lambda p: p["milestones"][0].update(route="shell"), + lambda p: p["milestones"][0]["action_proposals"][0].update(operation="invented"), + lambda p: p["milestones"][0]["checks"][0].update(expected=1), +]) +def test_invalid_stage_response_is_rejected(mutate): + value = deepcopy(stage(milestone("Open", "Save"))) + assert valid_stage(value) + mutate(value) + assert not valid_stage(value) + + +def test_cli_policy_is_enforced_in_plan_validation(): + value = stage({"route": "cli", "command": "ls", "current_goal": "Inspect", + "expected_observation": "Files", "checks": []}) + assert valid_stage(value) + assert not valid_stage(value, allow_cli=False) + assert not valid_stage(value, route_policy="gui") + assert not valid_stage(stage(milestone("Open", "Save")), route_policy="cli") + + +def test_component_factory_calls_stage_model_with_task_image_and_constraints(tmp_path): + import json + + from tandem.components import build_default + + class Client: + def json_request(self, messages, validator, *, role): + assert role == "plan_stage" + content = messages[1]["content"] + context = json.loads(content[0]["text"]) + assert context["task"] == "Configure and save" and context["allow_cli"] is False + assert content[1]["image_url"]["url"].startswith("data:image/png;base64,") + value = stage(milestone("Open", "Save")) + invalid = deepcopy(value) + invalid["milestones"] *= 9 + assert not validator(invalid) + assert "Invalid stage schema" in messages[-1]["content"] + assert validator(value) + return value + + config = SessionConfig(execution_mode="stage", allow_cli=False) + components = build_default(client=Client(), endpoint="http://unused", config=config) + observation = make_observation(tmp_path, 0) + from pathlib import Path + + assert components.planner("Configure and save", Path(observation.screenshot.path), [], [], + "", None, [], 10)["stage_goal"] == "Configure and save" + + +def test_known_wrong_state_replans_without_model_verify(tmp_path): + change = milestone("Open", "Save") + change["checks"] = [{"target": {"name": "Enabled", "role": "check_box"}, + "property": "checked", "expected": True}] + session, planner, verifier, _ = session_for([ + stage(change), stage(milestone("Other", "Saved")), + ]) + session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) + session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), + tree_xml=tree(("Other", 100)))) + assert verifier.calls == [] and len(planner.calls) == 2 + verified = next(e for e in session.events if e["route"] == "gui_verified")["verification"] + assert verified["outcome"] == "not_achieved" and verified["source"] == "rule_a11y" + + +def test_scroll_can_continue_but_never_claims_milestone_done_from_movement(tmp_path): + scroll = milestone("Open", "Save") + scroll["action_proposals"][0]["operation"] = "scroll_down" + scroll["checks"] = [{"target": {"name": "Enabled", "role": "check_box"}, + "property": "checked", "expected": True}] + session, planner, verifier, _ = session_for([stage(scroll)]) + session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) + result = session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), + tree_xml=tree(("Open", 10), ("New row", 100)))) + assert isinstance(result, GuiAction) and result.action_type == "scroll" + assert len(planner.calls) == 1 and verifier.calls == [] and session.stage.index == 0 + verified = next(e for e in session.events if e["route"] == "gui_verified")["verification"] + assert verified["outcome"] == "uncertain" and verified["continue_local"] is True + + +def test_terminal_rejection_does_not_accept_finished_stage_as_task_success(tmp_path): + session, planner, verifier, _ = session_for([ + stage(milestone("Open", "Save")), DONE, stage(milestone("Save", "Saved")), + ], [valid_verifier_response(task_status="incomplete")]) + session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) + result = session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), tree_xml=tree(("Save", 100)))) + assert isinstance(result, GuiAction) and len(planner.calls) == 3 + assert len(verifier.calls) == 1 + assert any(item["operation"] == "completion_rejected" for item in session.recent) + + +def test_disappearance_is_valid_but_missing_control_needs_independent_evidence(tmp_path): + from tandem.stage_rules import rule_verification + + item = milestone("Open", "Google (Default)") + item["checks"][0]["expected"] = False + assert valid_stage(stage(item)) + pending = {"kind": "cli", "plan": item, "before_tree": tree(("Google (Default)", 10))} + # A different active screen must not prove that a persistent setting was removed. + assert rule_verification(pending, cli_receipt(), tree(("Other screen", 100))) is None From e3c321cb5081256e1d35c48c43d761e39b17cd6e Mon Sep 17 00:00:00 2001 From: xxxyxun Date: Sat, 24 Oct 2026 04:54:04 +0800 Subject: [PATCH 2/4] docs: simplify stage guide and remove standalone stage tests --- docs/STAGE_EXECUTION.md | 210 +++++---------------------- tests/harness/test_stages.py | 265 ----------------------------------- 2 files changed, 36 insertions(+), 439 deletions(-) delete mode 100644 tests/harness/test_stages.py diff --git a/docs/STAGE_EXECUTION.md b/docs/STAGE_EXECUTION.md index 4439def..6539bf5 100644 --- a/docs/STAGE_EXECUTION.md +++ b/docs/STAGE_EXECUTION.md @@ -1,196 +1,58 @@ -# 阶段执行、规则验证与按需重规划 - -## 目的与边界 - -这是对 Tandem 决策策略的可选扩展，不替换环境、排序模型或动作协议。 -默认 `step` 模式保持原有逐步 Planner → 执行 → 模型 Verify 的行为。 -`stage` 模式一次规划一个有边界的阶段，阶段内按新观察持续执行，阶段结束再规划后续工作。 -不强制整任务只规划一次，也不保证固定数量动作内可以完成任意阶段。 - -```text -原始任务 → 需求抽取 → 慢模型规划当前阶段（1–8 个里程碑） - → 当前截图与 A11y → 当前里程碑目标重绑定 → 候选构造 → JEV/Kev 选择 → 真实执行 - → 回执与后续观察 → 规则检查优先，证据不足才用模型 Verify - → 已达成：推进下一里程碑；阶段完成后规划下一阶段 - → 仍可安全滚动：留在当前里程碑，继续观察和选择 - → 失败／无法绑定／证据不足未获确认：请求剩余工作的重规划 - → Planner 声称完成 → 原有输出文件探针与独立终态 Verify → Finish 或返回规划 -``` +# 阶段执行 + +`stage` 模式让慢模型一次规划多个里程碑，由 JEV/Kev 连续选择 GUI 动作，或执行计划中的 CLI 命令。 +默认仍为 `step` 模式，不改变原有单步流程和动作协议。 -CLI 里程碑执行慢模型明确给出的命令，stdout/stderr 和真实退出码进入验证。 -退出码为零、命令打印“成功”、文件名或界面变化都不能单独证明任务完成。 -本版没有“任意自然语言目标自动转换成完整 GUI 程序”的能力；不确定的新界面必须返回规划。 +## 执行规则 -## 配置和启动 +- 每步重新读取截图和 A11y、绑定控件、构造候选，不复用旧坐标或控件 ID。 +- 动作后先检查真实回执与规则条件；证据不足时调用模型 Verify。 +- 里程碑完成后继续执行，阶段完成后规划下一阶段。 +- 目标无法唯一绑定、执行失败、验证未通过或达到阶段动作上限时，请求重规划，并保留已验证进度。 +- 仅有状态变化的滚动动作可在当前里程碑内继续；不盲目重放点击或 CLI 命令。 +- 整任务完成仍经过独立终态 Verify，阶段完成不等于任务完成。 -唯一用户配置仍是 `infra/config.json`。增加可选段，省略时等同下列默认值： +## 启用 + +在 `infra/config.json` 的 `harness` 下设置： ```json "execution": { - "mode": "step", + "mode": "stage", "stage_max_actions": 24 } ``` -该段放在 `harness` 下。`stage_max_actions` 是单个阶段内实际执行动作的上限， -不是强制每隔 24 步调用模型；提前完成、不确定或失败可以更早返回规划。 -全任务步数、墙钟限制仍由现有 budget 和 runner 管理。 - -完成正常集群配置后，在仓库根执行： +也可通过 CLI 覆盖： ```bash -PYTHONPATH=src python -m tandem_eval rjob run \ - --config infra/config.json --repo . --run-id stage-smoke \ - --benchmark osworld1 --tasks bb5e4c0d-f964-439c-97b6-bdb9747de3f4 \ - --execution-mode stage --stage-max-actions 24 \ - --allow-cli true --route-policy auto --max-steps 50 --max-wall 1800 +--execution-mode stage --stage-max-actions 24 ``` -模型仍用现有 `--slow-model NAME`、`--fast-model NAME` 选择。 -恢复单步模式只需使用 `--execution-mode step`。 -集群权限、服务发现、共享环境与 rjob 生命周期约定见 `CLUSTER_RUNTIME.md`。 -当前主线 rjob 客户端存在实际任务名可能追加后缀的已知监控问题；本 PR 不修改该客户端。 -若平台追加后缀，必须按 submit 返回的完整名称检查、归档及清理，不能误判 worker 未启动。 - -## 内部计划格式 - -阶段计划是组件内部数据，不是 Action/Observation/Receipt 协议的新字段。 -计划由 `stage_planner.valid_stage` 校验。示例中的控件名只是说明，不能当真实任务预制动作： -校验失败时向既有客户端的下一次尝试提供格式纠正信息，不增加重试次数、不截断计划或放宽类型。 - -```json -{ - "done": false, - "stage_goal": "打开设置并启用选项", - "milestones": [ - { - "route": "gui", - "current_goal": "打开设置对话框", - "expected_observation": "出现设置中的 Enabled 控件", - "action_proposals": [ - {"operation": "click", "target_name": "Settings", "target_role": "push_button"} - ], - "checks": [ - {"target": {"name": "Enabled", "role": "check_box"}, "property": "visible", "expected": true} - ] - }, - { - "route": "gui", - "current_goal": "启用选项", - "expected_observation": "Enabled 处于选中状态", - "action_proposals": [ - {"operation": "click", "target_name": "Enabled", "target_role": "check_box"} - ], - "checks": [ - {"target": {"name": "Enabled", "role": "check_box"}, "property": "checked", "expected": true} - ] - } - ] -} -``` +`stage_max_actions` 是阶段动作上限，不是固定的模型调用间隔。 +切回原模式使用 `--execution-mode step`；完整集群启动方式见 [CLUSTER_RUNTIME.md](CLUSTER_RUNTIME.md)。 -- 每个里程碑给出一条具体状态转换；GUI 可以给 1–4 个动作提案供候选构造。 -- 不缓存 `target_id` 或坐标。按每帧的精确名称、role、可选 `parent_contains` 重绑。 -- 多个同名控件不能随意选第一个；无法唯一绑定会请求重规划。 -- CLI 里程碑使用 `route: cli`、`command`、目标和预期结果；通常 `checks: []`。 -- GUI 操作沿用现有 click、double_click、type_text、keypress、scroll、slider、wait 路径。 -- 条件支持 `visible: boolean`、精确 `value`、checked/selected/focused/expanded 状态。 - 缺失、歧义和未知状态返回 uncertain；不把“未找到”直接当作“不存在”的证明。 -- 所有后置条件满足且至少一个条件是动作后新出现的证据，才规则确认 achieved。 - 点击前已经可见的按钮本身不能证明点击成功。 -- 无规则、证据未知或所有条件动作前已满足时，交给原有独立模型 Verify。 -- 明确不满足的状态请求修复；仅在界面确有变化的滚动动作中允许局部继续，仍受阶段上限约束。 -- CLI 失败或回执未确认直接记录失败，不自动重放该命令。 - -## 代码地图与开发入口 +## 开发入口 | 文件 | 职责 | |---|---| -| `src/tandem/stage_planner.py` | 阶段 prompt 和内部计划校验；慢模型请求 role 为 plan_stage | -| `src/tandem/stages.py` | 每个 session 的阶段进度、语义重绑定、推进和重规划决策 | -| `src/tandem/stage_rules.py` | A11y 可观察条件与回执检查，不调用模型、不执行环境动作 | -| `src/tandem/components.py` | 根据执行模式选用规划组件，排序后端不变 | -| `src/tandem/session.py` | 保留单步路径，接入阶段组件；动作仍逐个交给环境 | -| `src/tandem/episode.py` | 动作验证分流；终态独立验证沿用原实现 | -| `src/tandem/config.py` | execution_mode、stage_max_actions 的会话配置 | -| `src/tandem_eval/config_schema.py`、`rjob_spec.py`、`__main__.py` | 配置校验、spec 和 CLI 接线 | -| `src/tandem_eval/entry.py` | mock 模式也识别阶段计划，仅用于明确标识的开发测试 | -| `tests/harness/test_stages.py`、`tests/eval/test_stage_config.py` | 执行与配置回归 | -| `tests/eval/test_mock_chain.py` | 明确标识的 step/stage 开发链路与结果目录回归 | -| `infra/config.example.json` | 可提交的执行模式配置模板，不含真实凭据 | -| `README.md`、`DEVIATIONS.md` | 功能入口及相对旧实现的行为差异 | -| `docs/CLUSTER_RUNTIME.md`、`docs/RESULT_DIR_CONTRACT.md`、本文 | 启动参数、事件字段与开发说明 | - -增加规则条件时先修改 `stage_rules.py` 和计划校验，再补对应证据不足/矛盾单测。 -更换阶段规划方法走 `components.py`，不要把特定厂商、任务 ID 或 benchmark 答案写进状态机。 -本 PR 不增加依赖，不修改官方任务、评分器、环境安装或模型服务部署。 -阶段 prompt 明确保留用户原始范围，不能自行增加匹配模式、选择过滤或例外来缩小要求。 - -## 记录与验收 - -沿用 `rjob_logs//episodes//`，不另设结果系统： - -- `events.jsonl`：planner 只在真实调用时记录；stage_local 表示消费阶段计划，不是新模型调用。 - stage_progress 记录里程碑证据；stage_replan 记录修复原因。 -- gui/cli 与 gui_verified/cli_verified 保留现有事件，验证增加 source 与 seconds。 - source 为 rule_receipt、rule_a11y 或 model。planner 事件增加 seconds。 -- `llm_calls.jsonl`：沿用原始请求/响应和 usage；新增 plan_stage role。 - 统计模型次数以该文件为准，不把阶段执行器每次运行计为 Planner 调用。 -- `timing.json`、`episode.json`、`evaluation.json`：继续使用原有 recorder 和官方评分。 - 本 PR 不改变 API 异常耗时的统计口径，不声称已经实现完整的异常开销剔除。 - -开发测试必须验证：阶段内至少两个动作不重复规划、每帧目标重新绑定、失败和未知证据不误报、 -单步路径保持通过、最终完成仍独立检查。真实 smoke 必须明确使用真实模型/JEV/VM， -单题通过不等于固定 50 题效果无退化；大规模同题对比仍需另跑。 - -## 2026-10-24 实际检查与真实试跑 - -本次检查：`ruff check src tests demo` 通过，`PYTHONPATH=src pytest -q` 为 274 passed， -`PYTHONPATH=src python -m tandem check` 为 16 个样例通过；`git diff --check` 通过。 -没有修改 Action/Observation/Receipt schema；没有训练或替换模型，没有添加运行时依赖。 - -真实试跑使用 MiniMax-M3.1-Flash（命名 API 为 minimax-shared）、reasoning_effort=low、 -max_tokens=32768，以及已有训练好的 Qwen-JEV。不是 Kimi/max 强度的历史成绩复现。 -每个 VM worker 为 evobox_proxy/private-machine=group、4 CPU、16384 MiB、0 GPU， -JEV 复用已有 GPU 服务。每次试跑是独立 VM，保留官方任务和原生评分。 -以下全部是模型实时执行，不是 mock 或固定动作回放。 - -结果根目录： - -```text -/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/tandem_stage_execution_pr_20261024/rjob_logs -``` +| `src/tandem/stage_planner.py` | 阶段规划 prompt 与内部计划校验 | +| `src/tandem/stages.py` | 阶段进度、目标重绑定、推进与重规划 | +| `src/tandem/stage_rules.py` | 回执和 A11y 规则验证 | +| `src/tandem/components.py` | 按模式组装规划组件 | +| `src/tandem/session.py`、`episode.py` | 接入执行循环、动作验证与终态检查 | -| run_id | 题目与模式 | 原生分数 | 结论 | -|---|---|---:|---| -| stage-real-gui-20261024 | Chrome 默认搜索引擎，GUI-only 阶段模式 | 1.0 | 5 个真实 GUI 动作；轨迹有同一计划连续两次执行；一次 JEV 超时后自行恢复 | -| stage-real-cli-gui-20261024 | VS Code 替换，初版 CLI+GUI 阶段模式 | 0.0 | Planner 擅自收窄为完整单词匹配，Verify 也接受了收窄解释；不是成功案例 | -| stage-real-scope-v2-20261024 | 同题，第一次约束修订 | 0.0 | 仍错误收窄范围；还暴露过长计划和条件字段格式问题，记录保留 | -| step-real-scope-v2-20261024 | 同题，原有单步模式 | 1.0 | 6 个真实 GUI 动作；允许 CLI，但模型本次选择 GUI | -| stage-real-scope-v3-20261024 | 同题，原文优先、明确字段、紧凑控件与格式纠正后的阶段模式 | 1.0 | 1 CLI + 1 JEV GUI，2 次有效 Planner 调用（含最终完成声明） | -| stage-real-final-20261024 | 同题，最终代码快照 | 1.0 | 8 JEV GUI + 1 CLI；6 次规则检查、3 次模型动作 Verify；4 次 Planner（含完成声明），另有独立终态 Verify | - -初次失败后只补通用约束：原文优先于抽取笔记、不得引入未请求的匹配范围、阶段不要覆盖整条 -未知任务、条件 property 使用明确枚举、格式失败在既有尝试内纠正。没有写入题目 ID、答案、 -目标文件内容或按评分结果修改任务。动作提案是同一前态下的备选，不是有序动作列表。 -修复请求携带本阶段已验证的里程碑，避免重做已确认工作。 -最终记录中 9 次动作验证有 6 次来自 rule_a11y，3 次来自 model，证明真实任务已走规则分流， -但这一个任务的覆盖率不能外推为整个 benchmark 的覆盖率。 - -每次部署保留独立源码目录（deploy、deploy-cli、deploy-scope-v2、deploy-scope-v3、 -deploy-final），不在运行中覆盖 worker 源码。对应私有 spec 在各目录的 job_specs 下，权限 600； -不进入 Git 或 PR。最终任务在 worker 内的实际入口为： +内部计划包含 `done`、`stage_goal` 和 1–8 个 `milestones`。 +每个里程碑描述一个状态转换：GUI 使用 `action_proposals`，CLI 使用 `command`，并给出目标、预期观察和可选 `checks`。 +`action_proposals` 是同一前态下的备选动作，不是有序动作序列。 -```bash -bash /mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/tandem_stage_execution_pr_20261024/deploy-final/infra/worker.sh \ - /mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/tandem_stage_execution_pr_20261024/deploy-final/job_specs/stage-real-final-20261024.json -``` +规则支持可见性、精确文本值和 checked/selected/focused/expanded 状态。 +缺失或歧义证据不能判成功；退出码为零、打印成功或动作前已满足的条件也不能单独证明达成。 + +## 日志与测试 -该命令只应在已申请的 worker 中执行，不能在开发机直接启动 VM。 -真实任务轨迹、截图、原始 LLM usage、timing 和 evaluation 在上述结果根的各 run 内。 -完成任务的平台记录先归档到同级 `rjob_archive/<完整任务名>/`，再删除 rjob；不删除轨迹。 -worker 创建的 run 目录由容器 root 所有，开发机归档使用独立的用户可写目录，未放宽共享目录权限。 +沿用现有结果目录；`events.jsonl` 新增 `stage_local`、`stage_progress`、`stage_replan`， +`llm_calls.jsonl` 新增 `plan_stage` role。字段定义见 [RESULT_DIR_CONTRACT.md](RESULT_DIR_CONTRACT.md)。 -尚未完成的验收：固定 50 题同题性能回归、OSWorld2/2.1 阶段模式回归，以及真实任务中规则 -验证覆盖率的充分验证。这些小样本不能证明准确率无退化或平均时间/token 的因果改善。 -原有单步模式仍为默认，新模式尚不应直接作为默认生产配置。 +配置与入口测试在 `tests/eval/test_stage_config.py`、`test_mock_chain.py`。 +已做 OSWorld1 小样本真实验证，固定 50 题及 OSWorld2/2.1 阶段模式回归尚未完成。 diff --git a/tests/harness/test_stages.py b/tests/harness/test_stages.py deleted file mode 100644 index 8ca1c8c..0000000 --- a/tests/harness/test_stages.py +++ /dev/null @@ -1,265 +0,0 @@ -"""Development fixtures, not benchmark evidence: real session/candidate/compiler paths.""" - -from __future__ import annotations - -from copy import deepcopy - -import pytest - -from tandem.config import SessionConfig -from tandem.protocol.models import CliAction, FailAction, FinishAction, GuiAction -from tandem.session import TandemSession -from tandem.stage_planner import valid_stage - -from .conftest import ( - FixedSelector, - ScriptedPlanner, - ScriptedVerifier, - cli_receipt, - gui_receipt, - make_components, - make_observation, - valid_verifier_response, -) - - -def tree(*buttons, checked=False): - return ('' - + ''.join(f'' - for name, x in buttons) - + f'') - - -def milestone(name, result): - return {"route": "gui", "current_goal": f"Open {name}", "expected_observation": result, - "action_proposals": [{"operation": "click", "target_name": name, - "target_role": "push_button"}], - "checks": [{"target": {"name": result, "role": "push_button"}, - "property": "visible", "expected": True}]} - - -def stage(*milestones): - return {"done": False, "stage_goal": "Configure and save", "milestones": list(milestones)} - - -DONE = {"done": True, "task_status": "completed", "current_goal": "Original task complete"} - - -def session_for(plans, responses=(), **config): - planner = ScriptedPlanner(list(plans)) - verifier, selector = ScriptedVerifier(list(responses)), FixedSelector() - session = TandemSession(make_components(planner, verifier, selector), - config=SessionConfig(execution_mode="stage", **config)) - session.begin("Configure and save", []) - return session, planner, verifier, selector - - -def test_two_actions_use_one_plan_and_fresh_target_ids_then_independent_terminal_check(tmp_path): - session, planner, verifier, selector = session_for( - [stage(milestone("Open", "Save"), milestone("Save", "Saved")), DONE], - [valid_verifier_response(task_status="completed")], - ) - first = session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) - assert isinstance(first, GuiAction) and first.target_id == "a11y:t0:0" - second = session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), - tree_xml=tree(("Save", 500)))) - assert isinstance(second, GuiAction) and second.target_id == "a11y:t1:0" - assert second.parameters["x"] == 540 # Never reuse the old target or coordinates. - assert len(planner.calls) == 1 and verifier.calls == [] and len(selector.calls) == 2 - terminal = session.act(make_observation(tmp_path, 2, receipt=gui_receipt(), - tree_xml=tree(("Saved", 50)))) - assert isinstance(terminal, FinishAction) - assert len(verifier.calls) == 1 and verifier.calls[0]["terminal_claim"] == DONE - verified = [e["verification"] for e in session.events if e["route"] == "gui_verified"] - assert [v["source"] for v in verified] == ["rule_a11y", "rule_a11y"] - assert all(v["task_status"] == "incomplete" for v in verified) - assert len([e for e in session.events if e["route"] == "planner"]) == 2 - - -def test_unbound_target_replans_without_executing_a_near_match(tmp_path): - session, planner, _, selector = session_for([ - stage(milestone("Missing", "Saved")), stage(milestone("Actual", "Saved")), - ]) - action = session.act(make_observation(tmp_path, 0, tree_xml=tree(("Actual", 10)))) - assert isinstance(action, GuiAction) - assert len(planner.calls) == 2 and len(selector.calls) == 1 - assert any(e.get("reason") == "target_unbound_or_ambiguous" for e in session.events) - - -def test_duplicate_names_require_parent_disambiguation(tmp_path): - session, _, _, selector = session_for([stage(milestone("Open", "Saved"))] * 3) - result = session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10), ("Open", 200)))) - assert isinstance(result, FailAction) and result.reason == "no_action_limit" - assert selector.calls == [] - - -def test_failed_cli_receipt_skips_model_verify_and_repairs_remaining_stage(tmp_path): - cli = {"route": "cli", "command": "false", "current_goal": "Read public file", - "expected_observation": "Actual file contents", "checks": []} - session, planner, verifier, _ = session_for([stage(cli), stage(milestone("Open", "Saved"))]) - assert isinstance(session.act(make_observation(tmp_path, 0)), CliAction) - result = session.act(make_observation(tmp_path, 1, receipt=cli_receipt(returncode=1), - tree_xml=tree(("Open", 10)))) - assert isinstance(result, GuiAction) and len(planner.calls) == 2 - assert verifier.calls == [] - assert next(e for e in session.events if e["route"] == "cli_verified")["verification"]["source"] == ( - "rule_receipt" - ) - - -def test_exit_zero_and_printed_success_are_not_rule_proof(tmp_path): - cli = {"route": "cli", "command": "echo success", "current_goal": "Inspect file", - "expected_observation": "Contents satisfy the task", "checks": []} - session, _, verifier, _ = session_for([stage(cli), DONE], [ - valid_verifier_response(), valid_verifier_response(task_status="completed"), - ]) - session.act(make_observation(tmp_path, 0)) - session.act(make_observation(tmp_path, 1, receipt=cli_receipt(stdout="success"))) - assert verifier.calls[0]["execution_result"]["stdout"] == "success" - assert len(verifier.calls) == 2 # Action + independent terminal verification. - - -def test_unknown_a11y_uses_model_not_automatic_success(tmp_path): - session, _, verifier, _ = session_for([stage(milestone("Open", "Saved")), DONE], [ - valid_verifier_response(), valid_verifier_response(task_status="completed"), - ]) - session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) - session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), tree_xml=None)) - assert len(verifier.calls) == 2 - verification = next(e for e in session.events if e["route"] == "gui_verified")["verification"] - assert verification["source"] == "model" - - -def test_preexisting_condition_does_not_prove_action_succeeded(tmp_path): - value = tree(("Open", 10), ("Saved", 100)) - session, _, verifier, _ = session_for([stage(milestone("Open", "Saved")), DONE], [ - valid_verifier_response(), valid_verifier_response(task_status="completed"), - ]) - session.act(make_observation(tmp_path, 0, tree_xml=value)) - session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), tree_xml=value)) - assert len(verifier.calls) == 2 - - -def test_stage_action_limit_requests_replan_before_next_action(tmp_path): - session, planner, _, _ = session_for([ - stage(milestone("Open", "Save"), milestone("Save", "Saved")), - stage(milestone("Save", "Saved")), - ], stage_max_actions=1) - session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) - session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), tree_xml=tree(("Save", 200)))) - assert len(planner.calls) == 2 - assert any(e.get("reason") == "stage_action_limit" for e in session.events) - assert planner.calls[1]["recent_tail"][0]["verified_milestones"] == ["Open Open"] - - -def test_begin_discards_prior_stage(tmp_path): - session, planner, _, _ = session_for([stage(milestone("Open", "Save"))] * 2) - session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) - session.begin("New task", []) - session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) - assert len(planner.calls) == 2 and session.stage.actions == 0 - - -@pytest.mark.parametrize("mutate", [ - lambda p: p.update(done=0), - lambda p: p.update(milestones=[]), - lambda p: p["milestones"][0].update(route="shell"), - lambda p: p["milestones"][0]["action_proposals"][0].update(operation="invented"), - lambda p: p["milestones"][0]["checks"][0].update(expected=1), -]) -def test_invalid_stage_response_is_rejected(mutate): - value = deepcopy(stage(milestone("Open", "Save"))) - assert valid_stage(value) - mutate(value) - assert not valid_stage(value) - - -def test_cli_policy_is_enforced_in_plan_validation(): - value = stage({"route": "cli", "command": "ls", "current_goal": "Inspect", - "expected_observation": "Files", "checks": []}) - assert valid_stage(value) - assert not valid_stage(value, allow_cli=False) - assert not valid_stage(value, route_policy="gui") - assert not valid_stage(stage(milestone("Open", "Save")), route_policy="cli") - - -def test_component_factory_calls_stage_model_with_task_image_and_constraints(tmp_path): - import json - - from tandem.components import build_default - - class Client: - def json_request(self, messages, validator, *, role): - assert role == "plan_stage" - content = messages[1]["content"] - context = json.loads(content[0]["text"]) - assert context["task"] == "Configure and save" and context["allow_cli"] is False - assert content[1]["image_url"]["url"].startswith("data:image/png;base64,") - value = stage(milestone("Open", "Save")) - invalid = deepcopy(value) - invalid["milestones"] *= 9 - assert not validator(invalid) - assert "Invalid stage schema" in messages[-1]["content"] - assert validator(value) - return value - - config = SessionConfig(execution_mode="stage", allow_cli=False) - components = build_default(client=Client(), endpoint="http://unused", config=config) - observation = make_observation(tmp_path, 0) - from pathlib import Path - - assert components.planner("Configure and save", Path(observation.screenshot.path), [], [], - "", None, [], 10)["stage_goal"] == "Configure and save" - - -def test_known_wrong_state_replans_without_model_verify(tmp_path): - change = milestone("Open", "Save") - change["checks"] = [{"target": {"name": "Enabled", "role": "check_box"}, - "property": "checked", "expected": True}] - session, planner, verifier, _ = session_for([ - stage(change), stage(milestone("Other", "Saved")), - ]) - session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) - session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), - tree_xml=tree(("Other", 100)))) - assert verifier.calls == [] and len(planner.calls) == 2 - verified = next(e for e in session.events if e["route"] == "gui_verified")["verification"] - assert verified["outcome"] == "not_achieved" and verified["source"] == "rule_a11y" - - -def test_scroll_can_continue_but_never_claims_milestone_done_from_movement(tmp_path): - scroll = milestone("Open", "Save") - scroll["action_proposals"][0]["operation"] = "scroll_down" - scroll["checks"] = [{"target": {"name": "Enabled", "role": "check_box"}, - "property": "checked", "expected": True}] - session, planner, verifier, _ = session_for([stage(scroll)]) - session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) - result = session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), - tree_xml=tree(("Open", 10), ("New row", 100)))) - assert isinstance(result, GuiAction) and result.action_type == "scroll" - assert len(planner.calls) == 1 and verifier.calls == [] and session.stage.index == 0 - verified = next(e for e in session.events if e["route"] == "gui_verified")["verification"] - assert verified["outcome"] == "uncertain" and verified["continue_local"] is True - - -def test_terminal_rejection_does_not_accept_finished_stage_as_task_success(tmp_path): - session, planner, verifier, _ = session_for([ - stage(milestone("Open", "Save")), DONE, stage(milestone("Save", "Saved")), - ], [valid_verifier_response(task_status="incomplete")]) - session.act(make_observation(tmp_path, 0, tree_xml=tree(("Open", 10)))) - result = session.act(make_observation(tmp_path, 1, receipt=gui_receipt(), tree_xml=tree(("Save", 100)))) - assert isinstance(result, GuiAction) and len(planner.calls) == 3 - assert len(verifier.calls) == 1 - assert any(item["operation"] == "completion_rejected" for item in session.recent) - - -def test_disappearance_is_valid_but_missing_control_needs_independent_evidence(tmp_path): - from tandem.stage_rules import rule_verification - - item = milestone("Open", "Google (Default)") - item["checks"][0]["expected"] = False - assert valid_stage(stage(item)) - pending = {"kind": "cli", "plan": item, "before_tree": tree(("Google (Default)", 10))} - # A different active screen must not prove that a persistent setting was removed. - assert rule_verification(pending, cli_receipt(), tree(("Other screen", 100))) is None From 8d3f53aa7b2e758d3bdab3fb45c965dac7dbda90 Mon Sep 17 00:00:00 2001 From: xxxyxun Date: Sat, 24 Oct 2026 04:54:37 +0800 Subject: [PATCH 3/4] docs: keep stage execution note brief --- docs/STAGE_EXECUTION.md | 57 +---------------------------------------- 1 file changed, 1 insertion(+), 56 deletions(-) diff --git a/docs/STAGE_EXECUTION.md b/docs/STAGE_EXECUTION.md index 6539bf5..60c90f9 100644 --- a/docs/STAGE_EXECUTION.md +++ b/docs/STAGE_EXECUTION.md @@ -1,58 +1,3 @@ # 阶段执行 -`stage` 模式让慢模型一次规划多个里程碑，由 JEV/Kev 连续选择 GUI 动作，或执行计划中的 CLI 命令。 -默认仍为 `step` 模式，不改变原有单步流程和动作协议。 - -## 执行规则 - -- 每步重新读取截图和 A11y、绑定控件、构造候选，不复用旧坐标或控件 ID。 -- 动作后先检查真实回执与规则条件；证据不足时调用模型 Verify。 -- 里程碑完成后继续执行，阶段完成后规划下一阶段。 -- 目标无法唯一绑定、执行失败、验证未通过或达到阶段动作上限时，请求重规划，并保留已验证进度。 -- 仅有状态变化的滚动动作可在当前里程碑内继续；不盲目重放点击或 CLI 命令。 -- 整任务完成仍经过独立终态 Verify，阶段完成不等于任务完成。 - -## 启用 - -在 `infra/config.json` 的 `harness` 下设置： - -```json -"execution": { - "mode": "stage", - "stage_max_actions": 24 -} -``` - -也可通过 CLI 覆盖： - -```bash ---execution-mode stage --stage-max-actions 24 -``` - -`stage_max_actions` 是阶段动作上限，不是固定的模型调用间隔。 -切回原模式使用 `--execution-mode step`；完整集群启动方式见 [CLUSTER_RUNTIME.md](CLUSTER_RUNTIME.md)。 - -## 开发入口 - -| 文件 | 职责 | -|---|---| -| `src/tandem/stage_planner.py` | 阶段规划 prompt 与内部计划校验 | -| `src/tandem/stages.py` | 阶段进度、目标重绑定、推进与重规划 | -| `src/tandem/stage_rules.py` | 回执和 A11y 规则验证 | -| `src/tandem/components.py` | 按模式组装规划组件 | -| `src/tandem/session.py`、`episode.py` | 接入执行循环、动作验证与终态检查 | - -内部计划包含 `done`、`stage_goal` 和 1–8 个 `milestones`。 -每个里程碑描述一个状态转换：GUI 使用 `action_proposals`，CLI 使用 `command`，并给出目标、预期观察和可选 `checks`。 -`action_proposals` 是同一前态下的备选动作，不是有序动作序列。 - -规则支持可见性、精确文本值和 checked/selected/focused/expanded 状态。 -缺失或歧义证据不能判成功；退出码为零、打印成功或动作前已满足的条件也不能单独证明达成。 - -## 日志与测试 - -沿用现有结果目录；`events.jsonl` 新增 `stage_local`、`stage_progress`、`stage_replan`， -`llm_calls.jsonl` 新增 `plan_stage` role。字段定义见 [RESULT_DIR_CONTRACT.md](RESULT_DIR_CONTRACT.md)。 - -配置与入口测试在 `tests/eval/test_stage_config.py`、`test_mock_chain.py`。 -已做 OSWorld1 小样本真实验证，固定 50 题及 OSWorld2/2.1 阶段模式回归尚未完成。 +通过 `--execution-mode stage` 启用：慢模型规划阶段，JEV/Kev 连续选择 GUI 动作，CLI 执行计划中的命令；优先使用规则验证，证据不足时调用模型 Verify，失败或状态偏离时按需重规划，最终完成仍需独立验证。默认保留原有 `step` 模式。 From 8428ab4636a2fc9a9c33458dfd6c699046498f71 Mon Sep 17 00:00:00 2001 From: xxxyxun Date: Sat, 24 Oct 2026 04:55:24 +0800 Subject: [PATCH 4/4] test: remove standalone stage configuration tests --- tests/eval/test_stage_config.py | 60 --------------------------------- 1 file changed, 60 deletions(-) delete mode 100644 tests/eval/test_stage_config.py diff --git a/tests/eval/test_stage_config.py b/tests/eval/test_stage_config.py deleted file mode 100644 index 40cbd54..0000000 --- a/tests/eval/test_stage_config.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Stage-mode knobs reach the same worker session through config, spec and CLI.""" - -from __future__ import annotations - -import pytest - -from tandem.config import SessionConfig -from tandem_eval.__main__ import main -from tandem_eval.config_schema import validate -from tandem_eval.entry import build_session -from tandem_eval.rjob_spec import build_job_spec - -from .rjob_fakes import CONFIG, with_paths - - -def test_stage_config_reaches_worker(tmp_path): - config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy"), results_root=str(tmp_path)) - config["harness"]["execution"] = {"mode": "stage", "stage_max_actions": 12} - assert validate(config) == [] - spec = build_job_spec(config=config, run_id="stage", repo_root=tmp_path, - spec_overrides={"benchmark": "mock"}) - assert spec["harness"]["mode"] == "stage" - session = build_session(spec) - assert session.config.execution_mode == "stage" and session.stage.max_actions == 12 - - -def test_old_config_keeps_single_step_default(tmp_path): - config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy"), results_root=str(tmp_path)) - spec = build_job_spec(config=config, run_id="step", repo_root=tmp_path, spec_overrides={}) - assert build_session(spec).stage is None - - -@pytest.mark.parametrize("execution", [{"mode": "invalid"}, {"stage_max_actions": False}, - {"stage_max_actions": 0}, {"stage_max_actions": 1.5}]) -def test_invalid_execution_config_is_reported(execution): - config = with_paths(CONFIG) - config["harness"]["execution"] = execution - assert any("harness.execution" in problem for problem in validate(config)) - - -def test_session_rejects_invalid_execution_knobs(): - with pytest.raises(ValueError, match="execution_mode"): - SessionConfig(execution_mode="unknown") - with pytest.raises(ValueError, match="stage_max_actions"): - SessionConfig(stage_max_actions=False) - - -def test_cli_execution_flags_reach_overrides(monkeypatch): - from pathlib import Path - - captured = {} - monkeypatch.setattr("tandem_eval.rjob.run_job", lambda **kwargs: ( - captured.update(kwargs) or {"status": "succeeded"} - )) - config_path = Path(__file__).resolve().parents[2] / "infra/config.example.json" - assert main(["rjob", "run", "--run-id", "stage", "--config", str(config_path), - "--execution-mode", "stage", "--stage-max-actions", "12"]) == 0 - assert captured["spec_overrides"]["harness_config"] == { - "execution_mode": "stage", "stage_max_actions": 12, - }