diff --git a/docs/CLUSTER_RUNTIME.md b/docs/CLUSTER_RUNTIME.md index a516811..5dfcbd5 100644 --- a/docs/CLUSTER_RUNTIME.md +++ b/docs/CLUSTER_RUNTIME.md @@ -98,7 +98,7 @@ cleanup stop（释放资源）+ delete（永久删除 rjob 记录）——默 | 参数 | 说明 | |---|---| | `--run-id` | 必填。run 目录名 = `rjob_logs//` | -| `--benchmark` | `mock` / `osworld1` / `osworld2` | +| `--benchmark` | `mock` / `osworld1` / `osworld2` / `osworld21` | | `--tasks` | 逗号分隔 task id（覆盖 manifest） | | `--limit N` | 取清单前 N 题 | | `--shard I/N` | 把（limit 后的）清单**交错**切成 N 片，本 run 跑第 I 片（并发用） | @@ -134,6 +134,9 @@ tandem-eval rjob watch tandem- --run-id --config infra/config.j 5. 用 `$OSWORLD_PYTHON` 跑 `tandem_eval entry`（Pillow / easyocr 已在冻结 venv 内）， 跑前先做一次依赖自检。 +OSW2.1 使用 `osw21_shared_env`、`osw21_proxy` 和 `osw21_inner_image`， +直接拉取固定版本镜像，不覆盖 `latest`；冻结 venv 只检查 Pillow，不要求 easyocr。 + worker 侧 `tandem_eval entry --spec ` 是实际执行体：起 VM → 跑任务 → 录事件 → 评分 → 写 run 目录。环境生命周期、预算强制、落盘都在这一层，harness 只管决策。 @@ -220,6 +223,6 @@ worker 侧各段日志： 真机 smoke： - `--benchmark mock` 走 mock 链路（不起 VM、不碰共享资源），可本地跑； -- 真实基准走 OSW1 / OSW2，按本文 §3–§7。 +- 真实基准走 OSW1 / OSW2 / OSW2.1，按本文 §3–§7。 上游重构总计划：`../../TANDEM_REFACTOR_PLAN_CN.md`（在仓库外，与本仓库同级目录的上级）。 diff --git a/infra/config.example.json b/infra/config.example.json index 10f30d2..48904bf 100644 --- a/infra/config.example.json +++ b/infra/config.example.json @@ -11,6 +11,10 @@ { "source": "gpfs://gpfs2/adpc-share", "target": "/mnt/shared-storage-user/adpc-share" + }, + { + "source": "gpfs://gpfs2/evobox-share-gpfs2", + "target": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2" } ], "rjob": { @@ -27,6 +31,14 @@ "osw2_root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/benchmarks/osworld2", "osw2_vm": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/vm/osworld2/osworld-v2-ubuntu-x86.qcow2", "asset_root": "/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld2_0808/assets", + "osw21_root": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/prepared/runtime", + "osw21_vm": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/vm/osworld-v2-ubuntu-x86.qcow2", + "osw21_asset_root": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/assets", + "osw21_shared_env": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/prepared/env.sh", + "_osw21_inner_image_comment": "Fill osw21_inner_image with the release-pinned image available to your registry account; no latest tag.", + "osw21_inner_image": "", + "osw21_proxy": "http://100.100.135.73:3128", + "osw21_volume_size": 64, "kimi": { "base_url": "https://your-kimi-endpoint/v1", "api_key": "sk-xxx", diff --git a/infra/worker.sh b/infra/worker.sh index ab4f7b6..ab1ee3a 100755 --- a/infra/worker.sh +++ b/infra/worker.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # rjob worker 内入口：读取 job spec，进入 tandem_eval entry。 # - mock 基准：直接用系统 python3 跑（无需共享资源/docker）。 -# - osworld1/2：加载共享资源 env.sh（冻结 venv/评测工具/JEV），启动 dockerd 并拉取 +# - osworld1/2/21：加载共享资源 env.sh（冻结 venv/评测工具/JEV），启动 dockerd 并拉取 # inner 镜像（VM 容器），然后用 OSWORLD_PYTHON 跑 entry（Pillow 等依赖已在冻结 venv 内）。 set -euo pipefail @@ -27,6 +27,9 @@ fi # ---- 真实基准：共享资源 + docker 准备 ---- SHARED_ENV="${SA_CUA_SHARED_ENV:-/mnt/shared-storage-user/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/env.sh}" +if [ "$BENCHMARK" = "osworld21" ]; then + SHARED_ENV="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["env"]["shared_env"])' "$SPEC")" +fi if [ ! -f "$SHARED_ENV" ]; then echo "缺少共享资源 env.sh: $SHARED_ENV" | tee "$OUTPUT_ROOT/worker/entry.log" >&2 exit 3 @@ -48,7 +51,8 @@ chmod 600 "$HOME/.docker/config.json" INNER_IMAGE="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["code"]["images"]["inner"])' "$SPEC")" # 平台可能预注入不可用的代理（真机 2026-10-20：httpproxy-headless-dev-test 无法解析）， # 一律覆盖为已知可用代理，不继承环境变量。 -PROXY="http://httpproxy-headless.kubebrain.svc.pjlab.local:3128" +PROXY="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("env", {}).get( + "proxy", "http://httpproxy-headless.kubebrain.svc.pjlab.local:3128"))' "$SPEC")" export HTTP_PROXY="$PROXY" HTTPS_PROXY="$PROXY" http_proxy="$PROXY" https_proxy="$PROXY" export NO_PROXY='10.0.0.0/8,100.96.0.0/12,localhost,127.0.0.1,.pjlab.org.cn' export no_proxy="$NO_PROXY" @@ -65,9 +69,14 @@ done docker info > "$OUTPUT_ROOT/worker/docker-info.txt" 2>&1 || { echo "dockerd 未就绪" | tee -a "$OUTPUT_ROOT/worker/entry.log" >&2; exit 3; } docker pull "$INNER_IMAGE" > "$OUTPUT_ROOT/worker/docker-pull.log" 2>&1 -docker tag "$INNER_IMAGE" happysixd/osworld-docker:latest +if [ "$BENCHMARK" = "osworld21" ]; then + DEPS_CHECK='import PIL' +else + docker tag "$INNER_IMAGE" happysixd/osworld-docker:latest + DEPS_CHECK='import easyocr, PIL; assert callable(easyocr.Reader)' +fi -"$OSWORLD_PYTHON" -c 'import easyocr, PIL; assert callable(easyocr.Reader)' \ +"$OSWORLD_PYTHON" -c "$DEPS_CHECK" \ > "$OUTPUT_ROOT/worker/deps-check.log" 2>&1 || { echo "共享 venv 依赖检查失败" | tee -a "$OUTPUT_ROOT/worker/entry.log" >&2; exit 3; } diff --git a/scripts/run_osw2_20x5.sh b/scripts/run_osw2_20x5.sh index bfd1636..8295769 100755 --- a/scripts/run_osw2_20x5.sh +++ b/scripts/run_osw2_20x5.sh @@ -6,6 +6,8 @@ # 可用环境变量： # LIMIT（默认 20）/ SHARDS（默认 5）/ MAX_STEPS（默认 30）/ MAX_WALL（默认 1800） # TIMEOUT（整片 watch 上限，默认 7200）/ PREFIX（run-id 前缀，默认 osw2-20） +# OSW2.1：BENCHMARK=osworld21 LIMIT=56 SHARDS=4 PREFIX=osw21-56 bash scripts/run_osw2_20x5.sh +# 单题：再设置 TASKS=004 LIMIT=1 SHARDS=1；ROUTE_POLICY=gui 可限定 GUI 路由。 set -u REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" @@ -16,6 +18,8 @@ SHARDS="${SHARDS:-5}" MAX_STEPS="${MAX_STEPS:-30}" MAX_WALL="${MAX_WALL:-1800}" TIMEOUT="${TIMEOUT:-7200}" +BENCHMARK="${BENCHMARK:-osworld2}" +TASKS="${TASKS:-}" PREFIX="${PREFIX:-osw2-20}" BATCH_TS="$(date +%Y%m%d-%H%M%S)" LOG_DIR="rjob_logs/${PREFIX}x${SHARDS}-${BATCH_TS}" @@ -26,6 +30,12 @@ echo "[$(date '+%F %T')] 批次启动: ${LIMIT} 题 / ${SHARDS} 并发 → 日 pids=() for i in $(seq 1 "$SHARDS"); do EXTRA=() + if [ -n "$TASKS" ]; then + EXTRA+=(--tasks "$TASKS") + fi + if [ -n "${ROUTE_POLICY:-}" ]; then + EXTRA+=(--route-policy "$ROUTE_POLICY") + fi # 只有第 1 片做 rsync：并发 --delete 同步会互删部署副本 if [ "$i" -gt 1 ]; then EXTRA+=(--no-sync) @@ -33,7 +43,7 @@ for i in $(seq 1 "$SHARDS"); do echo "[$(date '+%F %T')] 提交分片 $i/$SHARDS (run-id ${PREFIX}-s${i})" tandem-eval rjob run \ --run-id "${PREFIX}-s${i}" \ - --benchmark osworld2 \ + --benchmark "$BENCHMARK" \ --limit "$LIMIT" --shard "$i/$SHARDS" \ --max-steps "$MAX_STEPS" --max-wall "$MAX_WALL" \ --repo . --config infra/config.json \ diff --git a/src/tandem_eval/__main__.py b/src/tandem_eval/__main__.py index 2f53e7f..92f20a9 100644 --- a/src/tandem_eval/__main__.py +++ b/src/tandem_eval/__main__.py @@ -26,7 +26,7 @@ def main(argv: list[str] | None = None) -> int: _rjob_args(run_cmd) run_cmd.add_argument("--repo", default=".", help="本地仓库根") run_cmd.add_argument("--run-id", required=True) - run_cmd.add_argument("--benchmark", default="mock", choices=["mock", "osworld1", "osworld2"]) + run_cmd.add_argument("--benchmark", default="mock", choices=["mock", "osworld1", "osworld2", "osworld21"]) run_cmd.add_argument("--tasks", default="", help="逗号分隔 task id（覆盖 manifest）") run_cmd.add_argument("--limit", type=int, default=None, help="清单截取数量") run_cmd.add_argument("--shard", default=None, metavar="I/N", diff --git a/src/tandem_eval/entry.py b/src/tandem_eval/entry.py index 0625879..db89d7c 100644 --- a/src/tandem_eval/entry.py +++ b/src/tandem_eval/entry.py @@ -1,8 +1,8 @@ """entry：job spec → 环境/harness/runner 组装（worker 侧唯一入口，infra/worker.sh 调用）。 benchmark=mock：MockEnv + 确定性脚本组件（全链路自检）。 -benchmark=osworld1/2：OsWorldEnv（共享目录 runtime/VM）+ KimiClient + build_default 组件， -凭据与端点路径由 rjob.py 在提交侧从环境档案注入 spec（secrets 段）。 +真实基准：环境适配器（共享目录 runtime/VM）+ KimiClient + build_default 组件， +凭据与端点路径由提交侧从 infra/config.json 注入 spec（secrets 段）。 """ from __future__ import annotations @@ -155,7 +155,7 @@ def build_session(spec: dict, *, llm_tap: LlmTap | None = None) -> TandemSession benchmark = spec.get("benchmark") if benchmark == "mock": return TandemSession(mock_components(), config=config) - if benchmark in {"osworld1", "osworld2"}: + if benchmark in {"osworld1", "osworld2", "osworld21"}: from tandem.components import build_default from tandem.providers import KimiClient @@ -184,7 +184,7 @@ def build_env(spec: dict): if benchmark == "mock": base = Path(spec["output_root"]) / "mock_env" return MockEnv(base_dir=base) - if benchmark in {"osworld1", "osworld2"}: + if benchmark in {"osworld1", "osworld2", "osworld21"}: from .osworld import OsWorldEnv env_spec = spec["env"] @@ -196,6 +196,7 @@ def build_env(spec: dict): asset_root=Path(env_spec["asset_root"]) if env_spec.get("asset_root") else None, setup_cache_root=Path(env_spec["setup_cache_root"]) if env_spec.get("setup_cache_root") else None, startup_wait_seconds=float(env_spec.get("startup_wait_seconds", 30.0)), + volume_size=env_spec.get("volume_size"), ) raise NotImplementedError(f"未知 benchmark: {benchmark}") diff --git a/src/tandem_eval/manifest.py b/src/tandem_eval/manifest.py index 12eca54..adfbb56 100644 --- a/src/tandem_eval/manifest.py +++ b/src/tandem_eval/manifest.py @@ -1,5 +1,4 @@ -"""任务清单：OSW1 用共享目录 test_nogdrive.json（361 题），OSW2 用 test_v2.json（108 题）， -或显式 id 列表 / repo 内清单文件。""" +"""任务清单：OSW1/OSW2 使用共享索引，OSW2.1 使用 56 题清单，也支持显式题号或清单文件。""" from __future__ import annotations @@ -7,6 +6,13 @@ from dataclasses import dataclass from pathlib import Path +OSWORLD21_TASK_IDS = tuple(( + "001 002 003 004 006 009 010 011 012 014 017 022 023 024 " + "025 027 029 030 032 034 037 040 042 044 045 049 051 053 " + "054 055 059 061 062 063 066 067 071 074 075 076 078 080 " + "083 084 085 086 088 089 090 092 093 094 096 097 099 108" +).split()) + @dataclass(frozen=True) class TaskRef: @@ -48,6 +54,8 @@ def resolve_manifest(spec: dict) -> list[TaskRef]: if not path.is_absolute(): path = Path(spec["repo_root"]) / path refs = load_index(path) + elif benchmark == "osworld21": + refs = [TaskRef(task_id=task_id) for task_id in OSWORLD21_TASK_IDS] elif benchmark in {"osworld1", "osworld2"} and ( spec.get("index_root") or spec.get("env", {}).get("root") ): @@ -65,6 +73,10 @@ def resolve_manifest(spec: dict) -> list[TaskRef]: refs = [TaskRef(task_id=f"mock-{i:03d}", domain="mock") for i in range(1)] else: raise ValueError(f"无法解析 manifest：benchmark={benchmark} 且未提供 index_root/tasks/path") + if benchmark == "osworld21": + unsupported = [ref.task_id for ref in refs if ref.task_id not in OSWORLD21_TASK_IDS] + if unsupported: + raise ValueError(f"OSWorld 2.1 tasks outside the supported subset: {unsupported}") limit = manifest.get("limit") if isinstance(limit, int) and limit > 0: refs = refs[:limit] diff --git a/src/tandem_eval/osworld.py b/src/tandem_eval/osworld.py index 9331212..092d3bc 100644 --- a/src/tandem_eval/osworld.py +++ b/src/tandem_eval/osworld.py @@ -1,10 +1,6 @@ -"""OSW1/OSW2 真实环境适配器：DesktopEnv ↔ 协议消息。 +"""OSW1/OSW2/OSW2.1 统一环境适配器：DesktopEnv ↔ 协议消息。 -- 代码生成（gui_code/cli_code）语义对齐旧仓库 gui/actions.py + execution.py（DEVIATIONS #6）； -- 回执确认纪律：包装 controller.execute_python_command，无回执不重放（旧 execution.py:30-35）； -- want_observation="none" 当前仍返回完整观察（截图+a11y 由 controller 一次读取，成本低）， - 协议字段保留，严格跳过推迟到真机 profiling 后（DEVIATIONS #20）； -- 终态动作（finish/fail）经 env.step("DONE"/"FAIL") 送达 VM（评分钩子依赖它，旧 runtime 同款）。 +无回执不重放动作；终态 DONE/FAIL 送达 VM。want_observation="none" 暂仍返回完整观察。 """ from __future__ import annotations @@ -54,10 +50,7 @@ def gui_code(action: GuiAction) -> str: steps.append(f"pyautogui.{'press' if len(keys) == 1 else 'hotkey'}({keys_text})") return "; __import__('time').sleep(0.2); ".join(steps) if action.action_type == "scroll": - return ( - f"pyautogui.moveTo({params['x']}, {params['y']}); " - f"pyautogui.scroll({params['dy']})" - ) + return f"pyautogui.moveTo({params['x']}, {params['y']}); pyautogui.scroll({params['dy']})" if action.action_type == "type": steps = [] if params.get("click"): @@ -101,6 +94,7 @@ class OsWorldEnv: def __init__(self, *, benchmark: str, task_root: Path, vm_path: Path, run_root: Path, asset_root: Path | None = None, setup_cache_root: Path | None = None, + volume_size: int | None = None, startup_wait_seconds: float = 30.0, env_factory: Callable[..., Any] | None = None): self.benchmark = benchmark self.task_root = Path(task_root) @@ -108,12 +102,10 @@ def __init__(self, *, benchmark: str, task_root: Path, vm_path: Path, self.run_root = Path(run_root) self.asset_root = Path(asset_root) if asset_root else None self.setup_cache_root = Path(setup_cache_root) if setup_cache_root else None + self.volume_size = volume_size self.startup_wait_seconds = startup_wait_seconds self._env_factory = env_factory or self._import_desktop_env - # bootstrap 必须先于 fetch_task：OSW2 任务模块在 import 时（fetch_task 阶段） - # 就于顶层调用 asset() 固化资产 URL——晚设 OSWORLD_FILE_BASE_URL 会让 - # 模块级 URL 烘焙成默认 HF 地址（task_001 真机 404 的根因；函数体内调用 - # 的 task_066 恰好躲过） + # 任务模块在 import 时解析资产 URL，必须先设置本地资产根路径。 self._bootstrap_runtime() def _bootstrap_runtime(self) -> None: @@ -131,12 +123,13 @@ def _bootstrap_runtime(self) -> None: def name(self) -> str: return f"osworld-{self.benchmark}" - # ------------------------------------------------------------------ 装载 def _import_desktop_env(self, **kwargs): self._bootstrap_runtime() os.chdir(self.task_root) from desktop_env.desktop_env import DesktopEnv + if self.volume_size is not None: + kwargs["volume_size"] = self.volume_size return DesktopEnv(**kwargs) def _task_file(self, task_id: str, domain: str) -> Path: @@ -171,7 +164,6 @@ def fetch_task(self, task_id: str, domain: str = "") -> dict: ) return task - # ------------------------------------------------------------------ 观察 def _observation(self, task: dict, step_idx: int, budget: Budget, receipt: Receipt | None) -> Observation: stage = self.run_root / "obs_stage" / str(task["id"]) @@ -205,10 +197,8 @@ def _capture(self) -> None: self.last_screenshot = obs["screenshot"] self.last_a11y = obs.get("accessibility_tree") or "" - # ------------------------------------------------------------------ 协议 def _seed_setup_cache(self, task_id: str) -> None: - """把共享盘已缓存的 setup 文件播种进本次 evaluation_cache（旧 runtime.py:532-539）， - 避免任务 setup 走外网下载（HF 下载在 worker 内经代理，慢且可能失败）。""" + """把已缓存的 setup 文件复制到本次 evaluation_cache，避免外网下载。""" if self.setup_cache_root is None: return source = self.setup_cache_root / task_id diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py index a09bce1..84fe476 100644 --- a/src/tandem_eval/rjob_spec.py +++ b/src/tandem_eval/rjob_spec.py @@ -31,6 +31,11 @@ def _git_describe(repo_root: Path) -> str: def _env_section(benchmark: str, config: dict) -> dict: + if benchmark == "osworld21": + return {"root": config["osw21_root"], "vm": config["osw21_vm"], + "asset_root": config["osw21_asset_root"], "shared_env": config["osw21_shared_env"], + "proxy": config["osw21_proxy"], "volume_size": config["osw21_volume_size"], + "startup_wait_seconds": 30} if benchmark == "osworld1": return {"root": config["osw1_root"], "vm": config["osw1_vm"], "setup_cache_root": str(Path(config["osw1_root"]) / "cache"), @@ -42,7 +47,7 @@ def _env_section(benchmark: str, config: dict) -> dict: def _secrets_section(benchmark: str, config: dict) -> dict: - if benchmark not in {"osworld1", "osworld2"}: + if benchmark not in {"osworld1", "osworld2", "osworld21"}: return {} section = { "kimi": dict(config["kimi"]), @@ -74,6 +79,8 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path, spec_overrides: dict) -> dict: """组装 job spec 并写入部署副本 job_specs/.json（路径两侧一致，无视角换算）。""" benchmark = spec_overrides.get("benchmark", "mock") + if benchmark == "osworld21" and not config.get("osw21_inner_image"): + raise ValueError("osw21_inner_image must identify the release-pinned OSWorld 2.1 runtime image") spec = { "run_id": run_id, "repo_root": config["deploy_root"], @@ -89,7 +96,7 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path, "git_describe": _git_describe(repo_root), "images": { "outer": config["rjob"]["image"], - "inner": config["inner_image"], + "inner": config["osw21_inner_image"] if benchmark == "osworld21" else config["inner_image"], }, }, **spec_overrides, diff --git a/tests/eval/test_config.py b/tests/eval/test_config.py index 7d653ca..bba8bae 100644 --- a/tests/eval/test_config.py +++ b/tests/eval/test_config.py @@ -15,7 +15,8 @@ def test_repo_config_loads_and_is_absolute(): config = load_config(CONFIG_PATH) PATH_KEYS = ("repo_root", "deploy_root", "results_root", "osw1_root", "osw1_vm", - "osw2_root", "osw2_vm", "asset_root") + "osw2_root", "osw2_vm", "asset_root", "osw21_root", "osw21_vm", + "osw21_asset_root", "osw21_shared_env") for key in PATH_KEYS: assert config[key].startswith("/"), f"{key} 必须是绝对路径: {config[key]}" assert config["inner_image"].startswith("registry.") @@ -28,6 +29,8 @@ def test_repo_config_loads_and_is_absolute(): assert "auths" in config["registry"] assert config["results_root"].endswith("tandem/rjob_logs") assert config["repo_root"].endswith("/tandem") + for key in ("osw21_root", "osw21_vm", "osw21_asset_root", "osw21_shared_env"): + assert "/shared_resources/sa_cua/" in config[key] def test_mounts_cover_both_volumes_with_user_style_targets(): @@ -62,6 +65,45 @@ def test_real_config_is_gitignored_with_example_committed(): assert "infra/config.json" in ignore, "含密钥的真实配置必须 gitignore" +def test_osworld21_spec_uses_configured_environment_without_changing_osw2(tmp_path): + from tandem_eval.rjob_spec import build_job_spec + + config = load_config(CONFIG_PATH) + config.update(deploy_root=str(tmp_path / "deploy"), osw21_volume_size=80, + osw21_inner_image="registry.example/runtime:2.1") + spec = build_job_spec(config=config, run_id="new", repo_root=tmp_path, + spec_overrides={"benchmark": "osworld21"}) + for key in ("root", "vm", "asset_root", "shared_env", "proxy", "volume_size"): + assert spec["env"][key] == config[f"osw21_{key}"] + assert "setup_bundles" not in spec["env"] + assert spec["code"]["images"]["inner"] == config["osw21_inner_image"] + old = build_job_spec(config=config, run_id="old", repo_root=tmp_path, + spec_overrides={"benchmark": "osworld2"}) + assert old["env"]["root"] == config["osw2_root"] + assert old["code"]["images"]["inner"] == config["inner_image"] + config["osw21_inner_image"] = "" + with pytest.raises(ValueError, match="release-pinned"): + build_job_spec(config=config, run_id="bad", repo_root=tmp_path, + spec_overrides={"benchmark": "osworld21"}) + + +@pytest.mark.parametrize("benchmark", ["osworld1", "osworld2", "osworld21"]) +def test_entry_uses_one_osworld_adapter(tmp_path, monkeypatch, benchmark): + import sys + + from tandem_eval.entry import build_env + from tandem_eval.osworld import OsWorldEnv + + monkeypatch.setattr(sys, "path", sys.path.copy()) + monkeypatch.setenv("OSWORLD_FILE_BASE_URL", "before") + env = build_env({"benchmark": benchmark, "output_root": str(tmp_path / "run"), + "env": {"root": str(tmp_path / "runtime"), "vm": str(tmp_path / "vm"), + "volume_size": 80, "asset_root": str(tmp_path / "assets")}}) + assert type(env) is OsWorldEnv + assert env.task_root == tmp_path / "runtime" + assert env.volume_size == 80 + + def test_endpoint_discovery_prefers_explicit_then_file(tmp_path, monkeypatch): from tandem_eval.entry import read_endpoint_file, resolve_selector_endpoint diff --git a/tests/eval/test_manifest.py b/tests/eval/test_manifest.py index bdf3294..3f9c85a 100644 --- a/tests/eval/test_manifest.py +++ b/tests/eval/test_manifest.py @@ -4,7 +4,9 @@ import json -from tandem_eval.manifest import TaskRef, load_index, resolve_manifest, write_resolved +import pytest + +from tandem_eval.manifest import OSWORLD21_TASK_IDS, TaskRef, load_index, resolve_manifest, write_resolved def test_load_index_domain_dict_format(): @@ -41,6 +43,24 @@ def test_resolve_manifest_mock_default(): assert len(refs) == 1 and refs[0].task_id == "mock-000" +def test_osworld21_subset_and_sharding(): + refs = resolve_manifest({"benchmark": "osworld21"}) + assert tuple(ref.task_id for ref in refs) == OSWORLD21_TASK_IDS + assert len(OSWORLD21_TASK_IDS) == len(set(OSWORLD21_TASK_IDS)) == 56 + assert all(not ref.domain for ref in refs) + assert not set(OSWORLD21_TASK_IDS) & {"026", "035", "041", "069", "087", "095"} + shards = [resolve_manifest({"benchmark": "osworld21", "manifest": { + "shard": {"index": index, "total": 4}}}) for index in range(1, 5)] + assert sorted(ref.task_id for shard in shards for ref in shard) == sorted(OSWORLD21_TASK_IDS) + + +def test_osworld21_explicit_subset_limit_and_unsupported_task(): + spec = {"benchmark": "osworld21", "manifest": {"tasks": ["054", "003"], "limit": 1}} + assert [ref.task_id for ref in resolve_manifest(spec)] == ["054"] + with pytest.raises(ValueError, match="outside the supported subset"): + resolve_manifest({"benchmark": "osworld21", "manifest": {"tasks": ["069"]}}) + + def test_resolve_manifest_from_index_root(): import tempfile from pathlib import Path diff --git a/tests/eval/test_osworld_env.py b/tests/eval/test_osworld_env.py index 9789807..88fb199 100644 --- a/tests/eval/test_osworld_env.py +++ b/tests/eval/test_osworld_env.py @@ -5,6 +5,7 @@ import json import struct from pathlib import Path +from types import ModuleType import pytest @@ -203,12 +204,20 @@ def test_setup_cache_seeding(tmp_path): del _shutil -def test_osw2_module_level_asset_urls_resolve_locally(tmp_path): +@pytest.mark.parametrize("benchmark", ["osworld2", "osworld21"]) +def test_osw2_module_level_asset_urls_resolve_locally(tmp_path, monkeypatch, benchmark): """OSW2 任务模块在 import 时（fetch_task 阶段）顶层调用 asset()； OSWORLD_FILE_BASE_URL 必须在那之前就绪（task_001 真机 404 根因）。""" import os import sys + monkeypatch.setattr(sys, "path", sys.path.copy()) + monkeypatch.setenv("OSWORLD_FILE_BASE_URL", "before") + for name in ("desktop_env", "desktop_env.file_source", "task_loader", + "task_class", "task_class.task_001"): + # 同时记录原先不存在的模块，确保临时导入也能在测试结束时撤销。 + monkeypatch.setitem(sys.modules, name, sys.modules.get(name)) + monkeypatch.delitem(sys.modules, name, raising=False) root = tmp_path / "osw2" (root / "desktop_env").mkdir(parents=True) (root / "desktop_env" / "__init__.py").write_text("", encoding="utf-8") @@ -235,10 +244,48 @@ def test_osw2_module_level_asset_urls_resolve_locally(tmp_path): encoding="utf-8") assets = tmp_path / "assets" - env = OsWorldEnv(benchmark="osworld2", task_root=root, vm_path=tmp_path / "vm", + env = OsWorldEnv(benchmark=benchmark, task_root=root, vm_path=tmp_path / "vm", run_root=tmp_path / "run", asset_root=assets, startup_wait_seconds=0.0, env_factory=lambda **kw: FakeDesktopEnv()) task = env.fetch_task("001") # 先于 reset：模块级 asset() 必须已指向本地 assert task["THUMB"] == str(assets / "task_001" / "profile.tar.gz") assert os.environ.get("OSWORLD_FILE_BASE_URL") == str(assets) - sys.path[:] = [p for p in sys.path if not p.startswith(str(tmp_path))] + + +def test_native_task_setup_and_evaluation_are_preserved(tmp_path, monkeypatch): + import sys + + monkeypatch.setattr(sys, "path", sys.path.copy()) + calls = [] + + class NativeTask(dict): + def setup(self, controller): + calls.append("setup") + + def evaluate(self, env): + calls.append("evaluate") + return {"score": 0.25} + + task = NativeTask(id="003", instruction="test", config=[]) + loader = ModuleType("task_loader") + loader.load_task_from_file = lambda path: task + monkeypatch.setitem(sys.modules, "task_loader", loader) + + class NativeDesktop(FakeDesktopEnv): + def reset(self, task_config): + super().reset(task_config) + task_config.setup(self.controller) + + def evaluate(self): + return self.task_config.evaluate(self) + + env = OsWorldEnv(benchmark="osworld21", task_root=tmp_path / "runtime", vm_path=tmp_path / "vm", + run_root=tmp_path / "run", startup_wait_seconds=0, env_factory=NativeDesktop) + loaded = env.fetch_task("003") + assert loaded is task + env.reset(loaded) + assert env.env.task_config is task + assert env.score(loaded) == {"score": 0.25} + assert calls == ["setup", "evaluate"] + env.close() + assert env.env is None