From 37fbd9c8d296acd3327236b4e98b18e977180d15 Mon Sep 17 00:00:00 2001 From: xxxyxun Date: Fri, 23 Oct 2026 23:02:16 +0800 Subject: [PATCH 1/2] feat(eval): support full OSWorld 2.1 environment integration --- docs/CLUSTER_RUNTIME.md | 16 ++++- docs/RESULT_DIR_CONTRACT.md | 4 ++ infra/config.example.json | 8 ++- infra/worker.sh | 10 ++- scripts/run_osw2_20x5.sh | 2 +- src/tandem_eval/config_schema.py | 6 ++ src/tandem_eval/entry.py | 2 + src/tandem_eval/manifest.py | 19 ++---- src/tandem_eval/rjob_spec.py | 40 ++++++++++-- tests/eval/test_manifest.py | 20 +++--- tests/eval/test_osworld21_services.py | 93 +++++++++++++++++++++++++++ 11 files changed, 180 insertions(+), 40 deletions(-) create mode 100644 tests/eval/test_osworld21_services.py diff --git a/docs/CLUSTER_RUNTIME.md b/docs/CLUSTER_RUNTIME.md index d33cccc..fe6650b 100644 --- a/docs/CLUSTER_RUNTIME.md +++ b/docs/CLUSTER_RUNTIME.md @@ -65,6 +65,9 @@ worker 入口、共享资源与已知坑。 | `eval.benchmarks.osworld21.inner_image` | OSW2.1 运行镜像，必须发布固定（`@sha256:…`，禁 `latest`）；留空时跑 osworld21 会明确报错 | | `eval.benchmarks.osworld21.proxy` | OSW2.1 用的代理（覆盖 worker.sh 的默认代理） | | `eval.benchmarks.osworld21.volume_size` | 传给官方接口的**卷参数**，不等于根盘容量（实测根盘约 49GB、可用约 21GB） | +| `eval.benchmarks.osworld21.no_proxy` | 可选，逗号分隔的直连API主机；只追加到OSW2.1 worker的代理绕过列表 | +| `eval.benchmarks.osworld21.gitlab.private_token` | 可选，受信任的setup使用的管理token；公共网站配置不自动读取私有凭据 | +| `eval.benchmarks.osworld21.judge` / `.user_simulator` | 可选，各自配置 `provider` / `base_url` / `api_key` / `model`；judge还可指定 `reasoning_effort` / `max_tokens`，任务内显式参数优先 | | `eval.registry` | docker registry auth（内联传给 worker 写 `~/.docker/config.json`） | ### log —— 结果落盘 @@ -219,9 +222,16 @@ OSW2.1 走**独立环境**（独立 runtime / VM / 资产 / venv / env.sh）， `desktop_env/providers/docker/provider.py`。 - **依赖自检**：OSW2.1 的独立 venv 按官方 `uv.lock` 安装，**没有 easyocr**， worker 只检查 Pillow；旧环境仍检查 `easyocr + PIL`。 -- **题目子集**：`tandem_eval/manifest.py` 的 `OSWORLD21_TASK_IDS`（56 题）取自 - `osworld21/validation/coverage.json` 中 `official_vm.status == "passed"` 的题； - 该覆盖表由环境侧持续重生成，**子集要定期重新核对**。 +- **题目清单**：`tandem_eval/manifest.py` 的 `OSWORLD21_TASK_IDS` 覆盖官方001–108题。 + 清单不等于环境验收报告；API、VM或评分异常仍记为 `failed_unscored`。 +- **共享环境**：通用VM为 `vm/unified-20261022-complete.qcow2`，经 `prepared/runtime` + 接入环境准备适配。依赖缓存、网站DNS/证书和控制器传输适配在共享环境侧维护， + 不改变Tandem规划、候选、验证或评分逻辑。worker只读基盘与冻结venv，每题使用隔离VM。 +- **网站与权限**：从环境侧 `services/web/endpoint.json` 与 `services/gitlab/endpoint.json` + 发现内网地址，不依赖开发机的localhost。使用者需有共享盘、内网和镜像仓库权限； + 026、041另需获授权的GitLab管理token，模型API使用各自获授权的凭据。 + 凭据仅置于已gitignore的 `infra/config.json` 和权限600的私有job spec，不提交Git。 + 替代judge/模拟器的身份另记于 `code_version.json.resources`，不混作原生GPT配置。 - **卷参数**：`volume_size` 只是传给官方接口的卷参数，**不代表根盘真有这么大** （实测根盘约 49GB、可用约 21GB）。 diff --git a/docs/RESULT_DIR_CONTRACT.md b/docs/RESULT_DIR_CONTRACT.md index 40b01a5..b975157 100644 --- a/docs/RESULT_DIR_CONTRACT.md +++ b/docs/RESULT_DIR_CONTRACT.md @@ -18,6 +18,10 @@ └── / # episode 目录（有 domain 时为 -） ``` +OSWorld 2.1 显式配置替代 judge/用户模拟器时，`code_version.json.resources` +分别记录 `judge` / `user_simulator` 的 `{provider, model, configuration: "alternative"}`。 +凭据仅存在私有 job spec；结果中的 `run_spec.secrets` 保持全部脱敏。 + ## episode 目录（自包含：不依赖共享目录与旧仓库即可复盘） | 文件 | 内容 | 写入时机 | diff --git a/infra/config.example.json b/infra/config.example.json index 085da89..dcde4d8 100644 --- a/infra/config.example.json +++ b/infra/config.example.json @@ -89,12 +89,16 @@ }, "osworld21": { "root": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/prepared/runtime", - "vm": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/vm/osworld-v2-ubuntu-x86.qcow2", + "vm": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/vm/unified-20261022-complete.qcow2", "asset_root": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/assets", "shared_env": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/prepared/env.sh", "inner_image": "", "proxy": "http://100.100.135.73:3128", - "volume_size": 64 + "no_proxy": "", + "volume_size": 64, + "gitlab": {"private_token": ""}, + "judge": {"provider": "openai_compatible", "base_url": "", "api_key": "", "model": ""}, + "user_simulator": {"provider": "openai_compatible", "base_url": "", "api_key": "", "model": ""} } }, "registry": { diff --git a/infra/worker.sh b/infra/worker.sh index 10d627d..6121e9b 100755 --- a/infra/worker.sh +++ b/infra/worker.sh @@ -48,6 +48,9 @@ if [ ! -f "$SHARED_ENV" ]; then fi # shellcheck disable=SC1090 source "$SHARED_ENV" +if [ "$BENCHMARK" = "osworld21" ]; then + python3 "$OSWORLD21_ROOT/setup/shared_services.py" hosts +fi mkdir -p "$HOME/.docker" python3 -c ' @@ -66,7 +69,12 @@ INNER_IMAGE="$(spec_get code.images.inner)" PROXY="$(spec_get env.proxy)" PROXY="${PROXY:-http://httpproxy-headless.kubebrain.svc.pjlab.local:3128}" export HTTP_PROXY="$PROXY" HTTPS_PROXY="$PROXY" http_proxy="$PROXY" https_proxy="$PROXY" -export NO_PROXY='10.0.0.0/8,100.96.0.0/12,localhost,127.0.0.1,.pjlab.org.cn' +if [ "$BENCHMARK" = "osworld21" ]; then + OSWORLD21_NO_PROXY="$(spec_get env.no_proxy)" + export NO_PROXY="10.0.0.0/8,100.96.0.0/12,localhost,127.0.0.1,.pjlab.org.cn${NO_PROXY:+,$NO_PROXY}${OSWORLD21_NO_PROXY:+,$OSWORLD21_NO_PROXY}" +else + export NO_PROXY='10.0.0.0/8,100.96.0.0/12,localhost,127.0.0.1,.pjlab.org.cn' +fi export no_proxy="$NO_PROXY" DOCKER_ROOT="$(mktemp -d /tmp/tandem-docker.XXXXXX)" diff --git a/scripts/run_osw2_20x5.sh b/scripts/run_osw2_20x5.sh index 093b38d..f79d239 100755 --- a/scripts/run_osw2_20x5.sh +++ b/scripts/run_osw2_20x5.sh @@ -9,7 +9,7 @@ # BENCHMARK（基准，默认 osworld2）/ TASKS（逗号分隔题号，空 = 走清单）/ ROUTE_POLICY（路由，空 = 用 config） # SLOW_MODEL（慢模型 API 名，默认用 config 的 default）/ FAST_MODEL（快模型 API 名） # PREFIX 建议带上模型组合（如 osw2-kimi-kev），demo 按 run 对比时一眼能看出谁在跑 -# OSW2.1：BENCHMARK=osworld21 LIMIT=56 SHARDS=4 PREFIX=osw21-56 bash scripts/run_osw2_20x5.sh +# OSW2.1：BENCHMARK=osworld21 LIMIT=108 SHARDS=4 PREFIX=osw21-108 bash scripts/run_osw2_20x5.sh # 单题：再设置 TASKS=004 LIMIT=1 SHARDS=1；ROUTE_POLICY=gui 可限定 GUI 路由。 set -u diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py index 852d340..f5cca57 100644 --- a/src/tandem_eval/config_schema.py +++ b/src/tandem_eval/config_schema.py @@ -194,6 +194,12 @@ def _validate_osworld21(config: dict) -> list[str]: volume_size = section.get("volume_size") if volume_size is not None and (not isinstance(volume_size, int) or volume_size <= 0): problems.append("eval.benchmarks.osworld21.volume_size（应为正整数）") + for role in ("judge", "user_simulator"): + api = section.get(role, {}) + if api.get("api_key"): + for key in ("provider", "base_url", "model"): + if _is_missing(api.get(key)): + problems.append(f"eval.benchmarks.osworld21.{role}.{key}") return problems diff --git a/src/tandem_eval/entry.py b/src/tandem_eval/entry.py index 7577c0b..5edbe6c 100644 --- a/src/tandem_eval/entry.py +++ b/src/tandem_eval/entry.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +import os import subprocess from pathlib import Path @@ -217,6 +218,7 @@ def _mask_secrets(spec: dict) -> dict: def run_from_spec(spec: dict) -> list[dict]: + os.environ.update(spec.get("secrets", {}).get("environment", {})) run_root = Path(spec["output_root"]) run_root.mkdir(parents=True, exist_ok=True) repo_root = Path(spec.get("repo_root", ".")) diff --git a/src/tandem_eval/manifest.py b/src/tandem_eval/manifest.py index 57a6aae..4bb66d7 100644 --- a/src/tandem_eval/manifest.py +++ b/src/tandem_eval/manifest.py @@ -1,4 +1,4 @@ -"""任务清单：OSW1/OSW2 使用共享索引，OSW2.1 使用 56 题清单，也支持显式题号或清单文件。""" +"""任务清单：OSW1/OSW2 使用共享索引，OSW2.1 包含 108 题，支持显式题号与分片。""" from __future__ import annotations @@ -6,19 +6,8 @@ from dataclasses import dataclass from pathlib import Path -#: OSW2.1 子集题号（56 题）。 -#: -#: 出处：共享环境 `osworld21/validation/coverage.json` 中 `official_vm.status == "passed"` -#: 的题（2026-10-22 快照）。该覆盖表由 `osworld21/setup/summarize_environment.py` 持续 -#: 重生成，本清单是它的一次快照——覆盖表更新后必须重新核对，否则会与真实环境脱节。 -#: 快照时官方 passed 共 86 道，其余 30 道的排除口径尚未记录（待补）；本清单当前是 -#: "已验证通过"的保守子集，不含 failed / not_tested 的题。 -OSWORLD21_TASK_IDS = tuple(( - "001 002 003 004 006 009 010 011 012 014 017 022 023 024 " - "025 027 029 030 032 034 037 040 042 044 045 049 051 053 " - "054 055 059 061 062 063 066 067 071 074 075 076 078 080 " - "083 084 085 086 088 089 090 092 093 094 096 097 099 108" -).split()) +#: 官方完整清单；列入清单不等于已通过环境验收或完成解题。 +OSWORLD21_TASK_IDS = tuple(f"{number:03d}" for number in range(1, 109)) @dataclass(frozen=True) @@ -83,7 +72,7 @@ def resolve_manifest(spec: dict) -> list[TaskRef]: if benchmark == "osworld21": unsupported = [ref.task_id for ref in refs if ref.task_id not in OSWORLD21_TASK_IDS] if unsupported: - raise ValueError(f"OSWorld 2.1 tasks outside the supported subset: {unsupported}") + raise ValueError(f"OSWorld 2.1 task IDs outside 001–108: {unsupported}") limit = manifest.get("limit") if isinstance(limit, int) and limit > 0: refs = refs[:limit] diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py index 59b29a0..713bb95 100644 --- a/src/tandem_eval/rjob_spec.py +++ b/src/tandem_eval/rjob_spec.py @@ -16,6 +16,7 @@ from __future__ import annotations import json +import os import subprocess from pathlib import Path @@ -53,7 +54,8 @@ def _env_section(benchmark: str, config: dict) -> dict: section = benchmarks["osworld21"] return {"root": section["root"], "vm": section["vm"], "asset_root": section["asset_root"], "shared_env": section["shared_env"], - "proxy": section.get("proxy", ""), "volume_size": section.get("volume_size"), + "proxy": section.get("proxy", ""), "no_proxy": section.get("no_proxy", ""), + "volume_size": section.get("volume_size"), "startup_wait_seconds": 30} return {} @@ -62,7 +64,7 @@ def _secrets_section(benchmark: str, config: dict, slow_api: dict, fast_api: dic """凭据段：慢模型三件套 + 快模型地址 + registry auth（落盘时整体脱敏）。""" if benchmark not in {"osworld1", "osworld2", "osworld21"}: return {} - return { + secrets = { "slow_model": {key: slow_api[key] for key in ("base_url", "api_key", "model")}, # 快模型部署在闲时 GPU，ip/端口随重新部署变化：endpoint 为空时由 worker # 从 endpoint_file（部署后写入的 {"base_url": ...}）自动发现 @@ -72,6 +74,25 @@ def _secrets_section(benchmark: str, config: dict, slow_api: dict, fast_api: dic }, "registry": config["eval"].get("registry"), } + if benchmark == "osworld21": + section = config["eval"]["benchmarks"][benchmark] + environment = {} + if token := section.get("gitlab", {}).get("private_token"): + environment["GITLAB_PRIVATE_TOKEN"] = token + for role, prefix, model_key in (("judge", "OSWORLD_EVAL_MODEL_", "NAME"), + ("user_simulator", "OSWORLD_USER_SIM_", "MODEL")): + api = section.get(role, {}) + if api.get("api_key"): + environment.update({prefix + suffix: api[key] for key, suffix in ( + ("provider", "PROVIDER"), ("base_url", "BASE_URL"), + ("api_key", "API_KEY"), ("model", model_key))}) + if role == "judge": + for key, suffix in (("reasoning_effort", "REASONING_EFFORT"), + ("max_tokens", "MAX_OUTPUT_TOKENS")): + if key in api: + environment[prefix + suffix] = str(api[key]) + secrets["environment"] = environment + return secrets def _harness_config_section(config: dict, slow_api: dict, fast_api: dict) -> dict: @@ -166,11 +187,18 @@ def build_job_spec(*, config: dict, run_id: str, repo_root: Path, }, **overrides, } + if benchmark == "osworld21": + section = config["eval"]["benchmarks"][benchmark] + spec["code"]["resources"] = {role: { + "provider": section[role]["provider"], "model": section[role]["model"], + "configuration": "alternative"} for role in ("judge", "user_simulator") + if section.get(role, {}).get("api_key")} spec_dir = Path(config["eval"]["deploy_root"]) / "job_specs" - spec_dir.mkdir(parents=True, exist_ok=True) - (spec_dir / f"{run_id}.json").write_text( - json.dumps(spec, ensure_ascii=False, indent=2), encoding="utf-8" - ) + spec_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + descriptor = os.open(spec_dir / f"{run_id}.json", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + os.fchmod(handle.fileno(), 0o600) + json.dump(spec, handle, ensure_ascii=False, indent=2) return spec diff --git a/tests/eval/test_manifest.py b/tests/eval/test_manifest.py index 3a26654..a093a7a 100644 --- a/tests/eval/test_manifest.py +++ b/tests/eval/test_manifest.py @@ -54,22 +54,22 @@ def test_resolve_manifest_mock_default(): assert len(refs) == 1 and refs[0].task_id == "mock-000" -def test_osworld21_subset_and_sharding(): +def test_osworld21_full_list_and_sharding(): refs = resolve_manifest({"benchmark": "osworld21"}) assert tuple(ref.task_id for ref in refs) == OSWORLD21_TASK_IDS - assert len(OSWORLD21_TASK_IDS) == len(set(OSWORLD21_TASK_IDS)) == 56 + assert len(OSWORLD21_TASK_IDS) == len(set(OSWORLD21_TASK_IDS)) == 108 assert all(not ref.domain for ref in refs) - assert not set(OSWORLD21_TASK_IDS) & {"026", "035", "041", "069", "087", "095"} + assert {"026", "035", "041", "069", "087", "095"} <= set(OSWORLD21_TASK_IDS) shards = [resolve_manifest({"benchmark": "osworld21", "manifest": { "shard": {"index": index, "total": 4}}}) for index in range(1, 5)] assert sorted(ref.task_id for shard in shards for ref in shard) == sorted(OSWORLD21_TASK_IDS) -def test_osworld21_explicit_subset_limit_and_unsupported_task(): +def test_osworld21_explicit_tasks_limit_and_unsupported_task(): spec = {"benchmark": "osworld21", "manifest": {"tasks": ["054", "003"], "limit": 1}} assert [ref.task_id for ref in resolve_manifest(spec)] == ["054"] - with pytest.raises(ValueError, match="outside the supported subset"): - resolve_manifest({"benchmark": "osworld21", "manifest": {"tasks": ["069"]}}) + with pytest.raises(ValueError, match="outside 001–108"): + resolve_manifest({"benchmark": "osworld21", "manifest": {"tasks": ["109"]}}) def test_resolve_manifest_from_index_root(): @@ -128,12 +128,8 @@ def test_index_root_derived_from_env_section(tmp_path): @pytest.mark.skipif(_OSW21_TASK_CLASS is None, reason="OSW2.1 runtime 未挂载（worker 内才有 canonical 视角）") -def test_osworld21_subset_tasks_exist_in_runtime(): - """子集里的每个题号都必须在 runtime 的 task_class 里有对应任务文件。 - - 子集来自环境侧持续重生成的覆盖表快照（见 manifest.OSWORLD21_TASK_IDS 的说明）， - 环境更新后这道题会第一时间报出"清单里有 runtime 没有的题"。 - """ +def test_osworld21_tasks_exist_in_runtime(): + """完整清单里的每个题号都必须在 runtime 的 task_class 里有任务文件。""" missing = [tid for tid in OSWORLD21_TASK_IDS if not (_OSW21_TASK_CLASS / f"task_{tid}.py").is_file()] assert missing == [], f"子集里的题号在 runtime 中不存在: {missing}" diff --git a/tests/eval/test_osworld21_services.py b/tests/eval/test_osworld21_services.py new file mode 100644 index 0000000..88e28d8 --- /dev/null +++ b/tests/eval/test_osworld21_services.py @@ -0,0 +1,93 @@ +"""OSW2.1 私有凭据注入与公共服务配置，不调用模型或 VM。""" + +from __future__ import annotations + +import json +import stat +from pathlib import Path + +import pytest + +from tandem_eval.config_schema import validate +from tandem_eval.entry import _mask_secrets, run_from_spec +from tandem_eval.rjob_spec import build_job_spec + +from .rjob_fakes import CONFIG, with_paths + + +def test_osworld21_credentials_stay_private_and_profiles_are_attributable(tmp_path): + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) + section = config["eval"]["benchmarks"]["osworld21"] + section["gitlab"] = {"private_token": "private-gitlab-test-token"} + for role in ("judge", "user_simulator"): + section[role] = {"provider": "openai_compatible", "model": "alternative-test-model", + "base_url": "https://example.invalid/v1", "api_key": role + "-private-key"} + section["judge"].update(reasoning_effort="low", max_tokens=8192) + spec = build_job_spec(config=config, run_id="services", repo_root=tmp_path, + spec_overrides={"benchmark": "osworld21"}) + environment = spec["secrets"]["environment"] + assert environment["GITLAB_PRIVATE_TOKEN"] == section["gitlab"]["private_token"] + assert environment["OSWORLD_EVAL_MODEL_NAME"] == "alternative-test-model" + assert environment["OSWORLD_USER_SIM_MODEL"] == "alternative-test-model" + assert environment["OSWORLD_USER_SIM_API_KEY"] == "user_simulator-private-key" + assert environment["OSWORLD_EVAL_MODEL_REASONING_EFFORT"] == "low" + assert environment["OSWORLD_EVAL_MODEL_MAX_OUTPUT_TOKENS"] == "8192" + assert spec["code"]["resources"]["judge"]["configuration"] == "alternative" + recorded = json.dumps(_mask_secrets(spec)) + for role in ("judge", "user_simulator"): + assert section[role]["api_key"] not in recorded + assert section["gitlab"]["private_token"] not in recorded + path = tmp_path / "deploy/job_specs/services.json" + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + assert json.loads(path.read_text()) == spec + + +def test_osworld21_missing_private_configuration_does_not_copy_planner_key(tmp_path): + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) + spec = build_job_spec(config=config, run_id="public", repo_root=tmp_path, + spec_overrides={"benchmark": "osworld21"}) + assert spec["secrets"]["environment"] == {} + assert spec["code"]["resources"] == {} + + +def test_osworld21_direct_api_hosts_do_not_change_older_benchmarks(tmp_path): + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) + config["eval"]["benchmarks"]["osworld21"]["no_proxy"] = "api.example.invalid" + spec = build_job_spec(config=config, run_id="direct", repo_root=tmp_path, + spec_overrides={"benchmark": "osworld21"}) + assert spec["env"]["no_proxy"] == "api.example.invalid" + old = build_job_spec(config=config, run_id="old", repo_root=tmp_path, + spec_overrides={"benchmark": "osworld2"}) + assert "no_proxy" not in old["env"] + + +@pytest.mark.parametrize("role", ["judge", "user_simulator"]) +def test_configured_evaluation_api_requires_its_own_model_and_endpoint(role): + path = Path(__file__).resolve().parents[2] / "infra/config.example.json" + config = json.loads(path.read_text()) + config["eval"]["benchmarks"]["osworld21"][role] = {"api_key": "private-test-key"} + problems = validate(config) + assert f"eval.benchmarks.osworld21.{role}.base_url" in problems + assert f"eval.benchmarks.osworld21.{role}.model" in problems + + +def test_credentials_are_injected_before_environment_loading(tmp_path, monkeypatch): + from tandem_eval import entry + + class EnvironmentReached(Exception): + pass + + monkeypatch.delenv("GITLAB_PRIVATE_TOKEN", raising=False) + + def build_env(spec): + import os + + assert os.environ["GITLAB_PRIVATE_TOKEN"] == "private-test-token" + raise EnvironmentReached + + monkeypatch.setattr(entry, "build_env", build_env) + with pytest.raises(EnvironmentReached): + run_from_spec({"benchmark": "osworld21", "output_root": str(tmp_path), + "secrets": {"environment": {"GITLAB_PRIVATE_TOKEN": "private-test-token"}}}) + monkeypatch.delenv("GITLAB_PRIVATE_TOKEN") + assert "private-test-token" not in (tmp_path / "code_version.json").read_text() From 0e3f2ab11e255e8143c1c2b5774b1f1c45409802 Mon Sep 17 00:00:00 2001 From: xxxyxun Date: Fri, 23 Oct 2026 23:16:52 +0800 Subject: [PATCH 2/2] feat(eval): load shared GitLab test token from file --- docs/CLUSTER_RUNTIME.md | 10 +++++++--- infra/config.example.json | 5 ++++- src/tandem_eval/config_schema.py | 1 + src/tandem_eval/rjob_spec.py | 6 +++++- tests/eval/test_osworld21_services.py | 25 +++++++++++++++++++++++++ 5 files changed, 42 insertions(+), 5 deletions(-) diff --git a/docs/CLUSTER_RUNTIME.md b/docs/CLUSTER_RUNTIME.md index fe6650b..e44f84d 100644 --- a/docs/CLUSTER_RUNTIME.md +++ b/docs/CLUSTER_RUNTIME.md @@ -66,7 +66,8 @@ worker 入口、共享资源与已知坑。 | `eval.benchmarks.osworld21.proxy` | OSW2.1 用的代理（覆盖 worker.sh 的默认代理） | | `eval.benchmarks.osworld21.volume_size` | 传给官方接口的**卷参数**，不等于根盘容量（实测根盘约 49GB、可用约 21GB） | | `eval.benchmarks.osworld21.no_proxy` | 可选，逗号分隔的直连API主机；只追加到OSW2.1 worker的代理绕过列表 | -| `eval.benchmarks.osworld21.gitlab.private_token` | 可选，受信任的setup使用的管理token；公共网站配置不自动读取私有凭据 | +| `eval.benchmarks.osworld21.gitlab.private_token` | 可选，内联GitLab token；非空时优先于 `private_token_file` | +| `eval.benchmarks.osworld21.gitlab.private_token_file` | 可选，共享测试token文件的绝对路径；提交任务时读取，经私有job spec传给setup/evaluator | | `eval.benchmarks.osworld21.judge` / `.user_simulator` | 可选，各自配置 `provider` / `base_url` / `api_key` / `model`；judge还可指定 `reasoning_effort` / `max_tokens`，任务内显式参数优先 | | `eval.registry` | docker registry auth（内联传给 worker 写 `~/.docker/config.json`） | @@ -229,8 +230,11 @@ OSW2.1 走**独立环境**（独立 runtime / VM / 资产 / venv / env.sh）， 不改变Tandem规划、候选、验证或评分逻辑。worker只读基盘与冻结venv，每题使用隔离VM。 - **网站与权限**：从环境侧 `services/web/endpoint.json` 与 `services/gitlab/endpoint.json` 发现内网地址，不依赖开发机的localhost。使用者需有共享盘、内网和镜像仓库权限； - 026、041另需获授权的GitLab管理token，模型API使用各自获授权的凭据。 - 凭据仅置于已gitignore的 `infra/config.json` 和权限600的私有job spec，不提交Git。 + 026、041通过配置模板中的 `gitlab.private_token_file` 读取共享测试管理员token。 + 该文件位于共享环境 `shared_credentials/gitlab-test-admin.token`，按所有者授权以644权限供集群用户读取； + 仅用于这套独立测试GitLab，不包含GitHub、模型API或镜像仓库凭据。token轮换时由环境维护者更新该文件。 + 模型API及其他凭据仍使用各自获授权的配置，置于已gitignore的 `infra/config.json`； + 传给worker的job spec保持600权限，所有token均不提交Git或写入公开结果。 替代judge/模拟器的身份另记于 `code_version.json.resources`，不混作原生GPT配置。 - **卷参数**：`volume_size` 只是传给官方接口的卷参数，**不代表根盘真有这么大** （实测根盘约 49GB、可用约 21GB）。 diff --git a/infra/config.example.json b/infra/config.example.json index dcde4d8..33c965d 100644 --- a/infra/config.example.json +++ b/infra/config.example.json @@ -96,7 +96,10 @@ "proxy": "http://100.100.135.73:3128", "no_proxy": "", "volume_size": 64, - "gitlab": {"private_token": ""}, + "gitlab": { + "private_token": "", + "private_token_file": "/mnt/shared-storage-gpfs2/evobox-share-gpfs2/xuyuanxun/shared_resources/sa_cua/osworld21/shared_credentials/gitlab-test-admin.token" + }, "judge": {"provider": "openai_compatible", "base_url": "", "api_key": "", "model": ""}, "user_simulator": {"provider": "openai_compatible", "base_url": "", "api_key": "", "model": ""} } diff --git a/src/tandem_eval/config_schema.py b/src/tandem_eval/config_schema.py index f5cca57..0dab909 100644 --- a/src/tandem_eval/config_schema.py +++ b/src/tandem_eval/config_schema.py @@ -63,6 +63,7 @@ "eval.benchmarks.osworld21.vm", "eval.benchmarks.osworld21.asset_root", "eval.benchmarks.osworld21.shared_env", + "eval.benchmarks.osworld21.gitlab.private_token_file", "harness.fast_model.image_stage.write", "harness.fast_model.image_stage.read", ) diff --git a/src/tandem_eval/rjob_spec.py b/src/tandem_eval/rjob_spec.py index 713bb95..3b81493 100644 --- a/src/tandem_eval/rjob_spec.py +++ b/src/tandem_eval/rjob_spec.py @@ -77,7 +77,11 @@ def _secrets_section(benchmark: str, config: dict, slow_api: dict, fast_api: dic if benchmark == "osworld21": section = config["eval"]["benchmarks"][benchmark] environment = {} - if token := section.get("gitlab", {}).get("private_token"): + gitlab = section.get("gitlab", {}) + token = gitlab.get("private_token") + if not token and gitlab.get("private_token_file"): + token = Path(gitlab["private_token_file"]).read_text(encoding="utf-8").strip() + if token: environment["GITLAB_PRIVATE_TOKEN"] = token for role, prefix, model_key in (("judge", "OSWORLD_EVAL_MODEL_", "NAME"), ("user_simulator", "OSWORLD_USER_SIM_", "MODEL")): diff --git a/tests/eval/test_osworld21_services.py b/tests/eval/test_osworld21_services.py index 88e28d8..3c343dd 100644 --- a/tests/eval/test_osworld21_services.py +++ b/tests/eval/test_osworld21_services.py @@ -50,6 +50,31 @@ def test_osworld21_missing_private_configuration_does_not_copy_planner_key(tmp_p assert spec["code"]["resources"] == {} +@pytest.mark.parametrize("inline_token", ["", "inline-test-token"]) +def test_osworld21_shared_gitlab_token_file(tmp_path, inline_token): + token_path = tmp_path / "gitlab.token" + token_path.write_text("shared-test-token\n", encoding="utf-8") + config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) + config["eval"]["benchmarks"]["osworld21"]["gitlab"] = { + "private_token": inline_token, "private_token_file": str(token_path), + } + spec = build_job_spec(config=config, run_id="shared-token", repo_root=tmp_path, + spec_overrides={"benchmark": "osworld21"}) + assert spec["secrets"]["environment"]["GITLAB_PRIVATE_TOKEN"] == (inline_token or "shared-test-token") + recorded = json.dumps(_mask_secrets(spec)) + assert "shared-test-token" not in recorded + assert "inline-test-token" not in recorded + path = tmp_path / "deploy/job_specs/shared-token.json" + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + + +def test_shared_gitlab_token_file_requires_absolute_path(): + path = Path(__file__).resolve().parents[2] / "infra/config.example.json" + config = json.loads(path.read_text()) + config["eval"]["benchmarks"]["osworld21"]["gitlab"]["private_token_file"] = "relative.token" + assert "eval.benchmarks.osworld21.gitlab.private_token_file（必须是绝对路径）" in validate(config) + + def test_osworld21_direct_api_hosts_do_not_change_older_benchmarks(tmp_path): config = with_paths(CONFIG, deploy_root=str(tmp_path / "deploy")) config["eval"]["benchmarks"]["osworld21"]["no_proxy"] = "api.example.invalid"