From f811650217f3a87ad30950d8b2e20e3bc7774dc2 Mon Sep 17 00:00:00 2001 From: Birfy <15378933+Birfy@users.noreply.github.com> Date: Mon, 2 Nov 2026 19:56:48 +0800 Subject: [PATCH] [Feat] Offline step-level browser benchmark on myjev-browser-decisions (#109) --- CHANGELOG.md | 11 + benchmarks/README.md | 4 + benchmarks/legs/browser_steps.py | 480 +++++++++++++++++++++++++ docs/results/browser-steps/README.md | 100 ++++++ tests/benchmarks/test_browser_steps.py | 269 ++++++++++++++ 5 files changed, 864 insertions(+) create mode 100644 benchmarks/legs/browser_steps.py create mode 100644 docs/results/browser-steps/README.md create mode 100644 tests/benchmarks/test_browser_steps.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fae851b..7c83290 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver ## Unreleased +### Added + +- `python -m benchmarks.legs.browser_steps`: offline step-level scoring of browser decision backends on + [myjev-browser-decisions](https://huggingface.co/datasets/Mlchemist/myjev-browser-decisions) (pinned revision, + hash-checked at download, never vendored; CC BY-NC-SA 4.0). `predict` sends each step through `DecisionModel` + (any text backend), one request at a time, no retries, failures scored wrong; `score` reports operation and + element accuracy, step success, Brier/ECE and latency with denominators, on the full split and the Jev-answered + subset; `thresholds` turns a calibration run into a deferral threshold table. `--shape dataset` re-sends + jev-ultrafast's body unchanged, `--shape ours` swaps in our operation text and rules. Only a random floor is + reported so far. See [docs/results/browser-steps](docs/results/browser-steps/README.md) (#109). + ## 0.3.0 - 2026-10-30 ### Added diff --git a/benchmarks/README.md b/benchmarks/README.md index c0f83c8..42dc99e 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -11,6 +11,10 @@ The [accuracy leaderboard](accuracy-leaderboard.md) ranks that frozen run and defines how to add comparable results. Request latency has a separate [System1-Omni leaderboard](https://github.com/ThinkFlowLab/system1-omni/pull/131). +Browser decision backends are scored step by step, offline, on myjev-browser-decisions +with `python -m benchmarks.legs.browser_steps`; see +[docs/results/browser-steps](../docs/results/browser-steps/README.md). + ## Run Use the repository's development environment and a running service. Clone diff --git a/benchmarks/legs/browser_steps.py b/benchmarks/legs/browser_steps.py new file mode 100644 index 0000000..0ab33a3 --- /dev/null +++ b/benchmarks/legs/browser_steps.py @@ -0,0 +1,480 @@ +"""Score browser decision backends step by step on myjev-browser-decisions (issue #109). + +Each record of the dataset is one browser step in the request format jev-ultrafast sends to Jev: the page state, +the goal, the last actions and one Choice question per head (``operation`` and ``_target``), with gold keys. +``predict`` sends every step through the repository's ``DecisionModel`` layer, so any text backend can answer; +``score`` re-scores saved evidence; ``thresholds`` turns a calibration run into a deferral threshold table. + +The dataset is CC BY-NC-SA 4.0 and includes Mind2Web, whose authors ask that it not be republished: rows are +downloaded at run time from a pinned revision and checked against pinned hashes, never vendored. +""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import math +import statistics +import subprocess +import sys +import time +from collections.abc import Iterable, Iterator +from pathlib import Path +from typing import Any + +from benchmarks.evidence import outside_repo + +DATASET = "Mlchemist/myjev-browser-decisions" +REVISION = "8c0c5d532458f7ba25c9684b267694848e815d5a" +# LFS files carry their SHA-256; the small id list is a plain git blob, checked by its git object id. +SPLITS = { + "benchmark": ("data/benchmark.jsonl", "sha256", "f95b894c4161ad7c059624916e6f7e4db9460bbd16c3b16a485481534436bd37"), + "calibration": ( + "data/calibration.jsonl", + "sha256", + "c470db23180974d317ca4f511a43db408c275062ae622f9f6e6f71a6fd6bc910", + ), + "jev_answered_ids": ("data/benchmark_jev_answered_ids.txt", "git-blob", "fce3e76890c50848f6795c17feee2e69653d42ec"), +} +TARGET_OPERATIONS = ("CLICK", "TYPE_TEXT", "SELECT") +SHAPES = ("ours", "dataset") +ECE_BINS = 10 +THRESHOLDS = tuple(round(0.05 * step, 2) for step in range(20)) + + +# -- dataset files ----------------------------------------------------------------------------------------------- + + +def file_digest(data: bytes, kind: str) -> str: + if kind == "sha256": + return hashlib.sha256(data).hexdigest() + if kind == "git-blob": + return hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest() + raise ValueError(f"unknown digest kind {kind!r}") + + +def verify(path: Path, name: str) -> str: + """The pinned digest of ``name`` when ``path`` matches it; ``ValueError`` otherwise.""" + _, kind, expected = SPLITS[name] + actual = file_digest(path.read_bytes(), kind) + if actual != expected: + raise ValueError(f"{path} is not {DATASET}@{REVISION[:12]} {name}: {kind} {actual} != {expected}") + return expected + + +def url_for(name: str) -> str: + return f"https://huggingface.co/datasets/{DATASET}/resolve/{REVISION}/{SPLITS[name][0]}" + + +def fetch(names: Iterable[str], out: Path, *, transport: Any = None) -> dict[str, Path]: + """Download the pinned files into ``out`` (outside the repository), verifying each before it is kept.""" + import httpx + + out.mkdir(parents=True, exist_ok=True) + saved: dict[str, Path] = {} + with httpx.Client(timeout=600, follow_redirects=True, transport=transport) as client: + for name in names: + target = out / Path(SPLITS[name][0]).name + if not target.exists(): + response = client.get(url_for(name)) + response.raise_for_status() + partial = target.with_suffix(target.suffix + ".part") + partial.write_bytes(response.content) + try: + verify(partial, name) + except ValueError: + partial.unlink() + raise + partial.rename(target) + verify(target, name) + saved[name] = target + return saved + + +def read_records(path: Path) -> Iterator[dict[str, Any]]: + with path.open(encoding="utf-8") as stream: + for line in stream: + if line.strip(): + yield json.loads(line) + + +def read_ids(path: Path | None) -> set[str] | None: + if path is None: + return None + return {line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip()} + + +# -- request shapes ------------------------------------------------------------------------------------------------ + + +def to_questions(record: dict[str, Any], shape: str = "ours") -> tuple[Any, dict[str, Any]]: + """One record's observation and questions for ``DecisionModel.decide_many``. + + ``dataset`` keeps jev-ultrafast's text verbatim, so ``JevModel`` re-sends the dataset's body unchanged. + ``ours`` keeps the state, the question names and the element rows (they already have the keys our action space + sends) and replaces the operation descriptions and rules with the ones ``BrowserDecisionModel`` sends, so the + score measures the model on the questions our agent asks. See docs/results/browser-steps.md for the + differences that remain. + """ + from s1a.browser import prompts + from s1a.browser.action_space import _OPERATION_LABELS + from s1a.decision_models import ChoiceQuestion, Observation + + if shape not in SHAPES: + raise ValueError(f"shape must be one of {SHAPES}") + request = record["request"] + questions: dict[str, Any] = {} + for name, question in request["questions"].items(): + if question.get("type") != "choice": + raise ValueError(f"{record_id(record)}: question {name!r} is not a choice") + instructions = question.get("instructions") or {} + options = dict(question["criteria"]) + rules = instructions.get("rules") or () + if shape == "ours": + if name == "operation": + options = {op: _OPERATION_LABELS.get(op, text) for op, text in options.items()} + rules = prompts.OPERATION_RULES["en"] + else: + rules = (prompts.OPERATION_RULES["en"], prompts.TARGET_RULES["en"]) + questions[name] = ChoiceQuestion( + options, + goal=instructions.get("goal", ""), + rules=tuple(rules) if isinstance(rules, list) else rules, + operation=instructions.get("operation", ""), + ) + return Observation(request["state"]), questions + + +def record_id(record: dict[str, Any]) -> str: + return str((record.get("meta") or {}).get("id") or "") + + +def wire_body(model_id: str, observation: Any, questions: dict[str, Any]) -> dict[str, Any]: + """The Jev request body these questions become; saved with every step so the evidence shows what was asked.""" + from s1a.decision_models.jev import jev_question + + return { + "model": model_id, + "state": observation.state, + "questions": {name: jev_question(question) for name, question in questions.items()}, + } + + +# -- predict ------------------------------------------------------------------------------------------------------- + + +def agents_sha() -> str: + try: + out = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True, check=True, timeout=10) + return out.stdout.strip() + except (OSError, subprocess.SubprocessError): + return "" + + +async def predict_steps( + model: Any, + records: Iterable[dict[str, Any]], + results: Path, + *, + shape: str = "ours", + limit: int | None = None, + keep_request: bool = True, +) -> dict[str, int]: + """One request per step, in dataset order, no retries; every outcome is written, failures included.""" + counts = {"attempted": 0, "failed": 0} + with results.open("x", encoding="utf-8") as stream: + for record in records: + if limit is not None and counts["attempted"] >= limit: + break + step: dict[str, Any] = {"id": record_id(record), "ts": time.time()} + observation, questions = to_questions(record, shape) + if keep_request: + step["request"] = wire_body(model.model, observation, questions) + started = time.perf_counter() + try: + decision = await model.decide_many(observation, questions, attempts=1) + except Exception as exc: # noqa: BLE001 - every failure is evidence, scored as wrong + step["error"] = f"{type(exc).__name__}: {exc}" + counts["failed"] += 1 + else: + step["answers"] = {name: answer.as_dict() for name, answer in decision.answers.items()} + step["decision_ms"] = decision.latency_ms + step["model"] = decision.model + step["usage"] = { + "input_tokens": decision.usage.input_tokens, + "output_tokens": decision.usage.output_tokens, + "known": decision.usage.known, + } + step["wall_ms"] = round((time.perf_counter() - started) * 1000, 1) + counts["attempted"] += 1 + stream.write(json.dumps(step, ensure_ascii=False) + "\n") + stream.flush() + return counts + + +def predict(args: argparse.Namespace) -> None: + from s1a.decision_models import build_model + + if args.split in SPLITS and not args.unpinned: + digest = verify(args.data, args.split) + else: + digest = file_digest(args.data.read_bytes(), "sha256") + output = outside_repo(args.out) + environment = json.loads(args.run_meta.read_text(encoding="utf-8")) if args.run_meta else {} + model = build_model(args.model) + output.mkdir(parents=True, exist_ok=False) + meta = { + "dataset": DATASET, + "revision": REVISION, + "split": args.split, + "data_sha256_or_pin": digest, + "pinned": args.split in SPLITS and not args.unpinned, + "shape": args.shape, + "backend": args.model, + "model": model.model, + "agents_sha": agents_sha(), + "argv": sys.argv, + "limit": args.limit, + "environment": environment, + "started": time.time(), + } + (output / "run.meta.json").write_text(json.dumps(meta, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + async def run() -> dict[str, int]: + await model.warm() + try: + return await predict_steps( + model, read_records(args.data), output / "results.jsonl", shape=args.shape, limit=args.limit + ) + finally: + await model.close() + + counts = asyncio.run(run()) + completion = {**counts, "finished": time.time()} + (output / "completion.json").write_text(json.dumps(completion, indent=2) + "\n", encoding="utf-8") + print(json.dumps(completion)) + + +# -- scoring ------------------------------------------------------------------------------------------------------- + + +def judge(record: dict[str, Any], step: dict[str, Any] | None) -> dict[str, Any]: + """One step against its gold: a missing or failed step is wrong on every metric, never skipped.""" + gold = record["gold"] + gold_op = gold["operation"] + target_head = f"{gold_op.lower()}_target" + gold_keys = [str(key) for key in gold.get(target_head) or []] + out: dict[str, Any] = { + "id": record_id(record), + "gold_operation": gold_op, + "has_target": bool(gold_keys), + "answered": False, + "operation_ok": False, + "element_ok": False, + "step_ok": False, + "confidence": None, + "operation_probabilities": None, + "latency_ms": None, + } + if step is None or "answers" not in step: + return out + answers = step["answers"] + operation = answers.get("operation") or {} + out["answered"] = True + out["operation_ok"] = operation.get("choice") == gold_op + out["confidence"] = operation.get("confidence") + out["operation_probabilities"] = operation.get("probabilities") + out["latency_ms"] = step.get("decision_ms") + if gold_keys: + out["element_ok"] = str((answers.get(target_head) or {}).get("choice")) in gold_keys + out["step_ok"] = out["operation_ok"] and out["element_ok"] + else: + out["step_ok"] = out["operation_ok"] + return out + + +def percentile(values: list[float], q: float) -> float | None: + if not values: + return None + ordered = sorted(values) + position = (len(ordered) - 1) * q + low, high = math.floor(position), math.ceil(position) + return ordered[low] + (ordered[high] - ordered[low]) * (position - low) + + +def multiclass_brier(probabilities: dict[str, float], gold: str) -> float: + """Sum over the offered operations (and the gold one) of the squared gap to the one-hot gold vector.""" + keys = set(probabilities) | {gold} + return sum((float(probabilities.get(key, 0.0)) - float(key == gold)) ** 2 for key in keys) + + +def ratio(numerator: int, denominator: int) -> dict[str, Any]: + return {"value": numerator / denominator if denominator else None, "n": numerator, "of": denominator} + + +def summarize(judged: list[dict[str, Any]]) -> dict[str, Any]: + """Accuracies with their denominators, calibration of the operation confidence and decision latency.""" + total = len(judged) + targeted = [row for row in judged if row["has_target"]] + answered = [row for row in judged if row["answered"]] + confident = [row for row in answered if isinstance(row["confidence"], (int, float))] + brier = ( + statistics.fmean((row["confidence"] - float(row["operation_ok"])) ** 2 for row in confident) + if confident + else None + ) + multiclass = [ + multiclass_brier(row["operation_probabilities"], row["gold_operation"]) + for row in confident + if isinstance(row["operation_probabilities"], dict) + ] + latencies = [float(row["latency_ms"]) for row in answered if isinstance(row["latency_ms"], (int, float))] + return { + "steps": total, + "operation_accuracy": ratio(sum(row["operation_ok"] for row in judged), total), + "element_accuracy": ratio(sum(row["element_ok"] for row in targeted), len(targeted)), + "step_success": ratio(sum(row["step_ok"] for row in judged), total), + "invalid_or_failed": ratio(total - len(answered), total), + "operation_confidence_brier": brier, + "operation_multiclass_brier": statistics.fmean(multiclass) if multiclass else None, + "operation_confidence_ece": ece(confident), + "calibration_steps": len(confident), + "decision_ms_p50": percentile(latencies, 0.5), + "decision_ms_p95": percentile(latencies, 0.95), + "by_gold_operation": { + op: ratio(sum(row["step_ok"] for row in judged if row["gold_operation"] == op), count) + for op, count in sorted(_count(row["gold_operation"] for row in judged).items()) + }, + } + + +def _count(values: Iterable[str]) -> dict[str, int]: + counts: dict[str, int] = {} + for value in values: + counts[value] = counts.get(value, 0) + 1 + return counts + + +def ece(rows: list[dict[str, Any]], bins: int = ECE_BINS) -> float | None: + """Expected calibration error of the operation confidence against operation correctness, equal-width bins.""" + if not rows: + return None + buckets: list[list[dict[str, Any]]] = [[] for _ in range(bins)] + for row in rows: + buckets[min(bins - 1, int(float(row["confidence"]) * bins))].append(row) + return sum( + len(bucket) + / len(rows) + * abs(statistics.fmean(r["confidence"] for r in bucket) - statistics.fmean(r["operation_ok"] for r in bucket)) + for bucket in buckets + if bucket + ) + + +def load_run(run: Path) -> dict[str, dict[str, Any]]: + steps: dict[str, dict[str, Any]] = {} + for step in read_records(run / "results.jsonl"): + if step["id"] in steps: + raise ValueError(f"{run}: step {step['id']} appears twice") + steps[step["id"]] = step + return steps + + +def judge_run(run: Path, data: Path) -> list[dict[str, Any]]: + steps = load_run(run) + records = list(read_records(data)) + known = {record_id(record) for record in records} + orphans = sorted(set(steps) - known) + if orphans: + raise ValueError(f"{run}: {len(orphans)} step(s) are not in {data}, e.g. {orphans[0]}") + return [judge(record, steps.get(record_id(record))) for record in records] + + +def score(args: argparse.Namespace) -> dict[str, Any]: + judged = judge_run(args.run, args.data) + report: dict[str, Any] = {"run": str(args.run), "full": summarize(judged)} + if args.limit is not None: + report["attempted_prefix"] = summarize(judged[: args.limit]) + ids = read_ids(args.ids) + if ids is not None: + report["jev_answered"] = summarize([row for row in judged if row["id"] in ids]) + meta_path = args.run / "run.meta.json" + if meta_path.exists(): + report["meta"] = json.loads(meta_path.read_text(encoding="utf-8")) + return report + + +def threshold_table(judged: list[dict[str, Any]], thresholds: Iterable[float] = THRESHOLDS) -> list[dict[str, Any]]: + """Per threshold: how many steps the backend keeps (operation confidence at or above it) and how they fare. + + A failed step has no confidence, so it is always deferred; its share stays in the denominator. + """ + total = len(judged) + table = [] + for threshold in thresholds: + kept = [row for row in judged if isinstance(row["confidence"], (int, float)) and row["confidence"] >= threshold] + table.append( + { + "threshold": threshold, + "kept": ratio(len(kept), total), + "kept_step_success": ratio(sum(row["step_ok"] for row in kept), len(kept)), + "deferred": ratio(total - len(kept), total), + } + ) + return table + + +def thresholds(args: argparse.Namespace) -> dict[str, Any]: + judged = judge_run(args.run, args.data) + return {"run": str(args.run), "steps": len(judged), "table": threshold_table(judged)} + + +# -- CLI ----------------------------------------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + from s1a.decision_models import DECISION_MODEL_NAMES + + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + commands = parser.add_subparsers(dest="command", required=True) + + get = commands.add_parser("fetch", help="download and verify the pinned files") + get.add_argument("--out", type=Path, required=True) + get.add_argument("--split", choices=sorted(SPLITS), action="append", required=True) + + run = commands.add_parser("predict", help="send every step through one decision backend") + run.add_argument("--data", type=Path, required=True) + run.add_argument("--split", required=True, help="benchmark, calibration, or a name for an unpinned file") + run.add_argument("--model", choices=[n for n in DECISION_MODEL_NAMES if n not in ("rule",)], required=True) + run.add_argument("--shape", choices=SHAPES, default="ours") + run.add_argument("--limit", type=int) + run.add_argument("--run-meta", type=Path, help="JSON describing the backend: revision, hardware, timeout") + run.add_argument("--unpinned", action="store_true", help="accept a file that is not the pinned split") + run.add_argument("--out", type=Path, required=True) + + rescore = commands.add_parser("score", help="re-score a saved run") + rescore.add_argument("--run", type=Path, required=True) + rescore.add_argument("--data", type=Path, required=True) + rescore.add_argument("--ids", type=Path, help="benchmark_jev_answered_ids.txt for the Jev-answered subset") + rescore.add_argument("--limit", type=int, help="also report the first N steps when the run used --limit") + + table = commands.add_parser("thresholds", help="deferral threshold table from a calibration run") + table.add_argument("--run", type=Path, required=True) + table.add_argument("--data", type=Path, required=True) + + args = parser.parse_args(argv) + if args.command == "fetch": + for name, path in fetch(args.split, outside_repo(args.out)).items(): + print(f"{name}: {path}") + elif args.command == "predict": + predict(args) + elif args.command == "score": + print(json.dumps(score(args), indent=2, ensure_ascii=False)) + else: + print(json.dumps(thresholds(args), indent=2, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/docs/results/browser-steps/README.md b/docs/results/browser-steps/README.md new file mode 100644 index 0000000..ba8e7f0 --- /dev/null +++ b/docs/results/browser-steps/README.md @@ -0,0 +1,100 @@ +# Browser steps: offline step-level scoring (#109) + +`benchmarks.legs.browser_steps` asks a decision backend one browser step at a time from +[Mlchemist/myjev-browser-decisions](https://huggingface.co/datasets/Mlchemist/myjev-browser-decisions) +(revision `8c0c5d532458f7ba25c9684b267694848e815d5a`) and scores the answers against the gold operation and +element keys. It runs offline in minutes and needs no live site. + +**Step success is not task success.** A step is scored with the gold page state and gold history, so a backend that +gets most steps right can still fail whole tasks, and this does not replace the repeatable task suites of #13. + +## Licence and data handling + +The dataset is **CC BY-NC-SA 4.0** (it includes WebLINX), and its Mind2Web rows come from authors who ask that the +data not be republished. No dataset row is committed here: `fetch` downloads the pinned files at run time and +keeps a file only when its hash matches (SHA-256 for the two splits, the git blob id for the id list). Results +directories hold per-step answers keyed by the dataset's step id; keep them outside the repository and do not +publish the saved requests, which contain page text. + +## Run + +```bash +python -m benchmarks.legs.browser_steps fetch --out /data/myjev \ + --split benchmark --split calibration --split jev_answered_ids +python -m benchmarks.legs.browser_steps predict --data /data/myjev/benchmark.jsonl --split benchmark \ + --model jev --run-meta /path/environment.json --out /results/jev-benchmark +python -m benchmarks.legs.browser_steps score --run /results/jev-benchmark \ + --data /data/myjev/benchmark.jsonl --ids /data/myjev/benchmark_jev_answered_ids.txt +python -m benchmarks.legs.browser_steps predict --data /data/myjev/calibration.jsonl --split calibration \ + --model jev --out /results/jev-calibration +python -m benchmarks.legs.browser_steps thresholds --run /results/jev-calibration \ + --data /data/myjev/calibration.jsonl +``` + +`--model` takes any text backend `build_model` knows (`jev`, `laya`, `laya-served`, `cua`, `clm`, `random`), configured +from the environment as for the agents, so a local System One server is `--model jev` with `TYPESAFE_API_URL`. +Screenshot backends are out of scope: the set has no images. `--run-meta` is copied into `run.meta.json` next to +the dataset revision and hash, the request shape, the backend and model id and the Agents SHA; put the backend +revision, hardware and `S1A_DECISION_TIMEOUT_S` there. + +The collector sends one request per step, in dataset order, through `DecisionModel.decide_many` with one attempt +and no retries. A transport error, a timeout or an answer that fails validation is saved with its error and is +**wrong** on every metric; steps not attempted (`--limit`) stay in the denominator of the full-split numbers +(`score --limit N` adds the attempted prefix separately). + +## Metrics + +`score` prints, for the full split and for the Jev-answered subset (`--ids`), each with numerator and +denominator: + +- operation accuracy, over all steps; +- element accuracy, over steps whose gold operation takes a target: the answer to the gold operation's target + question is one of the gold keys, whatever operation was chosen (Mind2Web's element accuracy); +- step success: the operation is right and, when it takes a target, so is the element; +- invalid or failed answers; +- Brier score and 10-bin ECE of the operation confidence (`max p`) against operation correctness, and the + multiclass Brier of the operation probabilities; +- p50/p95 of the backend's decision latency; +- step success by gold operation. + +`thresholds` reads a calibration run and prints, per operation-confidence threshold from 0 to 0.95, the share of +steps the backend keeps, their step success and the share deferred (failed steps are always deferred). This is +the per-backend table the deferral of #94 / #107 needs; the calibration split shares no website with the +benchmark split. + +## Request shape: theirs and ours + +The dataset's request is what jev-ultrafast sends. `--shape dataset` re-sends it unchanged: over the 652 +calibration records, the body `JevModel` builds from the converted questions is identical to the dataset's +(checked on the real file; the tests pin the same property on a synthetic fixture). `--shape ours` (the default) +keeps the state, the question names and the element rows and swaps in the text `BrowserDecisionModel` sends. + +Checked on the calibration split, the structure already matches our action space: + +| | dataset | our agent | +|---|---|---| +| top level | `model`, `state`, `questions` | same | +| `state` | `page` {url, title, text}, `elements`, `recent_actions` (last 10, `page_changed` blanked) | same keys | +| element rows | `index`, `role`, `label`, `value`, `operations`, optional `checked`/`expanded`/`selected`/`options` | same, plus `region`, `blocked_by`, `click_did_nothing` when the probe sets them | +| questions | `operation`, `click_target`, `type_text_target`, `select_target` | same names; plus `press_enter_target` and `text_value` when offered | +| target options | `element`, `current_value`, `role`, optional state keys | same keys | +| SELECT keys | `index:option` | same | + +What `--shape ours` changes: the operation descriptions (our `_OPERATION_LABELS`) and the rules (our +`OPERATION_RULES["en"]`, and `TARGET_RULES["en"]` after it on target questions). What it cannot reproduce: our +`PRESS_ENTER` head and `text_value` question (the dataset has no such gold), our `option` field on SELECT rows +(the dataset folds the option label into `element` as `label → option`), and our probe's `region` / +`blocked_by` rows. Scores therefore measure the model on our wording over jev-ultrafast's page snapshots, not on +our probe. + +## Results + +| Backend | Split | Shape | Steps | Operation acc. | Element acc. | Step success | Failed | Op. conf. Brier | ECE | +|---|---|---|---|---|---|---|---|---|---| +| `random` (seed 0), floor | calibration | ours | 652 | 98/652 (15.0%) | 51/608 (8.4%) | 12/652 (1.8%) | 0/652 | 0.127 | 0.008 | +| `random` (seed 0), floor | calibration | dataset | 652 | 98/652 (15.0%) | 51/608 (8.4%) | 12/652 (1.8%) | 0/652 | 0.127 | 0.008 | + +The random rows were run offline on 2026-11-02 at Agents SHA `69527d9` (base of this change) with the code in +this directory; they only show the collector and scorer working end to end on real records and give the chance +floor. Hosted Jev and an open backend on the benchmark split have **not been run yet**: they need a TypeSafe key +and a GPU host, and their rows will be added here with backend revision, hardware and Agents SHA. diff --git a/tests/benchmarks/test_browser_steps.py b/tests/benchmarks/test_browser_steps.py new file mode 100644 index 0000000..d42184d --- /dev/null +++ b/tests/benchmarks/test_browser_steps.py @@ -0,0 +1,269 @@ +"""myjev-browser-decisions step benchmark on a synthetic fixture: no dataset rows, no network.""" + +import asyncio +import json +from pathlib import Path + +import httpx +import pytest + +from benchmarks.legs import browser_steps as bs +from s1a.decision_models import JevModel +from s1a.decision_models.fakes import ScriptedModel, ScriptedTransport + +RULES = "Advance the goal with one operation." +TARGET_RULES = [RULES, "Choose only an offered element index."] + + +def element_row(key, label, role="button", value=""): + return {"element": f"[{key}] {label}", "current_value": value, "role": role} + + +def record(step_id, gold, *, ops=("CLICK", "TYPE_TEXT", "SCROLL_DOWN", "WAIT", "DONE", "BLOCKED")): + """One synthetic step in the dataset's jev-ultrafast shape (invented content, not a dataset row).""" + goal = "Search for blue widgets" + questions = { + "operation": { + "type": "choice", + "criteria": {op: f"upstream text for {op}" for op in ops}, + "instructions": {"goal": goal, "rules": RULES}, + }, + "click_target": { + "type": "choice", + "criteria": {"1": element_row("1", "Home", "link"), "2": element_row("2", "Search")}, + "instructions": {"goal": goal, "operation": "CLICK", "rules": TARGET_RULES}, + }, + "type_text_target": { + "type": "choice", + "criteria": {"3": element_row("3", "Query", "textbox")}, + "instructions": {"goal": goal, "operation": "TYPE_TEXT", "rules": TARGET_RULES}, + }, + } + state = { + "page": {"url": "https://shop.invalid/", "title": "Shop", "text": "Home Search"}, + "elements": [ + {"index": "1", "role": "link", "label": "Home", "value": "", "operations": ["CLICK"]}, + {"index": "2", "role": "button", "label": "Search", "value": "", "operations": ["CLICK"]}, + {"index": "3", "role": "textbox", "label": "Query", "value": "", "operations": ["TYPE_TEXT", "CLICK"]}, + ], + "recent_actions": [], + } + return { + "request": {"model": "jev-latest", "state": state, "questions": questions}, + "gold": gold, + "meta": {"id": step_id, "source": "synthetic"}, + } + + +RECORDS = [ + record("s:1", {"operation": "CLICK", "click_target": ["2"]}), + record("s:2", {"operation": "TYPE_TEXT", "type_text_target": ["3"]}), + record("s:3", {"operation": "SCROLL_DOWN"}), + record("s:4", {"operation": "CLICK", "click_target": ["1", "2"]}), +] + + +def write_jsonl(path, rows): + path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + return path + + +def answer(choice, keys, p=0.9): + rest = (1 - p) / (len(keys) - 1) if len(keys) > 1 else 0.0 + return {"choice": choice, "probabilities": {k: (p if k == choice else rest) for k in keys}} + + +OPS = ["CLICK", "TYPE_TEXT", "SCROLL_DOWN", "WAIT", "DONE", "BLOCKED"] + + +def reply(op, click="2", text="3", p=0.9): + return { + "operation": answer(op, OPS, p), + "click_target": answer(click, ["1", "2"]), + "type_text_target": answer(text, ["3"], 1.0), + } + + +def test_dataset_shape_resends_the_dataset_body_unchanged(): + transport = ScriptedTransport() + model = JevModel(transport) + for item in RECORDS: + observation, questions = bs.to_questions(item, "dataset") + asyncio.run(model.decide_many(observation, questions)) + sent = transport.bodies[-1] + assert sent["state"] == item["request"]["state"] + assert json.dumps(sent["questions"]) == json.dumps(item["request"]["questions"]) + + +def test_ours_shape_swaps_only_the_text_our_agent_sends(): + from s1a.browser import prompts + from s1a.browser.action_space import _OPERATION_LABELS + + item = RECORDS[0] + observation, questions = bs.to_questions(item, "ours") + assert observation.state == item["request"]["state"] + assert list(questions) == list(item["request"]["questions"]) + operation = questions["operation"] + assert list(operation.options) == OPS + assert operation.options["CLICK"] == _OPERATION_LABELS["CLICK"] + assert operation.rules == (prompts.OPERATION_RULES["en"],) + click = questions["click_target"] + assert dict(click.options) == item["request"]["questions"]["click_target"]["criteria"] + assert click.rules == (prompts.OPERATION_RULES["en"], prompts.TARGET_RULES["en"]) + assert click.operation == "CLICK" and click.goal == "Search for blue widgets" + + +def test_gold_is_never_sent(tmp_path): + transport = ScriptedTransport() + asyncio.run(bs.predict_steps(JevModel(transport), RECORDS, tmp_path / "r.jsonl")) + assert len(transport.bodies) == len(RECORDS) + assert all("gold" not in json.dumps(body) and "meta" not in body for body in transport.bodies) + + +def run_and_judge(tmp_path, model, records=RECORDS, limit=None): + data = write_jsonl(tmp_path / "data.jsonl", records) + run = tmp_path / "run" + run.mkdir() + counts = asyncio.run(bs.predict_steps(model, records, run / "results.jsonl", limit=limit)) + return counts, bs.judge_run(run, data), run, data + + +def test_scores_with_denominators_and_failures_count_as_wrong(tmp_path): + from openjiuwen.core.common.exception.codes import StatusCode + from openjiuwen.core.common.exception.errors import build_error + + answers = [ + reply("CLICK", click="2"), # step right + reply("CLICK", click="2", p=0.6), # wrong operation (gold TYPE_TEXT) + {"operation": answer("SCROLL_DOWN", OPS)}, # incomplete reply: validation fails, counted wrong + reply("CLICK", click="1"), # one of two gold keys + ] + model = ScriptedModel(answers=answers) + counts, judged, _, _ = run_and_judge(tmp_path, model) + assert counts == {"attempted": 4, "failed": 1} + summary = bs.summarize(judged) + assert summary["steps"] == 4 + assert summary["step_success"] == {"value": 0.5, "n": 2, "of": 4} + assert summary["operation_accuracy"] == {"value": 0.5, "n": 2, "of": 4} + assert summary["element_accuracy"] == {"value": 1.0, "n": 3, "of": 3} # step 2's type target is still right + assert summary["invalid_or_failed"] == {"value": 0.25, "n": 1, "of": 4} + assert summary["calibration_steps"] == 3 + assert summary["operation_confidence_brier"] == pytest.approx(((0.9 - 1) ** 2 * 2 + 0.6**2) / 3) + assert summary["decision_ms_p50"] == 9 + assert summary["by_gold_operation"]["SCROLL_DOWN"] == {"value": 0.0, "n": 0, "of": 1} + assert build_error(StatusCode.MODEL_CALL_FAILED, error_msg="x") # the error type the layer raises exists + + +def test_transport_error_is_recorded_and_scored_wrong(tmp_path): + model = JevModel(ScriptedTransport(error=httpx.ConnectError("refused"))) + counts, judged, run, _ = run_and_judge(tmp_path, model) + assert counts["failed"] == 4 + steps = [json.loads(line) for line in (run / "results.jsonl").read_text().splitlines()] + assert all("ConnectError" in step["error"] and "request" in step for step in steps) + summary = bs.summarize(judged) + assert summary["step_success"]["value"] == 0.0 and summary["invalid_or_failed"]["value"] == 1.0 + assert summary["operation_confidence_brier"] is None and summary["decision_ms_p50"] is None + + +def test_unattempted_steps_stay_in_the_denominator(tmp_path): + model = ScriptedModel(answers=[reply("CLICK")]) + _, judged, _, _ = run_and_judge(tmp_path, model, limit=1) + assert bs.summarize(judged)["step_success"] == {"value": 0.25, "n": 1, "of": 4} + assert bs.summarize(judged[:1])["step_success"] == {"value": 1.0, "n": 1, "of": 1} + + +def test_orphan_and_duplicate_steps_are_errors(tmp_path): + data = write_jsonl(tmp_path / "data.jsonl", RECORDS) + run = tmp_path / "run" + run.mkdir() + write_jsonl(run / "results.jsonl", [{"id": "elsewhere:1"}]) + with pytest.raises(ValueError, match="not in"): + bs.judge_run(run, data) + write_jsonl(run / "results.jsonl", [{"id": "s:1"}, {"id": "s:1"}]) + with pytest.raises(ValueError, match="twice"): + bs.judge_run(run, data) + + +def test_ece_and_multiclass_brier(): + rows = [{"confidence": 0.9, "operation_ok": True}, {"confidence": 0.9, "operation_ok": False}] + assert bs.ece(rows) == pytest.approx(0.4) + assert bs.ece([]) is None + assert bs.multiclass_brier({"A": 0.7, "B": 0.3}, "A") == pytest.approx(0.18) + assert bs.multiclass_brier({"A": 1.0}, "B") == pytest.approx(2.0) + + +def test_threshold_table_defers_failures(tmp_path): + answers = [reply("CLICK", p=0.95), reply("CLICK", p=0.5), reply("SCROLL_DOWN", p=0.7), {}] + _, judged, _, _ = run_and_judge(tmp_path, ScriptedModel(answers=answers)) + table = {row["threshold"]: row for row in bs.threshold_table(judged, (0.0, 0.6, 0.9))} + assert table[0.0]["kept"] == {"value": 0.75, "n": 3, "of": 4} + assert table[0.6]["kept"]["n"] == 2 and table[0.6]["kept_step_success"] == {"value": 1.0, "n": 2, "of": 2} + assert table[0.9]["deferred"] == {"value": 0.75, "n": 3, "of": 4} + + +def test_verify_checks_pinned_digests(tmp_path, monkeypatch): + path = tmp_path / "calibration.jsonl" + path.write_bytes(b"{}\n") + with pytest.raises(ValueError, match="is not"): + bs.verify(path, "calibration") + monkeypatch.setitem( + bs.SPLITS, "calibration", ("data/calibration.jsonl", "sha256", bs.file_digest(b"{}\n", "sha256")) + ) + assert bs.verify(path, "calibration") + # git blob ids match `git hash-object` + assert bs.file_digest(b"hello\n", "git-blob") == "ce013625030ba8dba906f756967f9e9ca394464a" + + +def test_fetch_keeps_only_verified_files(tmp_path, monkeypatch): + body = b'{"x": 1}\n' + monkeypatch.setitem(bs.SPLITS, "calibration", ("data/calibration.jsonl", "sha256", bs.file_digest(body, "sha256"))) + seen = [] + + def handler(request): + seen.append(str(request.url)) + return httpx.Response(200, content=body) + + saved = bs.fetch(["calibration"], tmp_path, transport=httpx.MockTransport(handler)) + assert saved["calibration"].read_bytes() == body + assert seen == [f"https://huggingface.co/datasets/{bs.DATASET}/resolve/{bs.REVISION}/data/calibration.jsonl"] + + bad = tmp_path / "bad" + with pytest.raises(ValueError): + bs.fetch(["calibration"], bad, transport=httpx.MockTransport(lambda _: httpx.Response(200, content=b"no"))) + assert not any(bad.iterdir()) + + +def test_cli_predict_score_thresholds_end_to_end(tmp_path, monkeypatch, capsys): + data = write_jsonl(tmp_path / "data.jsonl", RECORDS) + monkeypatch.setattr("s1a.decision_models.build_model", lambda name: ScriptedModel(answers=[reply("CLICK")] * 4)) + out = tmp_path / "run" + bs.main(["predict", "--data", str(data), "--split", "synthetic", "--model", "random", "--out", str(out)]) + meta = json.loads((out / "run.meta.json").read_text()) + assert meta["pinned"] is False and meta["shape"] == "ours" and meta["revision"] == bs.REVISION + assert json.loads((out / "completion.json").read_text())["attempted"] == 4 + capsys.readouterr() + ids = tmp_path / "ids.txt" + ids.write_text("s:1\ns:3\n") + bs.main(["score", "--run", str(out), "--data", str(data), "--ids", str(ids)]) + report = json.loads(capsys.readouterr().out) + assert report["full"]["step_success"] == {"value": 0.5, "n": 2, "of": 4} + assert report["jev_answered"]["steps"] == 2 + bs.main(["thresholds", "--run", str(out), "--data", str(data)]) + assert len(json.loads(capsys.readouterr().out)["table"]) == len(bs.THRESHOLDS) + + +def test_predict_refuses_a_pinned_split_name_with_the_wrong_file(tmp_path): + data = write_jsonl(tmp_path / "data.jsonl", RECORDS) + with pytest.raises(ValueError, match="is not"): + bs.main( + ["predict", "--data", str(data), "--split", "benchmark", "--model", "random", "--out", str(tmp_path / "o")] + ) + + +def test_evidence_must_live_outside_the_repository(tmp_path): + data = write_jsonl(tmp_path / "data.jsonl", RECORDS) + inside = Path(bs.__file__).resolve().parents[1] / "should-not-exist" + with pytest.raises(ValueError, match="outside"): + bs.main( + ["predict", "--data", str(data), "--split", "x", "--unpinned", "--model", "random", "--out", str(inside)] + )