From efae4c35472e3598fec485ef082aa6625122a933 Mon Sep 17 00:00:00 2001 From: Danyang Chen Date: Wed, 4 Nov 2026 20:08:14 +0800 Subject: [PATCH] [Feat] Desktop --box-target via Jev Box + presence noul (#98) Locate unknown desktop click targets each step with hosted Jev 1.14 Box (gated by noul presence and box confidence), click the box centre with the same capture binding as --pixel-target. Adds BoxQuestion/Box to the decision-model layer, offline fake-transport tests, and a small measure_box_targets helper. Depends on #97 / issue-97-jev-images. --- CHANGELOG.md | 8 + docs/agents.md | 10 +- docs/capabilities.md | 2 +- docs/configuration.md | 2 + docs/decision-models.md | 10 +- evals/desktop/visual.md | 8 + s1a/agents/desktop.py | 83 +++++++++- s1a/decision_models/__init__.py | 6 + s1a/decision_models/fakes.py | 16 +- s1a/decision_models/jev.py | 13 ++ s1a/decision_models/laya.py | 4 +- s1a/decision_models/omnijev.py | 4 +- s1a/decision_models/types.py | 58 ++++++- s1a/decision_models/validation.py | 37 +++++ s1a/desktop/env.py | 131 +++++++++++++++- s1a/tool/models.py | 15 ++ scripts/measure_box_targets.py | 116 ++++++++++++++ tests/test_desktop_box_target.py | 251 ++++++++++++++++++++++++++++++ 18 files changed, 755 insertions(+), 19 deletions(-) create mode 100644 scripts/measure_box_targets.py create mode 100644 tests/test_desktop_box_target.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 69b60cd..1a194dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); ver ## Unreleased +### Added + +- Experimental desktop `--box-target KEY=description` (#98): each step asks hosted Jev (1.14+, `S1A_JEV_IMAGES`) a + presence `noul` and a `box` per target, offers only those above uncalibrated `S1A_BOX_PRESENCE_THRESHOLD` / + `S1A_BOX_CONFIDENCE_THRESHOLD` (defaults 0.5), and clicks the box centre with the same capture binding as + `--pixel-target`. Decision-model layer gains `BoxQuestion` / `Box`. Offline tests use a fake transport; live + fixture measurement needs TypeSafe images. Helper: `scripts/measure_box_targets.py`. Depends on #97 / PR #99. + ### Fixed - Served OneJev uses one HTTP decision deadline across its permitted retry. A retry receives only the remaining diff --git a/docs/agents.md b/docs/agents.md index 087cf1a..d4bb5a8 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -50,7 +50,7 @@ If a call's token usage is unknown, the reported cost stays unknown rather than `--batch on|off`, `--prefetch on|off`, `--goal-values on|off`, `--rethink on|off`, `--rethink-attempts`, `--rethink-timeout`, `--profile-out` and `--logs-dir`; `desktop` takes `--app`, `--app-path`, `--window-title`, `--goal`, `--expect`, `--execute`, `--plan`, `--clear`, `--text`, `--text-target`, `--text-mode`, `--verify-file`, -`--pixel-target` and the same three `--rethink` flags; `ticket_router` takes `--dataset` and `--batch-size`; +`--pixel-target`, `--box-target` and the same three `--rethink` flags; `ticket_router` takes `--dataset` and `--batch-size`; `injection_guard` takes `--labelled-set`. The four games take no flag of their own. ### Desktop text input @@ -71,6 +71,14 @@ among them when controls have no accessibility elements. Each click carries its Install the `cua-four-b` extra and set `CUA_S1_VARIANT=4b`, `CUA_S1_MODALITY=multimodal`. See [the macOS visual task](../evals/desktop/visual.md) for setup and verification commands. +`--box-target KEY=description` (**experimental**, needs #97 / `S1A_JEV_IMAGES=1`, `TYPESAFE_MODEL=jev-1.14.0` or +newer) locates each key with Jev's Box question instead of hand-written coordinates. Every step one request asks, +per target, a presence `noul` and a `box` over the current screenshot. Targets below +`S1A_BOX_PRESENCE_THRESHOLD` or `S1A_BOX_CONFIDENCE_THRESHOLD` (both default `0.5`, **uncalibrated**) are not +offered; the click is the box centre, bound to the current capture like `--pixel-target`. Do not treat this as a +general accuracy claim; measure on your own captures with `scripts/measure_box_targets.py`. A live run on the #30 +visual fixture is still required before calling the feature measured. Browser Box is out of scope. + ## Allrecipes `allrecipes` is the first Allrecipes task of the [WebVoyager](https://github.com/MinorJerry/WebVoyager) task set diff --git a/docs/capabilities.md b/docs/capabilities.md index 946f6e9..afdc16b 100644 --- a/docs/capabilities.md +++ b/docs/capabilities.md @@ -10,7 +10,7 @@ For issue trackers and serving support, see [System1-Omni model tracker #83](htt | Model / checkpoint | Input modality | Question types implemented | Scenario / front | Serving path | Hardware validation in this repository | |---|---|---|---|---|---| -| TypeSafe Jev (`--model jev`; actual served model comes from the configured endpoint) | Text state and enumerated options; with `S1A_JEV_IMAGES=1` on direct TypeSafe, also one current screenshot (**experimental, TypeSafe danger zone**: offline fake-transport tests only, no recorded live run or quality result, #97) | `choice`, `noul`; service protocol also defines `score`, but Agents has no corresponding question type | Tool decisions, browser decisions, rails, `decide`/`probe` and MCP | HTTP through TypeSafe or configured OpenRouter-compatible endpoint; configured compatible endpoint can use System1-Omni `/v1/systemone` | The adapter can address configured Jev-compatible services; exact model identity is endpoint-reported. Historical artifacts include Open-Jev-9B and an Open-Jev-27B-v1.1 load attempt; those are endpoint configurations, not model names guaranteed by the `jev` adapter. See below. Other combinations **unverified here.** | +| TypeSafe Jev (`--model jev`; actual served model comes from the configured endpoint) | Text state and enumerated options; with `S1A_JEV_IMAGES=1` on direct TypeSafe, also one current screenshot (**experimental, TypeSafe danger zone**: offline fake-transport tests only, no recorded live run or quality result, #97) | `choice`, `noul`; with `S1A_JEV_IMAGES` also experimental `box` (#98, Jev 1.14+); service protocol also defines `score`, but Agents has no corresponding question type | Tool decisions, browser decisions, rails, `decide`/`probe` and MCP | HTTP through TypeSafe or configured OpenRouter-compatible endpoint; configured compatible endpoint can use System1-Omni `/v1/systemone` | The adapter can address configured Jev-compatible services; exact model identity is endpoint-reported. Historical artifacts include Open-Jev-9B and an Open-Jev-27B-v1.1 load attempt; those are endpoint configurations, not model names guaranteed by the `jev` adapter. See below. Other combinations **unverified here.** | | JEV-27B-VL (`--model jev-vl-served`; `autotrust/JEV-27B-VL`) | Exactly one current PNG/JPEG plus text state and question context | `choice`, `noul` | Browser front and direct `build_model()` factory path; other fronts are not registered | HTTP native flat `/v1/systemone` through the System1-Omni frontend at `JEV_VL_URL` | Local adapter/registration tests use mocks. Single-H800 GPU checks and bounded R3 browser task completion with same-trial video are validated; other tasks and hardware remain unverified. See [validation scope and limits](decision-models.md#served-jev-vl). | | CLM-v0.1-8B (`--model clm`; default serving label `clm-latest`) | Text | `choice`, `noul` | Tool decisions, browser, rails, `decide`/`probe`, MCP | HTTP `/v1/systemone` to a compatible CLM service; real encoder is external | Adapter fixtures include stub responses. A historical ticket-router report records real Qwen3-8B through upstream `clm-serve` on an RTX 4090, but its Agents source SHA is not pinned in the report and the serving engine is not Omni native CLM. It reports 18/90 correct (20%) for the normal tool-loop framing and a 4/12 fit-probe verdict of “not a decision-model task”; the 90 decisions repeat the same 30-ticket set and are not 90 independent tickets. Treat this as poor, provenance-incomplete historical evidence, not current or broad validation. See [CLM report](../evals/ticket_router/CLM.md). | | LAYA (`--model laya`; default `convaiinnovations/laya`, English; `LAYA_SUBFOLDER=multilingual` selects multilingual) | Text | `choice`, `noul` | Tool, browser, rails, `decide`/`probe`; browser state/question shaping is implemented | In-process Python `laya` | Historical English and multilingual CPU/MPS runs are documented below. Other/typed/custom checkpoints and CUDA are unverified here. | diff --git a/docs/configuration.md b/docs/configuration.md index bf3cefe..bce66d7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -11,6 +11,8 @@ Variables can be exported in your shell or placed in a `.env` file at the root o | `TYPESAFE_API_KEY` | `jev` model | *(unset)* | API key for direct TypeSafe decisions endpoint (`https://api.typesafe.ai/v1/systemone`). | | `TYPESAFE_API_URL` | `jev` model | `https://openrouter.ai/api/alpha/decisions` | Endpoint URL for decisions; defaults to OpenRouter proxy, or can be overridden to a custom proxy URL. | | `TYPESAFE_MODEL` | `jev` model | `jev-1.13.0` direct, `typesafe/jev-1.13` through OpenRouter | Jev model id on either backend. Both defaults are pinned versions: no path sends the moving `jev-latest` or `jev-preview` aliases unless you set them here. On the direct path (`TYPESAFE_API_KEY` without `TYPESAFE_API_URL`) it takes a TypeSafe name such as `jev-1.14.0`; an OpenRouter id like `typesafe/jev-1.13` there is a config error. TypeSafe keeps `jev-1.13.0` available and promises 60 days' notice before deprecating it. Step records keep `model` (the id the response reports, else the one sent) and `requested_model` (the one sent). | +| `S1A_BOX_PRESENCE_THRESHOLD` | desktop `--box-target` | `0.5` (uncalibrated) | Minimum Jev `noul` presence probability for a `--box-target` to be offered that step. Experimental; tune with `scripts/measure_box_targets.py`. | +| `S1A_BOX_CONFIDENCE_THRESHOLD` | desktop `--box-target` | `0.5` (uncalibrated) | Minimum Jev Box confidence for a `--box-target` to be offered that step. Experimental; no recommended TypeSafe threshold yet. | | `S1A_JEV_IMAGES` | `jev` model (direct TypeSafe only) | *(unset: off)* | **Experimental (TypeSafe danger zone).** `1`/`true` lets hosted Jev read the observation's screenshot: the browser front then captures one per step, and the desktop `--pixel-target` path accepts `--model jev`. Each request carries one PNG/JPEG in TypeSafe's `assets` map, referenced once from `state.screenshot` (a fresh UUID reference per request), plus the `TypeSafe-Danger-Zone-Images: true` header. Screenshots whose longest side is over 1,568 px are scaled down (aspect kept, sent as PNG); a side under 280 px after that, a file over 16 MiB or a body over 32 MiB is a config error before anything is sent. Step records add `sent_image` (sent width, height, bytes, format, whether it was resized); image tokens are billed as `input_tokens`. Set with the OpenRouter proxy or `TYPESAFE_API_URL` it is a config error. An image request can take longer than the 5 s default deadline; raise `S1A_DECISION_TIMEOUT_S` if it times out. | | `S1A_DECISION_TIMEOUT_S` | `jev`, `jev-vl-served` models | `5` | Total deadline in seconds for one decision. Jev includes retries; JEV-VL shares it across sequential questions and does not retry HTTP requests. Must be finite and above zero. Raise it for a slower System One server behind `TYPESAFE_API_URL` (e.g. a local OneJev-27B or CLM-8B: `30`). | | `JEV_VL_URL` | `jev-vl-served` model | *(unset, required)* | Frontend base URL serving `autotrust/JEV-27B-VL` through System1-Omni's native flat `/v1/systemone` protocol, e.g. `http://127.0.0.1:8080`. Browser and direct factory paths only; requires exactly one current PNG/JPEG image. | diff --git a/docs/decision-models.md b/docs/decision-models.md index 8f4f5e7..85e5b69 100644 --- a/docs/decision-models.md +++ b/docs/decision-models.md @@ -21,9 +21,11 @@ The registered paths, input modalities and question types are summarized in the Input: an `Observation(state, images=())` (a JSON object or plain text) and typed questions by name: `ChoiceQuestion(options, goal=, rules=, operation=)` picks one key among `options` (a string per key, or a dict for -the browser's element rows); `NoulQuestion(question, criteria=)` asks whether a statement holds. Output: a -`Decision(answers, latency_ms, usage, model, raw)` with one `Choice(key, probabilities, confidence)` or -`Noul(p, confidence)` per question name. **`confidence` always means the same thing on every backend:** for a +the browser's element rows); `NoulQuestion(question, criteria=)` asks whether a statement holds; +`BoxQuestion(target)` (experimental, hosted Jev 1.14+ with images) asks where something is and returns a +`Box(top, bottom, left, right, confidence)` as image fractions. Output: a +`Decision(answers, latency_ms, usage, model, raw)` with one `Choice(key, probabilities, confidence)`, +`Noul(p, confidence)` or `Box(...)` per question name. **`confidence` always means the same thing on every backend:** for a choice it is `max_k p_k` (the peak probability); for a noul it is `max(p, 1 - p)`. Validation computes both from the validated probabilities in `s1a/decision_models/validation.py`, so Laya's normalised entropy, Jev's normalised margin, and OmniJev's recompute never become the reported `confidence`. Each backend's own number is kept as @@ -58,7 +60,7 @@ shorthands; `warm()` and `close()` open and release the backend. | `--model` | class | `name` | notes | |---|---|---|---| -| `jev` | `JevModel(transport)` | `jev` | the request body every front sent before the layer existed, byte for byte; `from_env` picks TypeSafe or the OpenRouter proxy (see [configuration.md](configuration.md)). Text only unless `S1A_JEV_IMAGES=1` on the direct TypeSafe backend (**experimental, TypeSafe danger zone**, #97): then one current screenshot rides in `assets`, referenced from `state.screenshot`, with the danger-zone header; no quality result is claimed for it yet | +| `jev` | `JevModel(transport)` | `jev` | the request body every front sent before the layer existed, byte for byte; `from_env` picks TypeSafe or the OpenRouter proxy (see [configuration.md](configuration.md)). Text only unless `S1A_JEV_IMAGES=1` on the direct TypeSafe backend (**experimental, TypeSafe danger zone**, #97): then one current screenshot rides in `assets`, referenced from `state.screenshot`, with the danger-zone header; with images on, `box` questions are accepted too (#98, Jev 1.14+); no quality result is claimed for screenshots or Box yet | | `jev-vl-served` | `ServedJevVLModel(url=, timeout_s=)` | `jev-vl-served` | `autotrust/JEV-27B-VL` through native flat `POST /v1/systemone`; exactly one current PNG/JPEG, choice and noul only; sequential questions share one deadline, with no HTTP retry or answer re-ask. Browser and direct factory paths; local tests use mocks, with separate [bounded R3 GPU/browser validation](#served-jev-vl). | | `clm` | `ClmModel(client, model=)` | `clm` | CLM behind `/v1/systemone`: a frozen Qwen3-8B encoder and two projection heads, scored by an engine that owns everything after the encoder. One POST per decision; `MODEL_SERVICE_CONFIG_ERROR` without `CLM_URL`, `MODEL_CALL_FAILED` when nothing is listening, on an error status, or on a body with no `answers`. Records `url`, the `X-Request-Id` it sent and `served_by` (the served names, read once — by `warm()` on the agent fronts, or on the first decision for `decide`/`probe`); choice and noul ([docs/clm.md](clm.md)) | | `laya` | `LayaModel(agent, model=)` | `laya` | one forward pass per call on a thread; `MODEL_SERVICE_CONFIG_ERROR` when `input_tokens` fills the window (Laya cuts the state silently; see `LAYA_MAX_LEN` and `LAYA_HEAD_MAX_LEN` in [configuration.md](configuration.md)); `ValueError` and `RuntimeError` from the library become `MODEL_CALL_FAILED` | diff --git a/evals/desktop/visual.md b/evals/desktop/visual.md index 0a324b7..51804e4 100644 --- a/evals/desktop/visual.md +++ b/evals/desktop/visual.md @@ -43,3 +43,11 @@ of the captured window. Every click carries its capture ID so the driver can rej The temporary selection file and ignored run outputs are generated locally. Screenshot bytes are passed to the model separately from the serialized observation state. + +## Experimental: `--box-target` instead of `--pixel-target` + +Requires hosted Jev 1.14+ with `S1A_JEV_IMAGES=1` (see #97 / PR #99) and a TypeSafe key. Replace the two +`--pixel-target` flags with e.g. `--box-target left=the tile labelled Save` / `--box-target right=the tile labelled Cancel` +only after measuring on this fixture; defaults for `S1A_BOX_*_THRESHOLD` are uncalibrated. This checkout has no +recorded live Box fixture run (no TypeSafe image key / GPU on the agent box) — keep any such PR draft until one lands. + diff --git a/s1a/agents/desktop.py b/s1a/agents/desktop.py index 9ca20af..ad4b952 100644 --- a/s1a/agents/desktop.py +++ b/s1a/agents/desktop.py @@ -26,7 +26,8 @@ from pathlib import Path from typing import Any, AsyncIterator -from s1a.decision_models.wire import decisions_backend_from_env, jev_images_from_env +from s1a.decision_models.wire import TYPESAFE_DEFAULT_MODEL, decisions_backend_from_env, jev_images_from_env +from s1a.desktop.env import DEFAULT_BOX_CONFIDENCE_THRESHOLD, DEFAULT_BOX_PRESENCE_THRESHOLD from s1a.desktop.driver import CuaDriver, DriverError, Snapshot, driver_from_env, opened from s1a.desktop.env import ABSTAIN, WindowEnv, clickable, observable from s1a.spec import Budget, Series, ToolAgentSpec, positive_float, positive_int @@ -71,6 +72,51 @@ def parse_pixel_targets(entries: list[str]) -> dict[str, tuple[float, float]]: return points +def parse_box_targets(entries: list[str]) -> dict[str, str]: + """Task-defined visual targets located each step by Jev Box; ``KEY=description`` (description may contain spaces).""" + targets: dict[str, str] = {} + for entry in entries: + key, sep, description = entry.partition("=") + if not sep or not key.strip() or not description.strip() or key in targets: + raise ValueError('--box-target requires unique KEY="description" entries') + targets[key] = description.strip() + return targets + + +def box_thresholds_from_env() -> tuple[float, float]: + """Presence (noul) and box-confidence gates; defaults are uncalibrated (see configuration.md).""" + + def one(name: str, default: float) -> float: + raw = (os.getenv(name) or "").strip() + if not raw: + return default + try: + value = float(raw) + except ValueError as exc: + raise ValueError(f"{name} must be a finite number in [0, 1], not {raw!r}") from exc + if not math.isfinite(value) or not 0 <= value <= 1: + raise ValueError(f"{name} must be a finite number in [0, 1], not {raw!r}") + return value + + return ( + one("S1A_BOX_PRESENCE_THRESHOLD", DEFAULT_BOX_PRESENCE_THRESHOLD), + one("S1A_BOX_CONFIDENCE_THRESHOLD", DEFAULT_BOX_CONFIDENCE_THRESHOLD), + ) + + +def _jev_supports_box(model: str) -> bool: + """True for Jev 1.14+ (Box question type) and the moving ``jev-latest`` / ``jev-preview`` aliases.""" + name = model.rsplit("/", 1)[-1].strip().lower() + if name in {"jev-latest", "jev-preview"}: + return True + parts = name.removeprefix("jev-").split(".") + try: + major, minor = int(parts[0]), int(parts[1]) if len(parts) > 1 else 0 + except (ValueError, IndexError): + return False + return (major, minor) >= (1, 14) + + def plan_rule(plan: tuple[tuple[str, ...], ...]) -> Any: """The baseline: the next action of the plan by how many actions were made.""" @@ -78,7 +124,10 @@ def rule(state: dict[str, Any], candidates: dict[str, str]) -> str: step = len(state["presses"]) if step >= len(plan): return ABSTAIN - keys = (label if label.startswith(("click:", "type:", "pixel:")) else f"click:{label}" for label in plan[step]) + keys = ( + label if label.startswith(("click:", "type:", "pixel:", "box:")) else f"click:{label}" + for label in plan[step] + ) return next((key for key in keys if key in candidates), ABSTAIN) return rule @@ -115,12 +164,14 @@ def make_series(flags: argparse.Namespace) -> Series: if flags.app_path and sys.platform != "darwin": raise ValueError("--app-path is supported only on macOS") pixel_targets = parse_pixel_targets(flags.pixel_target) + box_targets = parse_box_targets(flags.box_target) + presence_threshold, box_confidence_threshold = box_thresholds_from_env() visual_model = ( flags.model == "cua" and os.getenv("CUA_S1_VARIANT", "nano") == "4b" and os.getenv("CUA_S1_MODALITY", "multimodal") == "multimodal" ) - if pixel_targets and flags.model == "jev": + if (pixel_targets or box_targets) and flags.model == "jev": # experimental hosted Jev screenshots (#97): a config error on the proxy backend, False when unset visual_model = jev_images_from_env(decisions_backend_from_env()) if pixel_targets and flags.model not in {"rule", "random"} and not visual_model: @@ -129,6 +180,17 @@ def make_series(flags: argparse.Namespace) -> Series: "CUA_S1_MODALITY=multimodal, --model jev with S1A_JEV_IMAGES=1 on the direct TypeSafe backend " "(experimental), or an explicit rule/random baseline" ) + if box_targets: + if flags.model != "jev" or not visual_model: + raise ValueError( + "--box-target requires --model jev with S1A_JEV_IMAGES=1 on the direct TypeSafe backend " + "(experimental; Jev Box needs an image and the danger-zone header)" + ) + jev_model = (os.getenv("TYPESAFE_MODEL") or "").strip() or TYPESAFE_DEFAULT_MODEL + if not _jev_supports_box(jev_model): + raise ValueError( + f"--box-target requires Jev 1.14.0 or newer with Box (set TYPESAFE_MODEL=jev-1.14.0); got {jev_model!r}" + ) plan = parse_plan(flags.plan) if flags.plan else () if flags.verify_file and not flags.text: raise ValueError("--verify-file requires --text") @@ -166,7 +228,10 @@ def env_for(seed: int) -> WindowEnv: text_target=flags.text_target, text_mode=flags.text_mode, pixel_targets=pixel_targets, - screenshot=visual_model, + box_targets=box_targets, + presence_threshold=presence_threshold, + box_confidence_threshold=box_confidence_threshold, + screenshot=visual_model or bool(box_targets), ) return Series( @@ -219,6 +284,16 @@ def flags(parser: argparse.ArgumentParser) -> None: metavar="KEY=X,Y", help="task-defined screenshot point (fractions in [0,1)); repeat for closed visual choices", ) + parser.add_argument( + "--box-target", + action="append", + default=[], + metavar="KEY=DESCRIPTION", + help=( + "experimental: locate KEY each step with Jev Box (noul presence + box); " + "requires --model jev, S1A_JEV_IMAGES=1, TYPESAFE_MODEL=jev-1.14.0+" + ), + ) SPEC = ToolAgentSpec( diff --git a/s1a/decision_models/__init__.py b/s1a/decision_models/__init__.py index b3bdab2..092e2d5 100644 --- a/s1a/decision_models/__init__.py +++ b/s1a/decision_models/__init__.py @@ -18,6 +18,8 @@ from s1a.decision_models.onejev import OneJevModel from s1a.decision_models.types import ( Answer, + Box, + BoxQuestion, Choice, ChoiceQuestion, Decision, @@ -35,6 +37,7 @@ choice_faults, noul_confidence, validate_answers, + validate_box, validate_choice, validate_noul, ) @@ -42,6 +45,8 @@ __all__ = [ "DECISION_MODEL_NAMES", "Answer", + "BoxQuestion", + "Box", "DecisionModel", "JevTransport", "ScriptedModel", @@ -71,6 +76,7 @@ "noul_confidence", "jev_question", "validate_answers", + "validate_box", "validate_choice", "validate_noul", ] diff --git a/s1a/decision_models/fakes.py b/s1a/decision_models/fakes.py index ecd0416..710e901 100644 --- a/s1a/decision_models/fakes.py +++ b/s1a/decision_models/fakes.py @@ -4,7 +4,7 @@ from __future__ import annotations from s1a.decision_models.base import DecisionModel -from s1a.decision_models.types import ChoiceQuestion, Json, Observation, Question, Reply, Usage +from s1a.decision_models.types import BoxQuestion, ChoiceQuestion, Json, Observation, Question, Reply, Usage class ScriptedTransport: @@ -57,6 +57,13 @@ async def close(self) -> None: def _answer(self, question: Json) -> Json: if question["type"] == "noul": return {"noul": self._noul.pop(0) if self._noul else 0.5} + if question["type"] == "box": + # a centred half-size box with high confidence; override via answers= for specific cases + return { + "type": "box", + "box": {"top": 0.25, "bottom": 0.75, "left": 0.25, "right": 0.75}, + "confidence": 0.9, + } ids = list(question["criteria"]) key = self._choose or ids[0] return {"choice": key, "probabilities": {i: 1.0 if i == key else 0.0 for i in ids}, "confidence": 1.0} @@ -69,6 +76,7 @@ class ScriptedModel(DecisionModel): name = "scripted" bills_input_tokens = False supports_images = True + question_types = frozenset({"choice", "noul", "box"}) def __init__( self, @@ -105,6 +113,12 @@ async def _decide(self, observation: Observation, questions: dict[str, Question] return Reply(answers=answers, latency_ms=self._latency_ms, usage=self._usage, model=self._model) def _answer(self, question: Question) -> Json: + if isinstance(question, BoxQuestion): + return { + "type": "box", + "box": {"top": 0.25, "bottom": 0.75, "left": 0.25, "right": 0.75}, + "confidence": 0.9, + } if not isinstance(question, ChoiceQuestion): return {"noul": self._noul.pop(0) if self._noul else 0.5} key = self._choose or question.ids[0] diff --git a/s1a/decision_models/jev.py b/s1a/decision_models/jev.py index 204f5c8..835651f 100644 --- a/s1a/decision_models/jev.py +++ b/s1a/decision_models/jev.py @@ -16,6 +16,7 @@ from s1a.decision_models.base import DecisionModel, JevImageTransport, JevTransport from s1a.decision_models.types import ( + BoxQuestion, ChoiceQuestion, Image, Json, @@ -61,6 +62,8 @@ def jev_question(question: Question) -> Json: case NoulQuestion(): criteria = {"criteria": dict(question.criteria)} if question.criteria else {} return {"type": "noul", **criteria, "instructions": {"question": question.question}} + case BoxQuestion(): + return {"type": "box", "instructions": question.target} raise TypeError(f"not a question: {question!r}") @@ -141,12 +144,22 @@ def __init__(self, transport: JevTransport) -> None: self._transport = transport # experimental: only a direct TypeSafe client built with S1A_JEV_IMAGES on carries screenshots (#97) self.supports_images = bool(getattr(transport, "images", False)) + # Box needs an image and the danger-zone header (Jev 1.14+, #98) + if self.supports_images: + self.question_types = frozenset({"choice", "noul", "box"}) @property def model(self) -> str: return self._transport.model async def _decide(self, observation: Observation, questions: dict[str, Question]) -> Reply: + if any(isinstance(q, BoxQuestion) for q in questions.values()) and not ( + self.supports_images and observation.images + ): + raise build_error( + StatusCode.MODEL_SERVICE_CONFIG_ERROR, + error_msg="a box question needs hosted Jev images (S1A_JEV_IMAGES=1 on the direct TypeSafe backend)", + ) body = { "model": self.model, "state": observation.state, diff --git a/s1a/decision_models/laya.py b/s1a/decision_models/laya.py index a0c6f0d..1b13a8e 100644 --- a/s1a/decision_models/laya.py +++ b/s1a/decision_models/laya.py @@ -17,7 +17,7 @@ from s1a.decision_models.base import DecisionModel from s1a.decision_models.jev import jev_question -from s1a.decision_models.types import ChoiceQuestion, Json, Observation, Question, Reply, Usage +from s1a.decision_models.types import ChoiceQuestion, Json, NoulQuestion, Observation, Question, Reply, Usage LAYA_DEFAULT_MODEL = "convaiinnovations/laya" LAYA_DEFAULT_MAX_LEN = 512 # the window Laya assumes when a checkpoint config names none @@ -44,6 +44,8 @@ def laya_question(question: Question) -> Json: question needs a string instruction, and its true/false criteria are native.""" if isinstance(question, ChoiceQuestion): return jev_question(question) + if not isinstance(question, NoulQuestion): + raise TypeError(f"laya answers no {question.type} question") criteria = {"criteria": dict(question.criteria)} if question.criteria else {} return {"type": "noul", "instructions": question.question, **criteria} diff --git a/s1a/decision_models/omnijev.py b/s1a/decision_models/omnijev.py index 8b125a1..5af40bf 100644 --- a/s1a/decision_models/omnijev.py +++ b/s1a/decision_models/omnijev.py @@ -22,7 +22,7 @@ from openjiuwen.core.common.exception.errors import build_error from s1a.decision_models.base import DecisionModel -from s1a.decision_models.types import ChoiceQuestion, Image, Json, Observation, Question, Reply +from s1a.decision_models.types import ChoiceQuestion, Image, Json, NoulQuestion, Observation, Question, Reply OMNIJEV_DEFAULT_CHECKPOINT = "tinnel123/OmniJev-0.8B" OMNIJEV_DEFAULT_REVISION = "v1.1" @@ -87,6 +87,8 @@ def omnijev_question(question: Question, context: str, *, prompt: str = "full") ) criteria = {key: omnijev_option(description) for key, description in question.options.items()} return {"type": "choice", "instructions": "\n".join(lines), "criteria": criteria} + if not isinstance(question, NoulQuestion): + raise TypeError(f"omnijev answers no {question.type} question") lines.append(question.question) for verdict, text in (question.criteria or {}).items(): lines.append(f"{verdict}: {text}") diff --git a/s1a/decision_models/types.py b/s1a/decision_models/types.py index 5d98c46..49ca878 100644 --- a/s1a/decision_models/types.py +++ b/s1a/decision_models/types.py @@ -10,7 +10,7 @@ from typing import Any, Literal, Union Json = dict[str, Any] -QuestionType = Literal["choice", "noul"] +QuestionType = Literal["choice", "noul", "box"] PROVENANCE_KEYS = ("served_by", "url", "request_id", "server_timing", "requested_model", "sent_image") @@ -87,7 +87,26 @@ def type(self) -> QuestionType: return "noul" -Question = Union[ChoiceQuestion, NoulQuestion] +@dataclass(frozen=True) +class BoxQuestion: + """Where is ``target`` in the observation's image? Edges are fractions of width/height from the top-left. + + TypeSafe's Box always returns a box (there is no "not found"); pair it with a ``NoulQuestion`` presence check + and ignore the box when presence is low. Needs an image and the danger-zone header (Jev 1.14+, experimental). + """ + + target: str + + def __post_init__(self) -> None: + if not self.target.strip(): + raise ValueError("a box question needs a target description") + + @property + def type(self) -> QuestionType: + return "box" + + +Question = Union[ChoiceQuestion, NoulQuestion, BoxQuestion] @dataclass(frozen=True) @@ -131,7 +150,34 @@ def as_dict(self) -> Json: return out -Answer = Union[Choice, Noul] +@dataclass(frozen=True) +class Box: + """A validated box answer: four edges in [0, 1] (left < right, top < bottom) and a confidence in [0, 1]. + + ``confidence`` is the backend's own number (TypeSafe's Box confidence); there is no separate contract rescale. + The centre ``((left + right) / 2, (top + bottom) / 2)`` is the natural click point in screenshot fractions. + """ + + top: float + bottom: float + left: float + right: float + confidence: float + backend_confidence: float | None = None # Box.confidence is already the backend's; kept None for Answer parity + + @property + def centre(self) -> tuple[float, float]: + return ((self.left + self.right) / 2.0, (self.top + self.bottom) / 2.0) + + def as_dict(self) -> Json: + return { + "type": "box", + "box": {"top": self.top, "bottom": self.bottom, "left": self.left, "right": self.right}, + "confidence": self.confidence, + } + + +Answer = Union[Choice, Noul, Box] @dataclass(frozen=True) @@ -212,6 +258,12 @@ def noul(self, name: str) -> Noul: raise TypeError(f"{name!r} is a choice answer, not a noul") return answer + def box(self, name: str) -> Box: + answer = self.answers[name] + if not isinstance(answer, Box): + raise TypeError(f"{name!r} is not a box answer") + return answer + @property def provenance(self) -> Json: """Where a served answer came from, for step records: the server's identity, URL and timings, and for Jev the diff --git a/s1a/decision_models/validation.py b/s1a/decision_models/validation.py index 937a8f2..f457bff 100644 --- a/s1a/decision_models/validation.py +++ b/s1a/decision_models/validation.py @@ -17,6 +17,8 @@ from s1a.decision_models.types import ( Answer, + Box, + BoxQuestion, Choice, ChoiceQuestion, Decision, @@ -104,6 +106,39 @@ def validate_noul(fields: Json) -> Noul: ) +def validate_box(fields: Json) -> Box: + """The typed box: four edges in [0, 1] with left < right and top < bottom, plus confidence in [0, 1]. + + TypeSafe's Box confidence is kept as ``Box.confidence`` (there is no separate contract rescale). A missing or + malformed edge or confidence is ``MODEL_CALL_FAILED``. + """ + edges = fields.get("box") + if not isinstance(edges, dict): + raise build_error(StatusCode.MODEL_CALL_FAILED, error_msg="decisions answer is not a box: box is not an object") + try: + top, bottom, left, right = (float(edges[k]) for k in ("top", "bottom", "left", "right")) + except (KeyError, TypeError, ValueError) as exc: + raise build_error( + StatusCode.MODEL_CALL_FAILED, + cause=exc, + error_msg="decisions answer is not a box: edges missing or non-numeric", + ) from exc + confidence = fields.get("confidence") + if not all(unit_interval(v) for v in (top, bottom, left, right, confidence)): + raise build_error( + StatusCode.MODEL_CALL_FAILED, + error_msg="decisions answer is not a box: an edge or confidence is not in [0, 1]", + ) + assert unit_interval(confidence) # narrows Any | None to float for the type checker + if not (left < right and top < bottom): + raise build_error( + StatusCode.MODEL_CALL_FAILED, + error_msg=f"decisions answer is not a box: edges must satisfy left < right and top < bottom, got {edges}", + ) + conf = float(confidence) + return Box(top=top, bottom=bottom, left=left, right=right, confidence=conf, backend_confidence=conf) + + def validate_answer(fields: Any, question: Question) -> Answer: """One answer as its question's type; a non-object answer reads as an empty one and fails on its fields.""" fields = fields if isinstance(fields, dict) else {} @@ -112,6 +147,8 @@ def validate_answer(fields: Any, question: Question) -> Answer: return validate_choice(fields, question.ids) case NoulQuestion(): return validate_noul(fields) + case BoxQuestion(): + return validate_box(fields) raise TypeError(f"not a question: {question!r}") diff --git a/s1a/desktop/env.py b/s1a/desktop/env.py index 2c7e1ed..4917c6b 100644 --- a/s1a/desktop/env.py +++ b/s1a/desktop/env.py @@ -6,11 +6,13 @@ import hashlib import io from collections import Counter +from dataclasses import dataclass from typing import Any, Callable from PIL import Image as PILImage -from s1a.decision_models.types import Image +from s1a.decision_models.base import DecisionModel +from s1a.decision_models.types import Box, BoxQuestion, Image, NoulQuestion, Observation, Question from s1a.desktop.driver import Capture, Driver, DriverError, Element, Snapshot, Window DONE = "done" @@ -22,6 +24,36 @@ _EDITABLE_ROLES = ("textfield", "textarea", "textview", "searchfield", "combobox", "editabletext") _MENU_ROLES = frozenset({"menu", "menubar", "menubaritem", "menuitem"}) +# Uncalibrated defaults for --box-target gating (#98); tune on your own captures (see scripts/measure_box_targets.py). +DEFAULT_BOX_PRESENCE_THRESHOLD = 0.5 +DEFAULT_BOX_CONFIDENCE_THRESHOLD = 0.5 + + +@dataclass(frozen=True) +class BoxHit: + """One box target that passed presence and confidence gates for the current capture.""" + + key: str + description: str + box: Box + presence: float + capture_id: str + + @property + def centre(self) -> tuple[float, float]: + return self.box.centre + + def record(self) -> dict[str, Any]: + return { + "key": self.key, + "description": self.description, + "box": {"top": self.box.top, "bottom": self.box.bottom, "left": self.box.left, "right": self.box.right}, + "presence": self.presence, + "box_confidence": self.box.confidence, + "capture_id": self.capture_id, + "centre": list(self.centre), + } + def clickable(element: Element) -> bool: role = element.role.casefold().removeprefix("ax") @@ -77,6 +109,9 @@ def __init__( text_target: str = "", text_mode: str = "insert", pixel_targets: dict[str, tuple[float, float]] | None = None, + box_targets: dict[str, str] | None = None, + presence_threshold: float = DEFAULT_BOX_PRESENCE_THRESHOLD, + box_confidence_threshold: float = DEFAULT_BOX_CONFIDENCE_THRESHOLD, screenshot: bool = False, ) -> None: self._driver = driver @@ -94,7 +129,11 @@ def __init__( self._verified_value: str | None = None self._text_field: tuple[int, int, str, str] | None = None self._pixel_targets = dict(pixel_targets or {}) - self._screenshot = screenshot or bool(self._pixel_targets) + self._box_targets = dict(box_targets or {}) + self._presence_threshold = presence_threshold + self._box_confidence_threshold = box_confidence_threshold + self._box_hits: dict[str, BoxHit] = {} + self._screenshot = screenshot or bool(self._pixel_targets) or bool(self._box_targets) self._window: Window | None = None self._snapshot: Snapshot | None = None self._visual: dict[str, Any] | None = None @@ -144,6 +183,7 @@ async def _refresh(self) -> None: if self._screenshot and self._snapshot.capture is None: raise DriverError("visual observations require a valid screenshot capture") self._visual = None + self._box_hits = {} # a new capture invalidates prior box centres if self._screenshot: capture = self._snapshot.capture assert capture is not None @@ -216,6 +256,11 @@ async def candidates(self) -> dict[str, str]: for key, (x, y) in self._pixel_targets.items() } ) + for key, hit in self._box_hits.items(): + x, y = hit.centre + offered[f"box:{key}"] = ( + f'Click {key} ("{hit.description}"), at box centre {x:.0%} across and {y:.0%} down the attached screenshot' + ) return {**offered, **RESERVED} async def images(self) -> tuple[Image, ...]: @@ -234,6 +279,8 @@ async def step(self, key: str) -> None: self._planned = {"key": key} if element is not None: self._planned.update(role=element.role, label=element.label, token=element.token) + elif key.startswith("box:") and key.removeprefix("box:") in self._box_hits: + self._planned["box"] = self._box_hits[key.removeprefix("box:")].record() self._ended = "planned" return completion_before_action = self._done_when(self._require_snapshot()) @@ -268,12 +315,22 @@ async def step(self, key: str) -> None: self._presses.append(key) self._update_candidates() return - else: + elif key.startswith("pixel:"): x, y = self._pixel_targets[key.removeprefix("pixel:")] capture = self._require_snapshot().capture assert capture is not None await self._driver.click_at(window, capture, x * capture.width, y * capture.height) self._presses.append(key) + else: + hit = self._box_hits[key.removeprefix("box:")] + x, y = hit.centre + capture = self._require_snapshot().capture + assert capture is not None + if capture.capture_id != hit.capture_id: + raise DriverError(f"stale box target {key!r}: capture changed since locate") + # same window binding / bounds check as pixel targets (click_at rejects out-of-bounds) + await self._driver.click_at(window, capture, x * capture.width, y * capture.height) + self._presses.append(key) await self._refresh() if not self._done_when(self._require_snapshot()): self._completion_stale = False @@ -293,6 +350,74 @@ def score(self) -> float: return 0.0 return 1.0 if self._snapshot is not None and self._done_when(self._snapshot) else 0.0 + @property + def box_targets(self) -> dict[str, str]: + return dict(self._box_targets) + + def box_hit(self, key: str) -> BoxHit | None: + """The resolved hit for ``box:KEY`` or ``KEY``, if still offered for the current capture.""" + name = key.removeprefix("box:") + return self._box_hits.get(name) + + async def locate_box_targets(self, model: DecisionModel) -> dict[str, Any] | None: + """One request: per configured target, a presence ``noul`` and a ``box``; offer those that pass thresholds. + + Targets with presence below ``presence_threshold``, box confidence below ``box_confidence_threshold``, or a + centre outside the capture (fractions not in ``[0, 1)``) are not offered. Returns a locate record for the + step log, or ``None`` when no box targets are configured. + """ + self._box_hits = {} + if not self._box_targets: + return None + capture = self._require_snapshot().capture + if capture is None: + raise DriverError("box targets require a valid screenshot capture") + questions: dict[str, Question] = {} + for key, description in self._box_targets.items(): + questions[f"present:{key}"] = NoulQuestion(f"Is {description} visible in `screenshot`?") + questions[f"box:{key}"] = BoxQuestion(f"{description} in `screenshot`") + decision = await model.decide_many(Observation(await self.observe(), await self.images()), questions) + offered: dict[str, BoxHit] = {} + rejected: list[dict[str, Any]] = [] + for key, description in self._box_targets.items(): + presence = decision.noul(f"present:{key}").p + box = decision.box(f"box:{key}") + cx, cy = box.centre + reason = None + if presence < self._presence_threshold: + reason = "low_presence" + elif box.confidence < self._box_confidence_threshold: + reason = "low_box_confidence" + elif not (0 <= cx < 1 and 0 <= cy < 1): + reason = "centre_outside_capture" + if reason is not None: + rejected.append( + { + "key": key, + "reason": reason, + "presence": presence, + "box_confidence": box.confidence, + "centre": [cx, cy], + } + ) + continue + offered[key] = BoxHit( + key=key, description=description, box=box, presence=presence, capture_id=capture.capture_id + ) + self._box_hits = offered + return { + "offered": {key: hit.record() for key, hit in offered.items()}, + "rejected": rejected, + "presence_threshold": self._presence_threshold, + "box_confidence_threshold": self._box_confidence_threshold, + "model": decision.model, + "ms": decision.latency_ms, + "input_tokens": decision.usage.input_tokens, + "output_tokens": decision.usage.output_tokens, + "usage_known": decision.usage.known, + **decision.provenance, + } + @staticmethod def _token(element: Element) -> str: assert element.token is not None, "only elements with a token are offered" # clickable() guarantees it diff --git a/s1a/tool/models.py b/s1a/tool/models.py index ec1e77a..f91f055 100644 --- a/s1a/tool/models.py +++ b/s1a/tool/models.py @@ -132,6 +132,14 @@ async def _decide(self) -> AssistantMessage: if state.error is not None: return self._stop("BLOCKED", state.error) observation = await env.observe() + locate_record = None + locate_fn = getattr(env, "locate_box_targets", None) + if callable(locate_fn) and getattr(env, "box_targets", None): + try: + locate_record = await locate_fn(self._decision_model) + except BaseError as exc: + state.error = f"box locate failed: {exc}" + return self._stop("BLOCKED", state.error) candidates = await env.candidates() if not candidates: return self._stop("BLOCKED", "no candidates") @@ -168,6 +176,13 @@ async def _decide(self) -> AssistantMessage: } if choice.backend_confidence is not None: tick["backend_confidence"] = round(choice.backend_confidence, 3) + if locate_record is not None: + tick["box_locate"] = locate_record + hit_fn = getattr(env, "box_hit", None) + if callable(hit_fn) and choice.key.startswith("box:"): + hit = hit_fn(choice.key) + if hit is not None: + tick["box"] = hit.record() state.ticks.append(tick) state.blocked = set() # a block, the notices and the plan last one turn state.notices = [] diff --git a/scripts/measure_box_targets.py b/scripts/measure_box_targets.py new file mode 100644 index 0000000..f524aaf --- /dev/null +++ b/scripts/measure_box_targets.py @@ -0,0 +1,116 @@ +# coding: utf-8 +"""Offline helper: presence / box hit rate on saved captures with known rectangles (#98). + +Uncalibrated thresholds (defaults match the desktop agent). Not a general accuracy claim. + +Input JSONL, one object per capture:: + + { + "id": "fixture-1", + "targets": { + "Save": { + "present": true, + "rect": {"top": 0.4, "bottom": 0.7, "left": 0.55, "right": 0.85} + } + }, + "answers": { + "Save": { + "presence": 0.91, + "box": {"top": 0.41, "bottom": 0.69, "left": 0.56, "right": 0.84}, + "box_confidence": 0.88 + } + } + } + +``answers`` are whatever a prior locate step recorded (or a fake). The script does not call TypeSafe. + +Usage:: + + python scripts/measure_box_targets.py captures.jsonl + python scripts/measure_box_targets.py captures.jsonl --presence 0.5 --box-confidence 0.5 +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + + +def iou(a: dict[str, float], b: dict[str, float]) -> float: + top, left = max(a["top"], b["top"]), max(a["left"], b["left"]) + bottom, right = min(a["bottom"], b["bottom"]), min(a["right"], b["right"]) + if bottom <= top or right <= left: + return 0.0 + inter = (bottom - top) * (right - left) + area_a = (a["bottom"] - a["top"]) * (a["right"] - a["left"]) + area_b = (b["bottom"] - b["top"]) * (b["right"] - b["left"]) + return inter / (area_a + area_b - inter) if area_a + area_b > inter else 0.0 + + +def centre_in_rect(box: dict[str, float], rect: dict[str, float]) -> bool: + cx = (box["left"] + box["right"]) / 2.0 + cy = (box["top"] + box["bottom"]) / 2.0 + return rect["left"] <= cx <= rect["right"] and rect["top"] <= cy <= rect["bottom"] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("jsonl", type=Path, help="JSONL of captures with gold rects and recorded answers") + parser.add_argument("--presence", type=float, default=0.5, help="uncalibrated presence (noul) threshold") + parser.add_argument("--box-confidence", type=float, default=0.5, help="uncalibrated box confidence threshold") + parser.add_argument("--iou", type=float, default=0.5, help="IoU needed to count a box hit when present") + args = parser.parse_args(argv) + + presence_correct = presence_total = 0 + box_hits = box_total = 0 + offered_true = offered_false = 0 + for line in args.jsonl.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + row: dict[str, Any] = json.loads(line) + targets, answers = row["targets"], row.get("answers") or {} + for key, gold in targets.items(): + present = bool(gold["present"]) + answer = answers.get(key) or {} + presence = float(answer.get("presence", 0.0)) + predicted_present = presence >= args.presence + presence_total += 1 + presence_correct += int(predicted_present == present) + conf = float(answer.get("box_confidence", 0.0)) + box = answer.get("box") + offered = predicted_present and conf >= args.box_confidence and isinstance(box, dict) + if present: + offered_true += int(offered) + else: + offered_false += int(offered) + if not (present and offered and isinstance(box, dict)): + continue + box_total += 1 + rect = gold["rect"] + hit = centre_in_rect(box, rect) or iou(box, rect) >= args.iou + box_hits += int(hit) + + summary = { + "presence_accuracy": None if not presence_total else presence_correct / presence_total, + "presence_n": presence_total, + "box_hit_rate": None if not box_total else box_hits / box_total, + "box_n": box_total, + "false_offers": offered_false, + "true_offers": offered_true, + "thresholds": { + "presence": args.presence, + "box_confidence": args.box_confidence, + "iou": args.iou, + "note": "uncalibrated defaults; tune on your own captures", + }, + } + json.dump(summary, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_desktop_box_target.py b/tests/test_desktop_box_target.py new file mode 100644 index 0000000..59ce8b7 --- /dev/null +++ b/tests/test_desktop_box_target.py @@ -0,0 +1,251 @@ +# coding: utf-8 +"""Desktop --box-target: Jev Box locate + gated offers (#98), offline with a fake transport/model.""" + +from __future__ import annotations + +import json +import os +from unittest import IsolatedAsyncioTestCase, TestCase +from unittest.mock import AsyncMock, patch + +from s1a.agents import desktop +from s1a.decision_models import ( + Box, + BoxQuestion, + Image, + JevModel, + NoulQuestion, + Observation, + ScriptedModel, + ScriptedTransport, + validate_box, +) +from s1a.desktop.driver import Capture, Snapshot, Window +from s1a.desktop.env import WindowEnv +from s1a.tool.models import EvalState, ToolDecisionModel +from s1a.tool import series +from test_desktop_visual import _png + +WINDOW = Window(1, 2, "Canvas", "Canvas") + + +def _snapshot(capture_id: str = "c1", size: tuple[int, int] = (800, 600)) -> Snapshot: + return Snapshot( + WINDOW, + "s1", + (), + {}, + capture=Capture(capture_id, Image(_png((10, 20, 30))), *size), + ) + + +class TestBoxTypes(TestCase): + def test_validate_box_and_centre(self) -> None: + box = validate_box( + {"type": "box", "box": {"top": 0.2, "bottom": 0.6, "left": 0.1, "right": 0.5}, "confidence": 0.8} + ) + self.assertEqual(box.centre, (0.3, 0.4)) + self.assertEqual(box.confidence, 0.8) + with self.assertRaises(Exception): + validate_box({"box": {"top": 0.6, "bottom": 0.2, "left": 0.1, "right": 0.5}, "confidence": 0.8}) + + def test_jev_wire_box_question(self) -> None: + from s1a.decision_models.jev import jev_question + + self.assertEqual( + jev_question(BoxQuestion("the Save tile in `screenshot`")), + {"type": "box", "instructions": "the Save tile in `screenshot`"}, + ) + + +class TestBoxArguments(TestCase): + def test_parse_and_reject_bad_entries(self) -> None: + self.assertEqual(desktop.parse_box_targets(["Save=the Save tile"]), {"Save": "the Save tile"}) + for entries in (["bad"], ["=desc"], ["k="], ["a=one", "a=two"]): + with self.subTest(entries=entries), self.assertRaises(ValueError): + desktop.parse_box_targets(entries) + + def test_requires_jev_114_with_images(self) -> None: + args = series.parser(desktop.SPEC).parse_args( + [ + "--model", + "jev", + "--rethink", + "off", + "--episodes", + "1", + "--app", + "Canvas", + "--goal", + "click Save", + "--expect", + "Saved", + "--box-target", + "Save=the Save tile", + ] + ) + clean = { + "TYPESAFE_API_KEY": "k", + "TYPESAFE_API_URL": "", + "OPENROUTER_API_KEY": "", + "S1A_JEV_IMAGES": "", + "TYPESAFE_MODEL": "", + } + with patch.dict(os.environ, clean), self.assertRaisesRegex(ValueError, "S1A_JEV_IMAGES"): + desktop.make_series(args) + with patch.dict(os.environ, {**clean, "S1A_JEV_IMAGES": "1", "TYPESAFE_MODEL": "jev-1.13.0"}): + with self.assertRaisesRegex(ValueError, "1.14"): + desktop.make_series(args) + with patch.dict(os.environ, {**clean, "S1A_JEV_IMAGES": "1", "TYPESAFE_MODEL": "jev-1.14.0"}): + with patch.object(desktop, "driver_from_env", return_value=AsyncMock()): + series_obj = desktop.make_series(args) + env = series_obj.env_for(0) + self.assertEqual(env.box_targets, {"Save": "the Save tile"}) + + +class TestBoxLocate(IsolatedAsyncioTestCase): + async def test_absent_and_low_confidence_not_offered_centre_click_and_outside_rejected(self) -> None: + from s1a.desktop.env import BoxHit + + driver = AsyncMock() + driver.find_window.return_value = WINDOW + driver.window_state.return_value = _snapshot() + import math + + async def bounds_check(window, capture, x, y): + if not (math.isfinite(x) and math.isfinite(y) and 0 <= x < capture.width and 0 <= y < capture.height): + raise ValueError("pixel click outside capture bounds") + return {"effect": "confirmed"} + + driver.click_at.side_effect = bounds_check + env = WindowEnv( + driver, + app_name="Canvas", + goal="click Save", + done_when=lambda s: False, + execute=True, + clear_labels=(), + box_targets={"Save": "the Save tile", "Missing": "a unicorn", "Weak": "a faint ghost"}, + presence_threshold=0.5, + box_confidence_threshold=0.5, + ) + await env.reset() + model = ScriptedModel( + answers=[ + { + "present:Save": {"noul": 0.9}, + "box:Save": { + "type": "box", + "box": {"top": 0.4, "bottom": 0.6, "left": 0.7, "right": 0.9}, + "confidence": 0.85, + }, + "present:Missing": {"noul": 0.1}, + "box:Missing": { + "type": "box", + "box": {"top": 0.1, "bottom": 0.2, "left": 0.1, "right": 0.2}, + "confidence": 0.9, + }, + "present:Weak": {"noul": 0.9}, + "box:Weak": { + "type": "box", + "box": {"top": 0.1, "bottom": 0.2, "left": 0.1, "right": 0.2}, + "confidence": 0.2, + }, + } + ] + ) + record = await env.locate_box_targets(model) + assert record is not None + self.assertEqual(set(record["offered"]), {"Save"}) + reasons = {row["key"]: row["reason"] for row in record["rejected"]} + self.assertEqual(reasons["Missing"], "low_presence") + self.assertEqual(reasons["Weak"], "low_box_confidence") + candidates = await env.candidates() + self.assertIn("box:Save", candidates) + self.assertNotIn("box:Missing", candidates) + self.assertNotIn("box:Weak", candidates) + await env.step("box:Save") + # centre of left=0.7,right=0.9,top=0.4,bottom=0.6 → (0.8, 0.5) → (640, 300) on 800x600 + self.assertEqual(driver.click_at.await_args.args[2:], (640.0, 300.0)) + # a box whose centre maps outside the capture is rejected by the same bounds check as pixel targets + env._box_hits["Off"] = BoxHit( + key="Off", + description="off", + box=Box(-0.4, -0.1, -0.5, -0.2, 0.99), # centre outside [0, 1) + presence=0.99, + capture_id="c1", + ) + with self.assertRaisesRegex(ValueError, "bounds"): + await env.step("box:Off") + + async def test_tool_loop_records_box_on_tick(self) -> None: + driver = AsyncMock() + driver.find_window.return_value = WINDOW + capture = Capture("cap-9", Image(_png((1, 2, 3))), 400, 400) + driver.window_state.return_value = Snapshot(WINDOW, "s1", (), {}, capture=capture) + env = WindowEnv( + driver, + app_name="Canvas", + goal="click", + done_when=lambda s: False, + execute=False, + clear_labels=(), + box_targets={"Save": "the Save tile"}, + ) + await env.reset() + model = ScriptedModel( + choose="box:Save", + answers=[ + { + "present:Save": {"noul": 0.95}, + "box:Save": { + "type": "box", + "box": {"top": 0.25, "bottom": 0.75, "left": 0.25, "right": 0.75}, + "confidence": 0.9, + }, + }, + # second decide_many is the choice; ScriptedModel uses choose= + ], + ) + # After first answers dict is consumed, choice uses _answer / choose + state = EvalState() + slot = ToolDecisionModel(env, state, rules="look", decision_model=model, fallback=None) + msg = await slot._decide() + self.assertEqual(json.loads(msg.tool_calls[0].arguments), {"key": "box:Save"}) + self.assertEqual(state.ticks[0]["box"]["capture_id"], "cap-9") + self.assertEqual(state.ticks[0]["box"]["centre"], [0.5, 0.5]) + self.assertIn("box_locate", state.ticks[0]) + await env.step("box:Save") + self.assertEqual((await env.observe())["planned"]["box"]["capture_id"], "cap-9") + + async def test_jev_model_sends_box_with_images(self) -> None: + import io + from PIL import Image as PILImage + + transport = ScriptedTransport( + answers=[ + { + "present:Save": {"noul": 0.8}, + "box:Save": { + "type": "box", + "box": {"top": 0.1, "bottom": 0.3, "left": 0.2, "right": 0.4}, + "confidence": 0.7, + }, + } + ] + ) + transport.images = True + model = JevModel(transport) + buf = io.BytesIO() + PILImage.new("RGB", (320, 320), (40, 50, 60)).save(buf, format="PNG") + decision = await model.decide_many( + Observation({"goal": "click"}, images=(Image(buf.getvalue()),)), + { + "present:Save": NoulQuestion("Is the Save tile visible in `screenshot`?"), + "box:Save": BoxQuestion("the Save tile in `screenshot`"), + }, + ) + self.assertAlmostEqual(decision.box("box:Save").centre[0], 0.3) + self.assertAlmostEqual(decision.box("box:Save").centre[1], 0.2) + self.assertEqual(transport.bodies[0]["questions"]["box:Save"]["type"], "box") + self.assertIn("TypeSafe-Danger-Zone-Images", transport.headers[0])