From 59824c77dd607ff57addf9bbf8d3a8051e84a2f8 Mon Sep 17 00:00:00 2001 From: Andrey Buzin Date: Mon, 2 Nov 2026 11:18:08 -0800 Subject: [PATCH 1/4] Bump typesafe-sdk to 0.7.5 --- pyproject.toml | 4 ++-- uv.lock | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6bfd5968..d2f0e042 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ vercel = [ "vercel-oidc>=0.8.0", ] typesafe = [ - "typesafe-sdk>=0.7.2,<1", + "typesafe-sdk>=0.7.5,<1", ] [project.urls] @@ -82,7 +82,7 @@ dev = [ "vercel-oidc>=0.8.0", "opentelemetry-sdk>=1.43.0", "opentelemetry-exporter-otlp-proto-http>=1.43.0", - "typesafe-sdk>=0.7.2", + "typesafe-sdk>=0.7.5", ] [tool.uv] diff --git a/uv.lock b/uv.lock index d371b1aa..6166b164 100644 --- a/uv.lock +++ b/uv.lock @@ -71,7 +71,7 @@ requires-dist = [ { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'", specifier = ">=1.30" }, { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.30" }, { name = "pydantic", specifier = ">=2.13" }, - { name = "typesafe-sdk", marker = "extra == 'typesafe'", specifier = ">=0.7.2,<1" }, + { name = "typesafe-sdk", marker = "extra == 'typesafe'", specifier = ">=0.7.5,<1" }, { name = "typing-extensions", specifier = ">=4.15.0" }, { name = "vercel-oidc", marker = "extra == 'vercel'", specifier = ">=0.8.0" }, ] @@ -92,7 +92,7 @@ dev = [ { name = "rich", specifier = ">=14.2.0" }, { name = "ruff", specifier = "~=0.8.0" }, { name = "ty", specifier = "~=0.0.37" }, - { name = "typesafe-sdk", specifier = ">=0.7.2" }, + { name = "typesafe-sdk", specifier = ">=0.7.5" }, { name = "vercel-oidc", specifier = ">=0.8.0" }, ] @@ -1354,18 +1354,21 @@ wheels = [ [[package]] name = "typesafe-sdk" -version = "0.7.2" +version = "0.7.5" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "anyio" }, { name = "httpx2" }, + { name = "packaging" }, { name = "pydantic" }, { name = "pydantic-core" }, { name = "tenacity" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/42/ac/8bad2c02c038b85039cdb198ba106650d54bade0c0c986c68dc6e9cef8c3/typesafe_sdk-0.7.2.tar.gz", hash = "sha256:2434f4af5cdfc45d6ab5ff27048e8b257207f786b60456e69995e345b32e77e1", size = 23169, upload-time = "2026-09-26T21:20:24.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/26/4d/383aa14f4cfe5fb41023ea1cab52d7f79cc31ab200ee7314317cf89db744/typesafe_sdk-0.7.5.tar.gz", hash = "sha256:1eeeb0dc80a7161d84896db42a5016dc0a7c8872fa16540b70163e5357b6d398", size = 34152, upload-time = "2026-10-14T21:21:42.837Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/99/5e/2b97ba9b0b77612e3a480ddb60c0c0325b4809ef57a7b565341d65ef7bec/typesafe_sdk-0.7.2-py3-none-any.whl", hash = "sha256:0a961148187d52e18276ed7f2d02617cfac48e3b97673cb631a8749397d43d1e", size = 36448, upload-time = "2026-09-26T21:20:23.511Z" }, + { url = "https://files.pythonhosted.org/packages/1c/f6/bee4c3b49570c2d312057c9e37c1446cf37eb1b86bf5872f7a797e803326/typesafe_sdk-0.7.5-py3-none-any.whl", hash = "sha256:967a8b61494d536abb55ee3f3a2b10e81685cb6baad6ccdcab588bdc6c9b86d2", size = 49948, upload-time = "2026-10-14T21:21:41.337Z" }, ] [[package]] From e128db8404bb2d4e607a9d736dcd58fe112045a1 Mon Sep 17 00:00:00 2001 From: Andrey Buzin Date: Mon, 2 Nov 2026 13:23:19 -0800 Subject: [PATCH 2/4] Accept images in decisions Accept `FilePart`s anywhere in decision state and in question `instructions` and `criteria`, alongside text and JSON, e.g. `{"photo": ai.file_part(data), "note": "..."}`. `DecisionInput` is now text, a file, or JSON with files at any depth. It validates like `pydantic.JsonValue`: an invalid value gives one error at its path. A plain `str` is always text, and a dict shaped like a file stays JSON. The TypeSafe protocol turns each file part into an SDK `Image`, which the SDK sends as an asset with a reference in its place, and sends the image opt-in header. URLs are downloaded, a repeated file part is sent once, and files that are not images are rejected. AI Gateway v4 has no wire format for files in decision requests yet, so it rejects them before it sends the request. --- src/ai/ops/experimental/decisions.py | 64 +++++++- src/ai/providers/ai_gateway/protocol/v4.py | 16 ++ src/ai/providers/typesafe/_sdk.py | 1 + src/ai/providers/typesafe/protocol.py | 47 +++++- tests/ops/experimental/test_decisions.py | 143 ++++++++++++++++++ .../ai_gateway/protocol/test_v4_decisions.py | 28 ++++ tests/providers/typesafe/test_protocol.py | 99 +++++++++++- 7 files changed, 392 insertions(+), 6 deletions(-) diff --git a/src/ai/ops/experimental/decisions.py b/src/ai/ops/experimental/decisions.py index 018c6654..b3825381 100644 --- a/src/ai/ops/experimental/decisions.py +++ b/src/ai/ops/experimental/decisions.py @@ -22,6 +22,10 @@ class Answers(pydantic.BaseModel): output_type=Answers, ) result.value.requests_refund + +Images go anywhere in the state or question text as file parts:: + + state = {"photo": ai.file_part(photo_bytes), "note": "Delivered today."} """ from __future__ import annotations @@ -34,16 +38,67 @@ class Answers(pydantic.BaseModel): import pydantic from ... import experimental_telemetry as telemetry +from ...types import messages from .. import items if TYPE_CHECKING: from ...models.core import model as model_ -type DecisionInput = ( - str | dict[str, pydantic.JsonValue] | list[pydantic.JsonValue] +_JSON_TYPES = frozenset((str, int, float, bool, list, dict)) + + +def _decision_value_tag(value: Any) -> str: + # Like pydantic.JsonValue: pick one union member by type, so bad input + # gives one error at its path. Exact types take the fast path. Subclasses + # validate as their base type; bool goes before int because bool + # subclasses int. + exact = type(value) + if exact in _JSON_TYPES: + return exact.__name__ + for type_ in (bool, int, float, str, list, dict, messages.FilePart): + if isinstance(value, type_): + return type_.__name__ + return "None" if value is None else "invalid" + + +_VALUE_DISCRIMINATOR = pydantic.Discriminator( + _decision_value_tag, + custom_error_type="invalid-decision-value", + custom_error_message="input was not text, JSON, or a FilePart", +) +_INPUT_DISCRIMINATOR = pydantic.Discriminator( + _decision_value_tag, + custom_error_type="invalid-decision-input", + custom_error_message="input was not text, a FilePart, a dict, or a list", ) +# Static type checkers ignore the Annotated metadata and see a plain union. +type _DecisionValue = Annotated[ + Annotated[str, pydantic.Tag("str")] + | Annotated[int, pydantic.Tag("int")] + | Annotated[float, pydantic.Tag("float")] + | Annotated[bool, pydantic.Tag("bool")] + | Annotated[messages.FilePart, pydantic.Tag("FilePart")] + | Annotated[list[_DecisionValue], pydantic.Tag("list")] + | Annotated[dict[str, _DecisionValue], pydantic.Tag("dict")] + | Annotated[None, pydantic.Tag("None")], + _VALUE_DISCRIMINATOR, +] + +type DecisionInput = Annotated[ + Annotated[str, pydantic.Tag("str")] + | Annotated[messages.FilePart, pydantic.Tag("FilePart")] + | Annotated[dict[str, _DecisionValue], pydantic.Tag("dict")] + | Annotated[list[_DecisionValue], pydantic.Tag("list")], + _INPUT_DISCRIMINATOR, +] +"""Text, a file, or JSON data with files at any position. + +Files are :class:`~ai.types.messages.FilePart` objects, e.g. from +:func:`ai.file_part`. A plain ``str`` is always text, never a URL. +""" + class PredicateCriteria(TypedDict, total=False): """Optional descriptions of when a predicate answer is true or false.""" @@ -241,6 +296,11 @@ async def decide( value, or pass a question mapping to receive an answer mapping. Each choice, score, or predicate question is validated against its corresponding answer. + ``state``, question ``instructions``, and ``criteria`` accept files + (:class:`~ai.types.messages.FilePart`) alongside text and JSON, e.g. + ``{"photo": ai.file_part(data), "note": "..."}``. Providers reject files + they cannot send. + Experimental: not part of the stable API, may change or be removed. """ # Validate the shared state before dispatching it to a provider. diff --git a/src/ai/providers/ai_gateway/protocol/v4.py b/src/ai/providers/ai_gateway/protocol/v4.py index 480163ad..15233283 100644 --- a/src/ai/providers/ai_gateway/protocol/v4.py +++ b/src/ai/providers/ai_gateway/protocol/v4.py @@ -546,6 +546,22 @@ async def decide( params: ops.experimental.DecideParams, ) -> ops.items.Item[dict[str, Any]]: """Hit ``/evaluation-model`` and return raw decision answers.""" + + # The v4 spec does not define files in decision requests yet. + def has_file(value: Any) -> bool: + if isinstance(value, types.messages.FilePart): + return True + if isinstance(value, dict): + return any(has_file(item) for item in value.values()) + if isinstance(value, list): + return any(has_file(item) for item in value) + return False + + if has_file(state) or any( + has_file(dict(question)) for question in questions.values() + ): + raise ValueError("AI Gateway decisions do not support files") + wire_questions: dict[str, dict[str, Any]] = {} for question_id, question in questions.items(): wire_question = question.model_dump(mode="json", by_alias=True) diff --git a/src/ai/providers/typesafe/_sdk.py b/src/ai/providers/typesafe/_sdk.py index 7900fd52..94999319 100644 --- a/src/ai/providers/typesafe/_sdk.py +++ b/src/ai/providers/typesafe/_sdk.py @@ -12,6 +12,7 @@ class TypeSafeSDK(Protocol): AsyncTypeSafeClient: type[typesafe_sdk.AsyncTypeSafeClient] + Image: type[typesafe_sdk.Image] TypeSafeError: type[typesafe_sdk.TypeSafeError] TypeSafeAPIError: type[typesafe_sdk.TypeSafeAPIError] TypeSafeAPIConnectionError: type[typesafe_sdk.TypeSafeAPIConnectionError] diff --git a/src/ai/providers/typesafe/protocol.py b/src/ai/providers/typesafe/protocol.py index 40ec30a4..ef9704aa 100644 --- a/src/ai/providers/typesafe/protocol.py +++ b/src/ai/providers/typesafe/protocol.py @@ -2,10 +2,12 @@ from __future__ import annotations +import base64 from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Literal -from ... import ops +from ... import ops, types +from ...models import core from ...types import usage as usage_ from .. import base from . import _sdk, errors @@ -43,12 +45,47 @@ async def decide( """Hit ``/v1/systemone`` and return raw decision answers.""" typesafe = _sdk.import_sdk(provider=provider) + # Files become SDK images wherever they are. The SDK moves them to + # the request's assets and sends a reference in their place. One file + # part gives one image, so a repeated file is sent once. + images: dict[int, typesafe_sdk.Image] = {} + + async def to_wire(value: Any) -> Any: + match value: + case types.messages.FilePart(media_type=media_type, data=data): + if not media_type.startswith("image/"): + raise ValueError( + f"Unsupported media type for TypeSafe: {media_type}" + ) + if id(value) not in images: + match data: + case str() if types.media.is_downloadable_url(data): + data, _ = await core.helpers.files.download( + data + ) + case str(): + data = base64.b64decode( + types.media.data_to_base64(data) + ) + images[id(value)] = typesafe.Image(data) + return images[id(value)] + case dict(): + return { + key: await to_wire(item) for key, item in value.items() + } + case list(): + return [await to_wire(item) for item in value] + case _: + return value + + wire_state = await to_wire(state) + # The native wire format matches the question models, except that # TypeSafe calls the predicate primitive "noul" and an absent # predicate criteria is omitted rather than sent as null. wire_questions: dict[str, Any] = {} for question_id, question in questions.items(): - wire_question = question.model_dump(mode="json") + wire_question = await to_wire(dict(question)) if wire_question["type"] == "predicate": wire_question["type"] = "noul" if wire_question.get("criteria", ...) is None: @@ -62,9 +99,13 @@ async def decide( try: response = await client.system_one( - state, + wire_state, wire_questions, model=model.id, + # TypeSafe accepts images only with this opt-in header. + extra_headers=( + {"TypeSafe-Danger-Zone-Images": "true"} if images else None + ), extra_body=options, ) except typesafe.TypeSafeError as exc: diff --git a/tests/ops/experimental/test_decisions.py b/tests/ops/experimental/test_decisions.py index 7c3adece..5cc6af1b 100644 --- a/tests/ops/experimental/test_decisions.py +++ b/tests/ops/experimental/test_decisions.py @@ -99,6 +99,35 @@ async def decide( return ops.Item(value=self.answers) +class RecordingDecisionProvider(models.Provider): + provider_class_id: Literal["test-recording-decision-provider"] = ( + "test-recording-decision-provider" + ) + name: str = "mock-recording-decision" + default_base_url: str = "http://mock.test" + api_key_env: str | None = None + requests: list[Any] = pydantic.Field(default_factory=list) + + async def list_models(self) -> list[str]: + return [] + + async def decide( + self, + model: models.Model, + state: ops.experimental.DecisionInput, + questions: Mapping[str, DecisionQuestion], + *, + params: ops.experimental.DecideParams, + ) -> ops.Item[dict[str, Any]]: + self.requests.append((state, dict(questions))) + return ops.Item( + value={ + name: {"type": "predicate", "probability": 0.5} + for name in questions + } + ) + + def questions() -> Questions: return Questions( department=ops.experimental.ChoiceQuestion( @@ -428,6 +457,120 @@ async def test_decide_requires_matching_answer_fields() -> None: ) +async def test_decide_accepts_files_in_state_and_questions() -> None: + photo = ai.file_part(b"\x89PNG\r\n\x1a\n", media_type="image/png") + reference = ai.file_part("https://example.com/reference.jpg") + provider = RecordingDecisionProvider() + model = models.Model(id="mock-decision-model", provider=provider) + question = ops.experimental.PredicateQuestion( + instructions=[ + "Is the item in `photo` the same as this one?", + reference, + ], + criteria={"true": reference, "false": None}, + ) + + result = await ops.experimental.decide( + model, + {"photo": photo, "notes": ["Delivered today.", photo]}, + {"same_item": question}, + ) + + [(state, sent)] = provider.requests + assert state == {"photo": photo, "notes": ["Delivered today.", photo]} + assert state["photo"] is photo + assert sent["same_item"].instructions[1] is reference + assert sent["same_item"].criteria == {"true": reference, "false": None} + answer = result.value["same_item"] + assert isinstance(answer, ops.experimental.PredicateAnswer) + assert answer.probability == 0.5 + + +async def test_decide_accepts_a_file_as_state() -> None: + photo = ai.file_part(b"\x89PNG\r\n\x1a\n", media_type="image/png") + provider = RecordingDecisionProvider() + model = models.Model(id="mock-decision-model", provider=provider) + + await ops.experimental.decide( + model, + photo, + { + "damaged": ops.experimental.PredicateQuestion( + instructions="Damaged?" + ) + }, + ) + + [(state, _)] = provider.requests + assert state is photo + + +async def test_decide_keeps_file_shaped_dicts_as_json() -> None: + provider = RecordingDecisionProvider() + model = models.Model(id="mock-decision-model", provider=provider) + state: ops.experimental.DecisionInput = { + "kind": "file", + "data": "aGk=", + "media_type": "image/png", + } + + await ops.experimental.decide( + model, + state, + { + "damaged": ops.experimental.PredicateQuestion( + instructions="Damaged?" + ) + }, + ) + + [(sent, _)] = provider.requests + assert sent == state + assert not isinstance(sent, ai.types.messages.FilePart) + + +@pytest.mark.parametrize( + ("state", "loc", "error_type"), + [ + (1, (), "invalid-decision-input"), + ((1,), (), "invalid-decision-input"), + ( + {"a": [1, {"b": object()}]}, + ("dict", "a", "list", 1, "dict", "b"), + "invalid-decision-value", + ), + ], +) +async def test_decide_reports_one_error_per_invalid_value( + state: Any, loc: tuple[str | int, ...], error_type: str +) -> None: + model = models.Model(id="mock-decision-model", provider=DecisionProvider()) + + with pytest.raises(pydantic.ValidationError) as exc_info: + await ops.experimental.decide( + model, + cast("ops.experimental.DecisionInput", state), + questions(), + output_type=Answers, + ) + + [error] = exc_info.value.errors() + assert error["loc"] == loc + assert error["type"] == error_type + + +async def test_decide_rejects_bare_bytes() -> None: + model = models.Model(id="mock-decision-model", provider=DecisionProvider()) + + with pytest.raises(pydantic.ValidationError): + await ops.experimental.decide( + model, + cast("ops.experimental.DecisionInput", {"photo": b"\x89PNG"}), + questions(), + output_type=Answers, + ) + + async def test_decide_rejects_cyclic_state() -> None: state: list[Any] = [] state.append(state) diff --git a/tests/providers/ai_gateway/protocol/test_v4_decisions.py b/tests/providers/ai_gateway/protocol/test_v4_decisions.py index e305ccb3..8588dc39 100644 --- a/tests/providers/ai_gateway/protocol/test_v4_decisions.py +++ b/tests/providers/ai_gateway/protocol/test_v4_decisions.py @@ -335,3 +335,31 @@ def handler(request: httpx.Request) -> httpx.Response: ), output_type=PredicateAnswers, ) + + +@pytest.mark.parametrize("where", ["state", "instructions", "criteria"]) +async def test_decide_rejects_files(where: str) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(500) + + photo = ai.file_part(b"\x89PNG\r\n\x1a\n") + state: ops.experimental.DecisionInput = ( + {"photo": photo} if where == "state" else "state" + ) + question = ops.experimental.PredicateQuestion( + instructions=["Is this damaged?", photo] + if where == "instructions" + else "Is this damaged?", + criteria={"true": photo} if where == "criteria" else None, + ) + + with pytest.raises(ValueError, match="do not support files"): + await ops.experimental.decide( + mock_model(httpx.MockTransport(handler), model_id=_MODEL_ID), + state, + {"damaged": question}, + ) + assert requests == [] diff --git a/tests/providers/typesafe/test_protocol.py b/tests/providers/typesafe/test_protocol.py index f5ae24e6..5f31857b 100644 --- a/tests/providers/typesafe/test_protocol.py +++ b/tests/providers/typesafe/test_protocol.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 import json from typing import Any @@ -14,11 +15,13 @@ def _model(handler: Any) -> ai.Model: - # Disable SDK retries so error tests don't back off. + # Disable SDK retries so error tests don't back off, and compression so + # tests can read large request bodies. provider = TypeSafeProvider( client=typesafe_sdk.AsyncTypeSafeClient( api_key="ts-test", retry=typesafe_sdk.RetryPolicy(max_retries=0), + compression=False, transport=httpx2.MockTransport(handler), ), ) @@ -94,6 +97,7 @@ def handler(request: httpx2.Request) -> httpx2.Response: [request] = requests assert request.url == "https://api.typesafe.ai/v1/systemone" assert request.headers["authorization"] == "Bearer ts-test" + assert "typesafe-danger-zone-images" not in request.headers assert json.loads(request.content) == { "state": "Help! My payouts have been failing for 3 days.", "model": "jev-latest", @@ -168,6 +172,99 @@ def handler(request: httpx2.Request) -> httpx2.Response: assert result.provider_metadata == {"typesafe": {"model": "jev-1.13.0"}} +_PNG = b"\x89PNG\r\n\x1a\n-photo" +_JPEG = b"\xff\xd8\xff-reference" + + +def _multipart(request: httpx2.Request) -> tuple[dict[str, Any], list[bytes]]: + """Return the JSON request part and the asset bodies, in order.""" + boundary = request.headers["content-type"].split("boundary=")[1] + sections = request.content.split(b"--" + boundary.encode())[1:-1] + bodies = [section.split(b"\r\n\r\n", 1)[1][:-2] for section in sections] + return json.loads(bodies[0]), bodies[1:] + + +async def test_decide_sends_images_as_assets( + monkeypatch: pytest.MonkeyPatch, +) -> None: + requests: list[httpx2.Request] = [] + downloads: list[str] = [] + + async def download(url: str) -> tuple[bytes, str | None]: + downloads.append(url) + return _JPEG, "image/jpeg" + + monkeypatch.setattr(ai.models.core.helpers.files, "download", download) + + def handler(request: httpx2.Request) -> httpx2.Response: + requests.append(request) + return httpx2.Response( + 200, + json=_response( + { + "same": {"type": "noul", "noul": 0.9}, + "coat": { + "type": "choice", + "choice": "short", + "probabilities": {"short": 0.7, "long": 0.3}, + "confidence": 0.8, + }, + } + ), + ) + + # The same photo appears twice and as base64 text in a data URL. + photo = ai.file_part(_PNG) + inline = ai.file_part( + "data:image/png;base64," + base64.b64encode(_PNG).decode() + ) + reference = ai.file_part("https://example.com/reference.jpg") + + await decisions.decide( + _model(handler), + {"photo": photo, "notes": ["Found today.", photo, inline]}, + { + "same": decisions.PredicateQuestion( + instructions=["Is `photo` the same dog as this?", reference], + ), + "coat": decisions.ChoiceQuestion( + instructions="What is the coat like?", + criteria={"short": reference, "long": None}, + ), + }, + ) + + [request] = requests + assert request.headers["typesafe-danger-zone-images"] == "true" + body, assets = _multipart(request) + + # One asset per file part; each reference stands where its file was. + photo_ref = body["state"]["photo"] + inline_ref = body["state"]["notes"][2] + reference_ref = body["questions"]["same"]["instructions"][1] + assert body["state"]["notes"] == ["Found today.", photo_ref, inline_ref] + assert body["questions"]["coat"]["criteria"] == { + "short": reference_ref, + "long": None, + } + assert body["questions"]["same"]["type"] == "noul" + assert set(body["assets"]) == {photo_ref, inline_ref, reference_ref} + assert sorted(assets) == sorted([_PNG, _PNG, _JPEG]) + assert downloads == ["https://example.com/reference.jpg"] + + +async def test_decide_rejects_non_image_files() -> None: + model = _model(lambda request: httpx2.Response(500)) + document = ai.file_part(b"%PDF-1.7", media_type="application/pdf") + + with pytest.raises(ValueError, match="application/pdf"): + await decisions.decide( + model, + {"contract": document}, + {"q": decisions.PredicateQuestion(instructions="Signed?")}, + ) + + async def test_decide_rejects_non_mapping_provider_options() -> None: model = _model(lambda request: httpx2.Response(500)) From 41be4f3678d125e3b6bd45509dad4c3beb46b58a Mon Sep 17 00:00:00 2001 From: Andrey Buzin Date: Mon, 2 Nov 2026 13:23:22 -0800 Subject: [PATCH 3/4] Add decision image examples The TypeSafe example puts images inside structured state and in a score criterion. The OpenAI example puts them where the OpenAI Decisions API accepts them: in a list of text and file parts as state, and in one question's instructions. The OpenAI example raises `NotImplementedError` until the OpenAI decisions protocol lands. TypeSafe is not on models.dev yet, so its example builds the model with `TypeSafeProvider()`. --- examples/models/openai/decisions_images.py | 87 ++++++++++++++++++ examples/models/typesafe/decisions_images.py | 97 ++++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 examples/models/openai/decisions_images.py create mode 100644 examples/models/typesafe/decisions_images.py diff --git a/examples/models/openai/decisions_images.py b/examples/models/openai/decisions_images.py new file mode 100644 index 00000000..62a3f94f --- /dev/null +++ b/examples/models/openai/decisions_images.py @@ -0,0 +1,87 @@ +"""Ask decision questions about images with OpenAI ``gpt-6-luna``. + +Set ``OPENAI_API_KEY`` and run: + + uv run python examples/models/openai/decisions_images.py + +OpenAI decisions take images in two places: + +* in the state, as a list of text and file parts (one user message), and +* in a question's instructions, to give one question its own image. + +OpenAI does not accept images nested inside JSON state or in criteria. +""" + +import asyncio +import pathlib + +import pydantic + +import ai + + +class DeliveryQuestions(pydantic.BaseModel): + damaged: ai.ops.experimental.PredicateQuestion + condition: ai.ops.experimental.ScoreQuestion + matches_reference: ai.ops.experimental.PredicateQuestion + + +class DeliveryAnswers(pydantic.BaseModel): + damaged: ai.ops.experimental.PredicateAnswer + condition: ai.ops.experimental.ScoreAnswer + matches_reference: ai.ops.experimental.PredicateAnswer + + +async def main() -> None: + model = ai.get_model("openai:gpt-6-luna") + if not model.provider.is_configured(): + print("Set OPENAI_API_KEY to run this example.") + return + + image_path = ( + pathlib.Path(__file__).parents[2] / "media" / "sample_image.jpg" + ) + + # Shared evidence for all questions: text and an image, like the content + # of a user message. + state: ai.ops.experimental.DecisionInput = [ + "A customer uploaded this photo of a delivered item. This is photo A.", + ai.file_part(image_path.read_bytes()), + ] + + result = await ai.ops.experimental.decide( + model, + state, + DeliveryQuestions( + damaged=ai.ops.experimental.PredicateQuestion( + instructions=( + "Does the item in photo A have visible damage, such as a " + "crack, tear, or dent? Ignore shadows." + ), + ), + condition=ai.ops.experimental.ScoreQuestion( + instructions="What is the condition of the item in photo A?", + criteria=["Broken", "Worn", "Like new"], + ), + # This question has its own image. Here it is the same file, so + # the answer should be close to 100%. + matches_reference=ai.ops.experimental.PredicateQuestion( + instructions=[ + "Is candidate photo B a copy or variant of photo A, " + "including recompression or resizing?", + ai.file_part(image_path.read_bytes()), + ], + ), + ), + output_type=DeliveryAnswers, + ) + + answers = result.value + print("damaged:", f"{answers.damaged.probability:.0%}") + print("condition:", answers.condition.score) + print("matches reference:", f"{answers.matches_reference.probability:.0%}") + print("usage:", result.usage) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/models/typesafe/decisions_images.py b/examples/models/typesafe/decisions_images.py new file mode 100644 index 00000000..82158fbc --- /dev/null +++ b/examples/models/typesafe/decisions_images.py @@ -0,0 +1,97 @@ +"""Ask decision questions about images with TypeSafe. + +Set ``TYPESAFE_API_KEY`` and run: + + uv run python examples/models/typesafe/decisions_images.py + +Images are file parts. TypeSafe accepts them anywhere text can go: inside +JSON state, in a question's instructions, and in its criteria. Questions +refer to state fields by name in backticks. +""" + +import asyncio +import pathlib + +import pydantic + +import ai +from ai.providers.typesafe import TypeSafeProvider + + +class ListingQuestions(pydantic.BaseModel): + matches_listing: ai.ops.experimental.PredicateQuestion + lighting: ai.ops.experimental.ChoiceQuestion + photo_quality: ai.ops.experimental.ScoreQuestion + + +class ListingAnswers(pydantic.BaseModel): + matches_listing: ai.ops.experimental.PredicateAnswer + lighting: ai.ops.experimental.ChoiceAnswer + photo_quality: ai.ops.experimental.ScoreAnswer + + +async def main() -> None: + model = ai.Model(id="jev-latest", provider=TypeSafeProvider()) + if not model.provider.is_configured(): + print("Set TYPESAFE_API_KEY to run this example.") + return + + image_path = ( + pathlib.Path(__file__).parents[2] / "media" / "sample_image.jpg" + ) + photo = ai.file_part(image_path.read_bytes(), media_type="image/jpeg") + + # The image sits inside structured state, next to the data it belongs to. + listing: ai.ops.experimental.DecisionInput = { + "listing": { + "title": "Scenic print, framed", + "price_usd": 120, + "photo": photo, + }, + "buyer_report": { + "message": "This is what arrived.", + "photo": photo, + }, + } + + result = await ai.ops.experimental.decide( + model, + listing, + ListingQuestions( + matches_listing=ai.ops.experimental.PredicateQuestion( + instructions=( + "Does `buyer_report.photo` show the same item as " + "`listing.photo`?" + ), + ), + lighting=ai.ops.experimental.ChoiceQuestion( + instructions="What is the lighting in `listing.photo` like?", + criteria={ + "daylight": "Natural daylight", + "artificial": "Lamps or other artificial light", + "dark": "Little or no light", + }, + ), + # A criterion can be an image: here, the listing photo is the + # example of the top quality level. + photo_quality=ai.ops.experimental.ScoreQuestion( + instructions="How good is `buyer_report.photo` as evidence?", + criteria=[ + "Unusable: blurry, dark, or the item is not visible", + "Usable: the item is visible but details are unclear", + ["As clear as this reference photo:", photo], + ], + ), + ), + output_type=ListingAnswers, + ) + + answers = result.value + print("matches listing:", f"{answers.matches_listing.probability:.0%}") + print("lighting:", answers.lighting.choice) + print("photo quality:", answers.photo_quality.score) + print("usage:", result.usage) + + +if __name__ == "__main__": + asyncio.run(main()) From 2f6dbb6697c20454ffba56d6a50e6e45c96a6b3e Mon Sep 17 00:00:00 2001 From: Andrey Buzin Date: Mon, 2 Nov 2026 14:29:52 -0800 Subject: [PATCH 4/4] Accept read-only containers in decision inputs Statically type DecisionInput with covariant Sequence and Mapping so a typed list or dict such as list[str | int] is accepted without a cast. Validation still stores every sequence as a list and every mapping as a dict, so providers keep walking plain JSON shapes. Binary data is still rejected rather than read as a list of ints. --- src/ai/ops/experimental/decisions.py | 87 ++++++++++++++++-------- tests/ops/experimental/test_decisions.py | 37 +++++++++- 2 files changed, 96 insertions(+), 28 deletions(-) diff --git a/src/ai/ops/experimental/decisions.py b/src/ai/ops/experimental/decisions.py index b3825381..eee8e94d 100644 --- a/src/ai/ops/experimental/decisions.py +++ b/src/ai/ops/experimental/decisions.py @@ -32,7 +32,7 @@ class Answers(pydantic.BaseModel): import dataclasses import warnings -from collections.abc import Iterable, Mapping +from collections.abc import Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Annotated, Any, Literal, TypedDict, overload import pydantic @@ -52,13 +52,20 @@ def _decision_value_tag(value: Any) -> str: # Like pydantic.JsonValue: pick one union member by type, so bad input # gives one error at its path. Exact types take the fast path. Subclasses # validate as their base type; bool goes before int because bool - # subclasses int. + # subclasses int. Other sequences and mappings validate as list and dict, + # except binary data, which is never a list of ints. exact = type(value) if exact in _JSON_TYPES: return exact.__name__ - for type_ in (bool, int, float, str, list, dict, messages.FilePart): + for type_ in (bool, int, float, str, messages.FilePart): if isinstance(value, type_): return type_.__name__ + if isinstance(value, Mapping): + return "dict" + if isinstance(value, Sequence) and not isinstance( + value, bytes | bytearray | memoryview + ): + return "list" return "None" if value is None else "invalid" @@ -70,34 +77,60 @@ def _decision_value_tag(value: Any) -> str: _INPUT_DISCRIMINATOR = pydantic.Discriminator( _decision_value_tag, custom_error_type="invalid-decision-input", - custom_error_message="input was not text, a FilePart, a dict, or a list", + custom_error_message=( + "input was not text, a FilePart, a mapping, or a sequence" + ), ) -# Static type checkers ignore the Annotated metadata and see a plain union. -type _DecisionValue = Annotated[ - Annotated[str, pydantic.Tag("str")] - | Annotated[int, pydantic.Tag("int")] - | Annotated[float, pydantic.Tag("float")] - | Annotated[bool, pydantic.Tag("bool")] - | Annotated[messages.FilePart, pydantic.Tag("FilePart")] - | Annotated[list[_DecisionValue], pydantic.Tag("list")] - | Annotated[dict[str, _DecisionValue], pydantic.Tag("dict")] - | Annotated[None, pydantic.Tag("None")], - _VALUE_DISCRIMINATOR, -] +if TYPE_CHECKING: + # Read-only containers are covariant, so a ``list[str]`` or a + # ``dict[str, int]`` is accepted as is. + type _DecisionValue = ( + str + | int + | float + | bool + | messages.FilePart + | Sequence[_DecisionValue] + | Mapping[str, _DecisionValue] + | None + ) + type DecisionInput = ( + str + | messages.FilePart + | Mapping[str, _DecisionValue] + | Sequence[_DecisionValue] + ) + """Text, a file, or JSON data with files at any position. -type DecisionInput = Annotated[ - Annotated[str, pydantic.Tag("str")] - | Annotated[messages.FilePart, pydantic.Tag("FilePart")] - | Annotated[dict[str, _DecisionValue], pydantic.Tag("dict")] - | Annotated[list[_DecisionValue], pydantic.Tag("list")], - _INPUT_DISCRIMINATOR, -] -"""Text, a file, or JSON data with files at any position. + Files are :class:`~ai.types.messages.FilePart` objects, e.g. from + :func:`ai.file_part`. A plain ``str`` is always text, never a URL. Any + sequence becomes a list and any mapping a dict. + """ +else: + # Validation stores every sequence as a list and every mapping as a dict. + _AS_LIST = pydantic.BeforeValidator(list) + _AS_DICT = pydantic.BeforeValidator(dict) + + type _DecisionValue = Annotated[ + Annotated[str, pydantic.Tag("str")] + | Annotated[int, pydantic.Tag("int")] + | Annotated[float, pydantic.Tag("float")] + | Annotated[bool, pydantic.Tag("bool")] + | Annotated[messages.FilePart, pydantic.Tag("FilePart")] + | Annotated[list[_DecisionValue], _AS_LIST, pydantic.Tag("list")] + | Annotated[dict[str, _DecisionValue], _AS_DICT, pydantic.Tag("dict")] + | Annotated[None, pydantic.Tag("None")], + _VALUE_DISCRIMINATOR, + ] -Files are :class:`~ai.types.messages.FilePart` objects, e.g. from -:func:`ai.file_part`. A plain ``str`` is always text, never a URL. -""" + type DecisionInput = Annotated[ + Annotated[str, pydantic.Tag("str")] + | Annotated[messages.FilePart, pydantic.Tag("FilePart")] + | Annotated[dict[str, _DecisionValue], _AS_DICT, pydantic.Tag("dict")] + | Annotated[list[_DecisionValue], _AS_LIST, pydantic.Tag("list")], + _INPUT_DISCRIMINATOR, + ] class PredicateCriteria(TypedDict, total=False): diff --git a/tests/ops/experimental/test_decisions.py b/tests/ops/experimental/test_decisions.py index 5cc6af1b..480ed82d 100644 --- a/tests/ops/experimental/test_decisions.py +++ b/tests/ops/experimental/test_decisions.py @@ -2,6 +2,7 @@ from __future__ import annotations +import types from collections.abc import Mapping from typing import Any, Literal, assert_type, cast @@ -529,11 +530,45 @@ async def test_decide_keeps_file_shaped_dicts_as_json() -> None: assert not isinstance(sent, ai.types.messages.FilePart) +async def test_decide_accepts_typed_and_read_only_containers() -> None: + photo = ai.file_part(b"\x89PNG\r\n\x1a\n", media_type="image/png") + path: list[str | int] = ["body", 0] + counts: dict[str, int] = {"items": 2} + provider = RecordingDecisionProvider() + model = models.Model(id="mock-decision-model", provider=provider) + + await ops.experimental.decide( + model, + { + "path": path, + "counts": counts, + "photos": (photo,), + "meta": types.MappingProxyType({"note": "Delivered."}), + }, + { + "damaged": ops.experimental.PredicateQuestion( + instructions=("Damaged?", photo), + ) + }, + ) + + [(state, sent)] = provider.requests + assert state == { + "path": ["body", 0], + "counts": {"items": 2}, + "photos": [photo], + "meta": {"note": "Delivered."}, + } + assert type(state["meta"]) is dict + assert state["photos"][0] is photo + assert sent["damaged"].instructions == ["Damaged?", photo] + + @pytest.mark.parametrize( ("state", "loc", "error_type"), [ (1, (), "invalid-decision-input"), - ((1,), (), "invalid-decision-input"), + (b"\x89PNG", (), "invalid-decision-input"), ( {"a": [1, {"b": object()}]}, ("dict", "a", "list", 1, "dict", "b"),