diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1808541c..625c6a41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,10 +42,10 @@ jobs: matrix: include: - sdk-versions: minimum - openai: openai==2.34.0 + openai: openai==3.42.0 anthropic: anthropic==0.83.0 - sdk-versions: latest - openai: openai>=2.34.0,<4.0.0 + openai: openai>=3.42.0,<4.0.0 anthropic: anthropic>=0.83.0,<2.0.0 name: Provider SDKs (${{ matrix.sdk-versions }}) steps: diff --git a/examples/models/openai/decisions_images.py b/examples/models/openai/decisions_images.py index 62a3f94f..fbf61520 100644 --- a/examples/models/openai/decisions_images.py +++ b/examples/models/openai/decisions_images.py @@ -27,7 +27,11 @@ class DeliveryQuestions(pydantic.BaseModel): class DeliveryAnswers(pydantic.BaseModel): - damaged: ai.ops.experimental.PredicateAnswer + # OpenAI can refuse a question. This field accepts a refusal; a refusal + # for the other fields raises ValueError. + damaged: ( + ai.ops.experimental.PredicateAnswer | ai.ops.experimental.RefusalAnswer + ) condition: ai.ops.experimental.ScoreAnswer matches_reference: ai.ops.experimental.PredicateAnswer @@ -77,7 +81,10 @@ async def main() -> None: ) answers = result.value - print("damaged:", f"{answers.damaged.probability:.0%}") + if isinstance(answers.damaged, ai.ops.experimental.RefusalAnswer): + print("damaged: refused") + else: + print("damaged:", f"{answers.damaged.probability:.0%}") print("condition:", answers.condition.score) print("matches reference:", f"{answers.matches_reference.probability:.0%}") print("usage:", result.usage) diff --git a/pyproject.toml b/pyproject.toml index d2f0e042..99d9ad1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ [project.optional-dependencies] anthropic = ["anthropic>=0.83.0,<2.0.0"] mcp = ["mcp>=2.0.0,<3.0.0"] -openai = ["openai>=2.34.0,<4.0.0"] +openai = ["openai>=3.42.0,<4.0.0"] otel = [ "opentelemetry-exporter-otlp-proto-http>=1.30", "opentelemetry-sdk>=1.30", @@ -75,7 +75,7 @@ dev = [ "pytest-asyncio>=0.24", "rich>=14.2.0", "mypy~=2.1.0", - "openai>=2.14.0", + "openai>=3.42.0", "ruff~=0.8.0", "async-solipsism>=0.9", "ty~=0.0.37", diff --git a/src/ai/ops/experimental/__init__.py b/src/ai/ops/experimental/__init__.py index 36a66697..32ebb340 100644 --- a/src/ai/ops/experimental/__init__.py +++ b/src/ai/ops/experimental/__init__.py @@ -11,6 +11,7 @@ PredicateAnswer, PredicateCriteria, PredicateQuestion, + RefusalAnswer, ScoreAnswer, ScoreQuestion, decide, @@ -56,6 +57,7 @@ def __getattr__(name: str) -> Any: "PredicateAnswer", "PredicateCriteria", "PredicateQuestion", + "RefusalAnswer", "ScoreAnswer", "ScoreQuestion", "decide", diff --git a/src/ai/ops/experimental/decisions.py b/src/ai/ops/experimental/decisions.py index eee8e94d..cacb073c 100644 --- a/src/ai/ops/experimental/decisions.py +++ b/src/ai/ops/experimental/decisions.py @@ -268,6 +268,18 @@ def noul(self) -> float: return self.probability +class RefusalAnswer(pydantic.BaseModel): + """The model declined to answer this question. + + Annotate an output field as ``XAnswer | RefusalAnswer`` to accept a + refusal for that question. + """ + + type: Literal["refusal"] = "refusal" + + model_config = pydantic.ConfigDict(frozen=True) + + @dataclasses.dataclass(frozen=True, kw_only=True) class DecideParams: """Parameters for decisions.""" @@ -279,7 +291,7 @@ class DecideParams: type _Question = ChoiceQuestion | ScoreQuestion | PredicateQuestion -type _Answer = ChoiceAnswer | ScoreAnswer | PredicateAnswer +type _Answer = ChoiceAnswer | ScoreAnswer | PredicateAnswer | RefusalAnswer _QUESTION_TYPES = (ChoiceQuestion, ScoreQuestion, PredicateQuestion) _ANSWER_TYPES: dict[type[_Question], type[pydantic.BaseModel]] = { @@ -334,6 +346,11 @@ async def decide( ``{"photo": ai.file_part(data), "note": "..."}``. Providers reject files they cannot send. + Some providers can refuse a question. A refusal is a + :class:`RefusalAnswer`. In typed mode, annotate the output field as + ``XAnswer | RefusalAnswer`` to accept it; otherwise a refusal raises + ``ValueError``. + Experimental: not part of the stable API, may change or be removed. """ # Validate the shared state before dispatching it to a provider. @@ -372,21 +389,29 @@ async def decide( # Normalize either public input form to the mapping expected by providers. normalized: dict[str, _Question] = {} + # Questions that accept a refusal. In dynamic mode, all of them do. + refusable: set[str] = set() for name, question in values: if not isinstance(name, str): raise TypeError("question IDs must be strings") if not isinstance(question, _QUESTION_TYPES): raise TypeError(f"question field {name!r} must contain a question") - # In typed mode, verify each question has its corresponding answer type. + # In typed mode, verify each question has its corresponding answer + # type, optionally in a union with RefusalAnswer. if answer_fields is not None: expected = _ANSWER_TYPES[type(question)] actual = answer_fields[name].annotation - if actual is not expected: + if actual == expected | RefusalAnswer: + refusable.add(name) + elif actual is not expected: raise TypeError( f"output field {name!r} must be annotated as " - f"{expected.__name__}" + f"{expected.__name__} or " + f"{expected.__name__} | RefusalAnswer" ) + else: + refusable.add(name) normalized[name] = question # Start telemetry after local validation, immediately around provider work. @@ -410,6 +435,20 @@ async def decide( if raw_item.value.keys() != normalized.keys(): raise ValueError("answer fields must match question fields") + # A refusal needs an output field that accepts it. + refused = { + name + for name, raw_answer in raw_item.value.items() + if isinstance(raw_answer, Mapping) + and raw_answer.get("type") == "refusal" + } + unaccepted = sorted(refused - refusable) + if unaccepted: + raise ValueError( + f"the model refused question {unaccepted[0]!r}; annotate its " + "output field with '| RefusalAnswer' to accept refusals" + ) + if output_type is not None: # Typed mode validates the complete response into the user's model. value: pydantic.BaseModel | dict[str, _Answer] = ( @@ -421,7 +460,9 @@ async def decide( answers: dict[str, _Answer] = {} for name, question in normalized.items(): raw_answer = raw_item.value[name] - if isinstance(question, ChoiceQuestion): + if name in refused: + answers[name] = RefusalAnswer.model_validate(raw_answer) + elif isinstance(question, ChoiceQuestion): answers[name] = ChoiceAnswer.model_validate(raw_answer) elif isinstance(question, ScoreQuestion): answers[name] = ScoreAnswer.model_validate(raw_answer) diff --git a/src/ai/providers/openai/_sdk.py b/src/ai/providers/openai/_sdk.py index 92c2ac23..ab95328e 100644 --- a/src/ai/providers/openai/_sdk.py +++ b/src/ai/providers/openai/_sdk.py @@ -10,6 +10,7 @@ from collections.abc import Callable import openai + from openai.types import decision class OpenAISDK(Protocol): @@ -26,6 +27,11 @@ class OpenAIPydantic(Protocol): to_strict_json_schema: Callable[[Any], dict[str, Any]] +class OpenAIDecisionTypes(Protocol): + AnswerAnswerResourceChoice: type[decision.AnswerAnswerResourceChoice] + AnswerAnswerResourceScore: type[decision.AnswerAnswerResourceScore] + + def import_sdk(*, provider: str = "openai") -> OpenAISDK: return cast( "OpenAISDK", @@ -44,3 +50,14 @@ def import_pydantic(*, provider: str = "openai") -> OpenAIPydantic: extra="openai", ), ) + + +def import_decision_types(*, provider: str = "openai") -> OpenAIDecisionTypes: + return cast( + "OpenAIDecisionTypes", + _optional.import_optional_sdk( + "openai.types.decision", + provider=provider, + extra="openai", + ), + ) diff --git a/src/ai/providers/openai/protocol.py b/src/ai/providers/openai/protocol.py index 147d2d14..54fbf1f4 100644 --- a/src/ai/providers/openai/protocol.py +++ b/src/ai/providers/openai/protocol.py @@ -15,6 +15,8 @@ from ... import types from ...models import core from ...models.core import params as params_ +from ...ops import items +from ...ops.experimental import decisions from .. import base, history_utils from . import _sdk, errors from . import params as openai_params @@ -1898,6 +1900,182 @@ async def _stream_responses( ) from exc +# --------------------------------------------------------------------------- +# Decisions — ``/v1/decisions`` +# --------------------------------------------------------------------------- + + +_DECISIONS_PROTECTED_PARAMS = frozenset({"model", "input", "questions"}) + + +async def _decide( + sdk_client: openai.AsyncOpenAI, + model: core.model.Model, + state: decisions.DecisionInput, + questions: Mapping[ + str, + decisions.ChoiceQuestion + | decisions.ScoreQuestion + | decisions.PredicateQuestion, + ], + *, + params: decisions.DecideParams, + provider: str, +) -> items.Item[dict[str, Any]]: + openai_sdk = _sdk.import_sdk(provider=provider) + + # Provider options are extra top-level request body fields, e.g. + # prompt_cache_key or safety_identifier. + options = params.provider_options.get(provider) + if options is not None and not isinstance(options, Mapping): + raise TypeError(f"provider_options[{provider!r}] must be a mapping") + protected = sorted(_DECISIONS_PROTECTED_PARAMS & (options or {}).keys()) + if protected: + raise ValueError( + "openai decisions provider options cannot override " + "protocol-owned fields: " + ", ".join(protected) + ) + + def has_file(value: Any) -> bool: + if isinstance(value, types.messages.FilePart): + return True + if isinstance(value, dict): + return any(has_file(item) for item in value.values()) + if isinstance(value, list): + return any(has_file(item) for item in value) + return False + + # OpenAI takes content parts: text, images, and JSON objects. A list is + # an ordered list of parts; any other value is one part. Images use the + # Responses API image part. OpenAI does not take files inside JSON. + async def to_parts(value: Any) -> list[dict[str, Any]]: + parts: list[dict[str, Any]] = [] + for item in value if isinstance(value, list) else [value]: + match item: + case str(): + parts.append({"type": "input_text", "text": item}) + case types.messages.FilePart(media_type=media_type): + if not media_type.startswith("image/"): + raise ValueError( + "Unsupported media type for OpenAI decisions: " + f"{media_type}" + ) + parts.append(await _file_part_to_responses(item)) + case dict() if has_file(item): + raise ValueError( + "OpenAI decisions do not support files inside JSON" + ) + case dict(): + parts.append({"type": "input_json", "data": item}) + case _: + raise ValueError( + "OpenAI decisions accept only text, files, and JSON " + f"objects in a list, got {type(item).__name__}" + ) + return parts + + # OpenAI choice and level descriptions are text only. + def description(name: str, criterion: Any) -> dict[str, str]: + if criterion is None: + return {} + if not isinstance(criterion, str): + raise ValueError( + "OpenAI decisions accept only text criteria, " + f"see question {name!r}" + ) + return {"description": criterion} + + # Questions go as a list; the question ID is the name. OpenAI answers + # by name, so each answer goes back to its question. + wire_questions: list[dict[str, Any]] = [] + for name, question in questions.items(): + wire_question: dict[str, Any] = { + "type": question.type, + "name": name, + "instructions": question.instructions + if isinstance(question.instructions, str) + else await to_parts(question.instructions), + } + if isinstance(question, decisions.ChoiceQuestion): + wire_question["choices"] = [ + {"value": value, **description(name, criterion)} + for value, criterion in question.criteria.items() + ] + elif isinstance(question, decisions.ScoreQuestion): + # OpenAI levels need a label. The label is the level index, + # which is also the key of the answer's probabilities. + wire_question["levels"] = [ + {"label": str(index), **description(name, criterion)} + for index, criterion in enumerate(question.criteria) + ] + elif question.criteria and any( + criterion is not None for criterion in question.criteria.values() + ): + raise ValueError( + "OpenAI decisions do not support predicate criteria, " + f"see question {name!r}" + ) + wire_questions.append(wire_question) + + # Shared state is a text string or one user message. + api_kwargs: dict[str, Any] = { + "model": model.id, + "input": state + if isinstance(state, str) + else [{"role": "user", "content": await to_parts(state)}], + "questions": wire_questions, + } + # The SDK takes only known fields as arguments, so provider options go + # in the body. + if options: + api_kwargs["extra_body"] = dict(options) + + try: + response = await sdk_client.decisions.create(**api_kwargs) + except openai_sdk.OpenAIError as exc: + raise errors.map_error( + exc, provider=provider, model_id=model.id + ) from exc + + # Answers come as a list with names. Probability lists become maps + # keyed by choice value or level index, like the shared answer models. + decision_types = _sdk.import_decision_types(provider=provider) + answers: dict[str, Any] = {} + for answer in response.answers: + if ( + answer.name is None + or answer.name not in questions + or answer.name in answers + ): + raise ValueError( + "decision response has an unexpected answer name: " + f"{answer.name!r}" + ) + data = answer.model_dump(mode="json", exclude={"name"}) + if isinstance( + answer, decision_types.AnswerAnswerResourceChoice + ) or isinstance(answer, decision_types.AnswerAnswerResourceScore): + data["probabilities"] = { + str(item.value): item.probability + for item in answer.probabilities + } + answers[answer.name] = data + + usage = response.usage + return items.Item( + value=answers, + usage=types.usage.Usage( + input_tokens=usage.input_tokens, + output_tokens=usage.output_tokens, + reasoning_tokens=usage.output_tokens_details.reasoning_tokens, + cache_read_tokens=usage.input_tokens_details.cached_tokens, + cache_write_tokens=usage.input_tokens_details.cache_write_tokens, + raw=usage.model_dump(mode="json"), + ), + provider_metadata={provider: {"model": response.model}}, + ) + + class OpenAIResponsesProtocol(base.ProviderProtocol[Any]): """OpenAI Responses API protocol.""" @@ -1924,6 +2102,31 @@ def stream( provider=provider, ) + async def decide( + self, + client: openai.AsyncOpenAI, + model: core.model.Model, + state: decisions.DecisionInput, + questions: Mapping[ + str, + decisions.ChoiceQuestion + | decisions.ScoreQuestion + | decisions.PredicateQuestion, + ], + *, + params: decisions.DecideParams, + provider: str, + ) -> items.Item[dict[str, Any]]: + """Hit ``/v1/decisions`` and return raw decision answers.""" + return await _decide( + client, + model, + state, + questions, + params=params, + provider=provider, + ) + def default_protocol(provider: str) -> base.ProviderProtocol[Any]: """Return the best default OpenAI-compatible protocol for *provider*.""" diff --git a/tests/ops/experimental/test_decisions.py b/tests/ops/experimental/test_decisions.py index 480ed82d..b56ad2b8 100644 --- a/tests/ops/experimental/test_decisions.py +++ b/tests/ops/experimental/test_decisions.py @@ -200,7 +200,8 @@ async def test_decide_dynamic_questions() -> None: str, ops.experimental.ChoiceAnswer | ops.experimental.ScoreAnswer - | ops.experimental.PredicateAnswer, + | ops.experimental.PredicateAnswer + | ops.experimental.RefusalAnswer, ] ], ) @@ -215,7 +216,11 @@ async def test_decide_dynamic_questions() -> None: async def test_decide_raises_not_implemented() -> None: provider = ai.get_provider("openai", api_key="[redacted]") - model = ai.Model(id="decision-test", provider=provider) + model = ai.Model( + id="decision-test", + provider=provider, + protocol=ai.providers.openai.OpenAIChatCompletionsProtocol(), + ) class PredicateQuestions(pydantic.BaseModel): answer: ops.experimental.PredicateQuestion @@ -458,6 +463,81 @@ async def test_decide_requires_matching_answer_fields() -> None: ) +def refusal_model() -> models.Model: + return models.Model( + id="mock-decision-model", + provider=StaticDecisionProvider( + answers={ + "department": {"type": "refusal"}, + "severity": {"type": "score", "score": 1.5}, + "refund": {"type": "predicate", "probability": 0.98}, + } + ), + ) + + +async def test_decide_accepts_refusals_in_opted_in_fields() -> None: + class RefusableAnswers(pydantic.BaseModel): + department: ( + ops.experimental.ChoiceAnswer | ops.experimental.RefusalAnswer + ) + severity: ops.experimental.ScoreAnswer + refund: ops.experimental.PredicateAnswer + + result = await ops.experimental.decide( + refusal_model(), + "state", + questions(), + output_type=RefusableAnswers, + ) + + assert result.value.department == ops.experimental.RefusalAnswer() + assert result.value.severity.score == 1.5 + assert result.value.refund.probability == 0.98 + + +async def test_decide_rejects_refusals_in_other_fields() -> None: + with pytest.raises(ValueError, match="refused question 'department'"): + await ops.experimental.decide( + refusal_model(), + "state", + questions(), + output_type=Answers, + ) + + +async def test_decide_returns_refusals_in_dynamic_mode() -> None: + dynamic_questions: dict[str, DecisionQuestion] = { + "department": questions().department, + "severity": questions().severity, + "refund": questions().refund, + } + + result = await ops.experimental.decide( + refusal_model(), "state", dynamic_questions + ) + + assert result.value["department"] == ops.experimental.RefusalAnswer() + assert isinstance(result.value["severity"], ops.experimental.ScoreAnswer) + + +async def test_decide_requires_refusal_union_with_matching_answer() -> None: + class WrongAnswers(pydantic.BaseModel): + department: ( + ops.experimental.PredicateAnswer | ops.experimental.RefusalAnswer + ) + severity: ops.experimental.ScoreAnswer + refund: ops.experimental.PredicateAnswer + + with pytest.raises(TypeError, match=r"ChoiceAnswer \| RefusalAnswer"): + await ops.experimental.decide( + refusal_model(), + "state", + questions(), + output_type=WrongAnswers, + ) + + async def test_decide_accepts_files_in_state_and_questions() -> None: photo = ai.file_part(b"\x89PNG\r\n\x1a\n", media_type="image/png") reference = ai.file_part("https://example.com/reference.jpg") diff --git a/tests/providers/openai/test_adapter.py b/tests/providers/openai/test_adapter.py index f62b61d2..f8b7b76f 100644 --- a/tests/providers/openai/test_adapter.py +++ b/tests/providers/openai/test_adapter.py @@ -6,10 +6,10 @@ from __future__ import annotations -import importlib from types import SimpleNamespace from typing import Any, cast +import httpx2 as httpx import openai import pydantic import pytest @@ -20,10 +20,6 @@ from ai.providers.openai import tools as openai_tools from ai.types import events, messages, tools -httpx = importlib.import_module( - "httpx2" if int(openai.__version__.partition(".")[0]) >= 3 else "httpx" -) - class _Answer(pydantic.BaseModel): answer: str diff --git a/tests/providers/openai/test_probe.py b/tests/providers/openai/test_probe.py index ee2c66e5..e8064d08 100644 --- a/tests/providers/openai/test_probe.py +++ b/tests/providers/openai/test_probe.py @@ -1,18 +1,13 @@ from __future__ import annotations -import importlib import json from typing import Any -import openai +import httpx2 as httpx import pytest import ai -httpx = importlib.import_module( - "httpx2" if int(openai.__version__.partition(".")[0]) >= 3 else "httpx" -) - def _client_with_mock( status_code: int = 200, diff --git a/tests/providers/openai/test_protocol.py b/tests/providers/openai/test_protocol.py new file mode 100644 index 00000000..d9ecf03a --- /dev/null +++ b/tests/providers/openai/test_protocol.py @@ -0,0 +1,414 @@ +"""Tests for the OpenAI ``/v1/decisions`` protocol.""" + +from __future__ import annotations + +import base64 +import json +from typing import Any + +import httpx2 as httpx +import pydantic +import pytest + +import ai +from ai import ops + +_PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16 +_PNG_URL = "data:image/png;base64," + base64.b64encode(_PNG).decode() + + +def _model(handler: Any) -> ai.Model: + provider = ai.get_provider( + "openai", + base_url="https://openai.test/v1", + api_key="sk-test", + client=httpx.AsyncClient(transport=httpx.MockTransport(handler)), + ) + return ai.Model(id="gpt-6-luna", provider=provider) + + +def _response(answers: list[dict[str, Any]]) -> dict[str, Any]: + return { + "model": "gpt-6-luna", + "answers": answers, + "usage": { + "input_tokens": 42, + "input_tokens_details": { + "cached_tokens": 10, + "cache_write_tokens": 2, + }, + "output_tokens": 0, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 42, + }, + } + + +class Questions(pydantic.BaseModel): + department: ops.experimental.ChoiceQuestion + severity: ops.experimental.ScoreQuestion + refund: ops.experimental.PredicateQuestion + + +class Answers(pydantic.BaseModel): + department: ops.experimental.ChoiceAnswer + severity: ops.experimental.ScoreAnswer + refund: ops.experimental.PredicateAnswer + + +async def test_decide_sends_native_request_and_parses_answers() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + json=_response( + [ + { + "type": "choice", + "name": "department", + "choice": "billing", + "confidence": 0.8, + "probabilities": [ + {"value": "billing", "probability": 0.9}, + {"value": "support", "probability": 0.1}, + ], + }, + { + "type": "score", + "name": "severity", + "score": 1.1, + "confidence": 0.55, + "probabilities": [ + {"value": 0, "label": "0", "probability": 0.1}, + {"value": 1, "label": "1", "probability": 0.7}, + {"value": 2, "label": "2", "probability": 0.2}, + ], + }, + { + "type": "predicate", + "name": "refund", + "probability": 0.95, + }, + ] + ), + ) + + result = await ops.experimental.decide( + _model(handler), + "I was charged twice.", + Questions( + department=ops.experimental.ChoiceQuestion( + instructions="Which team should handle this?", + criteria={"billing": "Payments and refunds.", "support": None}, + ), + severity=ops.experimental.ScoreQuestion( + instructions="How severe is this?", + criteria=["Cosmetic", "Workaround", None], + ), + refund=ops.experimental.PredicateQuestion( + instructions="Does the customer want a refund?", + ), + ), + output_type=Answers, + ) + + assert len(requests) == 1 + assert str(requests[0].url) == "https://openai.test/v1/decisions" + assert json.loads(requests[0].content) == { + "model": "gpt-6-luna", + "input": "I was charged twice.", + "questions": [ + { + "type": "choice", + "name": "department", + "instructions": "Which team should handle this?", + "choices": [ + { + "value": "billing", + "description": "Payments and refunds.", + }, + {"value": "support"}, + ], + }, + { + "type": "score", + "name": "severity", + "instructions": "How severe is this?", + "levels": [ + {"label": "0", "description": "Cosmetic"}, + {"label": "1", "description": "Workaround"}, + {"label": "2"}, + ], + }, + { + "type": "predicate", + "name": "refund", + "instructions": "Does the customer want a refund?", + }, + ], + } + assert result.value == Answers( + department=ops.experimental.ChoiceAnswer( + choice="billing", + confidence=0.8, + probabilities={"billing": 0.9, "support": 0.1}, + ), + severity=ops.experimental.ScoreAnswer( + score=1.1, + confidence=0.55, + probabilities={"0": 0.1, "1": 0.7, "2": 0.2}, + ), + refund=ops.experimental.PredicateAnswer(probability=0.95), + ) + assert result.usage == ai.types.usage.Usage( + input_tokens=42, + output_tokens=0, + reasoning_tokens=0, + cache_read_tokens=10, + cache_write_tokens=2, + raw=_response([])["usage"], + ) + assert result.provider_metadata == {"openai": {"model": "gpt-6-luna"}} + + +async def test_decide_sends_files_and_json_as_parts() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + json=_response( + [{"type": "predicate", "name": "match", "probability": 0.99}] + ), + ) + + await ops.experimental.decide( + _model(handler), + [ + "This is photo A.", + ai.file_part(_PNG, media_type="image/png"), + {"order": {"charges": [49, 49]}}, + ], + { + "match": ops.experimental.PredicateQuestion( + instructions=[ + "Is photo B a copy of photo A?", + ai.file_part( + "https://example.com/b.jpg", media_type="image/jpeg" + ), + ], + ), + }, + ) + + body = json.loads(requests[0].content) + assert body["input"] == [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "This is photo A."}, + {"type": "input_image", "image_url": _PNG_URL}, + { + "type": "input_json", + "data": {"order": {"charges": [49, 49]}}, + }, + ], + } + ] + assert body["questions"][0]["instructions"] == [ + {"type": "input_text", "text": "Is photo B a copy of photo A?"}, + {"type": "input_image", "image_url": "https://example.com/b.jpg"}, + ] + + +@pytest.mark.parametrize( + ("state", "content"), + [ + ({"a": 1}, [{"type": "input_json", "data": {"a": 1}}]), + ( + ai.file_part(_PNG, media_type="image/png"), + [{"type": "input_image", "image_url": _PNG_URL}], + ), + ], +) +async def test_decide_sends_single_values_as_one_part( + state: Any, content: list[dict[str, Any]] +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + json=_response( + [{"type": "predicate", "name": "ok", "probability": 0.5}] + ), + ) + + await ops.experimental.decide( + _model(handler), + state, + {"ok": ops.experimental.PredicateQuestion(instructions="OK?")}, + ) + + assert json.loads(requests[0].content)["input"] == [ + {"role": "user", "content": content} + ] + + +async def test_decide_returns_refusals() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json=_response( + [ + {"type": "refusal", "name": "unsafe"}, + {"type": "predicate", "name": "safe", "probability": 0.1}, + ] + ), + ) + + result = await ops.experimental.decide( + _model(handler), + "state", + { + "unsafe": ops.experimental.PredicateQuestion(instructions="A?"), + "safe": ops.experimental.PredicateQuestion(instructions="B?"), + }, + ) + + assert result.value == { + "unsafe": ops.experimental.RefusalAnswer(), + "safe": ops.experimental.PredicateAnswer(probability=0.1), + } + + +async def test_decide_sends_provider_options() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response( + 200, + json=_response( + [{"type": "predicate", "name": "ok", "probability": 0.5}] + ), + ) + + await ops.experimental.decide( + _model(handler), + "state", + {"ok": ops.experimental.PredicateQuestion(instructions="OK?")}, + params=ops.experimental.DecideParams( + provider_options={ + "openai": {"prompt_cache_key": "tickets"}, + "gateway": {"ignored": True}, + } + ), + ) + + body = json.loads(requests[0].content) + assert body["prompt_cache_key"] == "tickets" + assert "ignored" not in body + + +@pytest.mark.parametrize( + ("state", "question", "match"), + [ + ( + "state", + ops.experimental.PredicateQuestion( + instructions="OK?", criteria={"true": "Yes."} + ), + "predicate criteria", + ), + ( + "state", + ops.experimental.ChoiceQuestion( + instructions="Which?", criteria={"a": {"rule": 1}, "b": None} + ), + "text criteria", + ), + ( + ai.file_part(b"%PDF-1.4", media_type="application/pdf"), + ops.experimental.PredicateQuestion(instructions="OK?"), + "application/pdf", + ), + ( + {"photo": ai.file_part(_PNG, media_type="image/png")}, + ops.experimental.PredicateQuestion(instructions="OK?"), + "files inside JSON", + ), + ( + ["text", 1], + ops.experimental.PredicateQuestion(instructions="OK?"), + "got int", + ), + ], +) +async def test_decide_rejects_unsupported_input( + state: Any, question: Any, match: str +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + raise AssertionError("request must not be sent") + + with pytest.raises(ValueError, match=match): + await ops.experimental.decide(_model(handler), state, {"ok": question}) + + +@pytest.mark.parametrize( + ("options", "error", "match"), + [ + ("bad", TypeError, "must be a mapping"), + ({"model": "other"}, ValueError, "protocol-owned fields: model"), + ], +) +async def test_decide_rejects_invalid_provider_options( + options: Any, error: type[Exception], match: str +) -> None: + def handler(request: httpx.Request) -> httpx.Response: + raise AssertionError("request must not be sent") + + with pytest.raises(error, match=match): + await ops.experimental.decide( + _model(handler), + "state", + {"ok": ops.experimental.PredicateQuestion(instructions="OK?")}, + params=ops.experimental.DecideParams( + provider_options={"openai": options} + ), + ) + + +async def test_decide_maps_status_errors() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 429, json={"error": {"message": "slow down", "type": "rate"}} + ) + + with pytest.raises(ai.errors.ProviderRateLimitError): + await ops.experimental.decide( + _model(handler), + "state", + {"ok": ops.experimental.PredicateQuestion(instructions="OK?")}, + ) + + +async def test_decide_rejects_unexpected_answer_names() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json=_response( + [{"type": "predicate", "name": "other", "probability": 0.5}] + ), + ) + + with pytest.raises(ValueError, match="unexpected answer name: 'other'"): + await ops.experimental.decide( + _model(handler), + "state", + {"ok": ops.experimental.PredicateQuestion(instructions="OK?")}, + ) diff --git a/tests/providers/openai/test_provider.py b/tests/providers/openai/test_provider.py index 002b868f..e19bd805 100644 --- a/tests/providers/openai/test_provider.py +++ b/tests/providers/openai/test_provider.py @@ -3,6 +3,7 @@ import importlib from typing import Any +import httpx2 as httpx import openai import pytest @@ -12,10 +13,6 @@ OpenAIResponsesProtocol, ) -httpx = importlib.import_module( - "httpx2" if int(openai.__version__.partition(".")[0]) >= 3 else "httpx" -) - async def test_list_models_gets_models_with_auth_header_and_sorts_ids() -> None: captured_urls: list[str] = [] diff --git a/uv.lock b/uv.lock index 6166b164..6f5415f9 100644 --- a/uv.lock +++ b/uv.lock @@ -67,7 +67,7 @@ requires-dist = [ { name = "httpx2", specifier = ">=2.12.0" }, { name = "mcp", marker = "extra == 'mcp'", specifier = ">=2.0.0,<3.0.0" }, { name = "modelsdotdev", specifier = "==0.*" }, - { name = "openai", marker = "extra == 'openai'", specifier = ">=2.34.0,<4.0.0" }, + { name = "openai", marker = "extra == 'openai'", specifier = ">=3.42.0,<4.0.0" }, { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'", specifier = ">=1.30" }, { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.30" }, { name = "pydantic", specifier = ">=2.13" }, @@ -83,7 +83,7 @@ dev = [ { name = "async-solipsism", specifier = ">=0.9" }, { name = "mcp", specifier = ">=2.0.0,<3.0.0" }, { name = "mypy", specifier = "~=2.1.0" }, - { name = "openai", specifier = ">=2.14.0" }, + { name = "openai", specifier = ">=3.42.0" }, { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.43.0" }, { name = "opentelemetry-sdk", specifier = ">=1.43.0" }, { name = "pytest", specifier = ">=8.0" }, @@ -803,7 +803,7 @@ wheels = [ [[package]] name = "openai" -version = "3.6.0" +version = "3.42.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -813,9 +813,9 @@ dependencies = [ { name = "sniffio" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ac/76/913b755a1a6b54e2d9140eb8d488aa0d47c7359b1d7eac5e864cb7913bbf/openai-3.6.0.tar.gz", hash = "sha256:18fe3f6e96390ef41ee27b152fc9effefca321c33673bd9b956a572493d3ab9b", size = 1455376, upload-time = "2026-08-28T22:29:18.268Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/ef/c17c9ca921c1991b0a7185e8282da1996d09292b0caaf2e7cffd7e051eb6/openai-3.42.0.tar.gz", hash = "sha256:deffe27f9e20e4f1d9d471978d12f23aa119382494c0a9860795e66943d0fe42", size = 2353482, upload-time = "2026-10-31T00:27:26.502Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/94/805b87ecc951c49ec8f247f5e8eb324ab064bd2ad73b6a0e704dd49aa073/openai-3.6.0-py3-none-any.whl", hash = "sha256:508e2158bf971687f953b62e44b02f207792c815aac306816386d7ba34d37f5f", size = 1699841, upload-time = "2026-08-28T22:29:16.436Z" }, + { url = "https://files.pythonhosted.org/packages/c5/79/904afac12186d9ab9e3f731eb798e36dc0f5b991a557c4304b0a43e393c1/openai-3.42.0-py3-none-any.whl", hash = "sha256:efb9d089ecf66c668289f647717389df98b40b954cd86d4fe54eddce39aabb3f", size = 2397860, upload-time = "2026-10-31T00:27:24.493Z" }, ] [[package]]