From dfe7bf05287dd27a4916b6d1e23d1cbebce38540 Mon Sep 17 00:00:00 2001 From: Andrey Buzin Date: Mon, 2 Nov 2026 11:06:12 -0800 Subject: [PATCH] Rename evaluate to decide and Noul to predicate Align decision primitives with the OpenAI Decisions API: rename `ai.ops.experimental.evaluate` to `decide`, and `NoulQuestion`, `NoulAnswer`, and `NoulCriteria` to `Predicate*`, with `type="predicate"` and a `.probability` answer value. `EvaluationInput`, `EvaluationParams`, and `EvaluateSpanData` become `DecisionInput`, `DecideParams`, and `DecideSpanData`, and provider/protocol `evaluate()` becomes `decide()`. The TypeSafe and AI Gateway adapters translate their `noul` and `boolean` wire formats. The old names, `type="noul"`, and `PredicateAnswer.noul` still work as aliases that emit a `DeprecationWarning`. Update docs, examples, and tests. --- .../content/docs/basics/model-operations.mdx | 20 +- .../ai.providers/ai-gateway/index.mdx | 4 +- .../docs/reference/ai.providers/index.mdx | 2 +- docs/ai-python/content/docs/reference/ops.mdx | 32 ++- .../content/docs/reference/telemetry.mdx | 4 +- .../gateway/{evaluation.py => decisions.py} | 32 +-- examples/models/gateway/jev_python_ast.py | 2 +- skills/ai/SKILL.md | 4 +- src/ai/experimental_telemetry/__init__.py | 23 +- src/ai/experimental_telemetry/otel.py | 10 +- src/ai/experimental_telemetry/span.py | 8 +- .../experimental_telemetry/utils/console.py | 4 +- src/ai/ops/experimental/__init__.py | 61 ++++- .../{evaluation.py => decisions.py} | 132 ++++++--- src/ai/providers/ai_gateway/protocol/v4.py | 30 +-- src/ai/providers/base.py | 34 +-- src/ai/providers/typesafe/__init__.py | 4 +- src/ai/providers/typesafe/protocol.py | 23 +- tests/experimental_telemetry/test_otel.py | 8 +- .../{test_evaluation.py => test_decisions.py} | 254 ++++++++++-------- ..._v4_evaluation.py => test_v4_decisions.py} | 50 ++-- tests/providers/typesafe/test_protocol.py | 65 ++--- 22 files changed, 466 insertions(+), 340 deletions(-) rename examples/models/gateway/{evaluation.py => decisions.py} (81%) rename src/ai/ops/experimental/{evaluation.py => decisions.py} (70%) rename tests/ops/experimental/{test_evaluation.py => test_decisions.py} (54%) rename tests/providers/ai_gateway/protocol/{test_v4_evaluation.py => test_v4_decisions.py} (88%) diff --git a/docs/ai-python/content/docs/basics/model-operations.mdx b/docs/ai-python/content/docs/basics/model-operations.mdx index 77f63759..06abd478 100644 --- a/docs/ai-python/content/docs/basics/model-operations.mdx +++ b/docs/ai-python/content/docs/basics/model-operations.mdx @@ -1,8 +1,8 @@ --- title: Non-LLM models -description: Generate media and embeddings, transcribe audio, rerank documents, and evaluate state. +description: Generate media and embeddings, transcribe audio, rerank documents, and make decisions. type: guide -summary: Use dedicated models for images, video, speech, embeddings, transcription, reranking, and evaluation. +summary: Use dedicated models for images, video, speech, embeddings, transcription, reranking, and decisions. --- Use `ai.ops` for model calls that do not produce a language-model message. @@ -253,9 +253,9 @@ for ranked in result.value: Each result contains the document's original index and relevance score. Results are ordered from the highest score to the lowest. -## Evaluate state +## Make decisions -Use `ai.ops.experimental.evaluate` to ask choice, score, and Noul questions about one shared state. Define matching Pydantic models for the questions and answers: +Use `ai.ops.experimental.decide` to ask choice, score, and predicate questions about one shared state. Define matching Pydantic models for the questions and answers: ```python import pydantic @@ -264,16 +264,16 @@ import pydantic class Questions(pydantic.BaseModel): department: ai.ops.experimental.ChoiceQuestion severity: ai.ops.experimental.ScoreQuestion - requests_refund: ai.ops.experimental.NoulQuestion + requests_refund: ai.ops.experimental.PredicateQuestion class Answers(pydantic.BaseModel): department: ai.ops.experimental.ChoiceAnswer severity: ai.ops.experimental.ScoreAnswer - requests_refund: ai.ops.experimental.NoulAnswer + requests_refund: ai.ops.experimental.PredicateAnswer -result = await ai.ops.experimental.evaluate( +result = await ai.ops.experimental.decide( ai.get_model("typesafe-ai/jev"), {"message": "Please refund the duplicate charge."}, Questions( @@ -285,7 +285,7 @@ result = await ai.ops.experimental.evaluate( instructions="How severe is this?", criteria=["Cosmetic", "Has workaround", "Blocking"], ), - requests_refund=ai.ops.experimental.NoulQuestion( + requests_refund=ai.ops.experimental.PredicateQuestion( instructions="Is the customer requesting a refund?", ), ), @@ -299,11 +299,11 @@ When question IDs are dynamic, pass a mapping and omit `output_type`. The item value is then a validated answer mapping: ```python -result = await ai.ops.experimental.evaluate( +result = await ai.ops.experimental.decide( ai.get_model("typesafe-ai/jev"), state, { - question_id: ai.ops.experimental.NoulQuestion(instructions=instructions) + question_id: ai.ops.experimental.PredicateQuestion(instructions=instructions) for question_id, instructions in dynamic_questions.items() }, ) diff --git a/docs/ai-python/content/docs/reference/ai.providers/ai-gateway/index.mdx b/docs/ai-python/content/docs/reference/ai.providers/ai-gateway/index.mdx index 0963c99f..fbfa1b4b 100644 --- a/docs/ai-python/content/docs/reference/ai.providers/ai-gateway/index.mdx +++ b/docs/ai-python/content/docs/reference/ai.providers/ai-gateway/index.mdx @@ -549,7 +549,7 @@ The Python provider currently supports: - Structured output. - Function and provider-executed tools. - Model listing and connection checks. -- Image, video, and speech generation, embeddings, transcription, reranking, and evaluation through `ai.ops`. +- Image, video, and speech generation, embeddings, transcription, reranking, and decisions through `ai.ops`. It does not currently expose realtime sessions, credit queries, spend reports, or generation lookup helpers. @@ -586,7 +586,7 @@ Methods and properties: - `embed(...)`: Embeds text values. - `transcribe(...)`: Transcribes audio. - `rerank(...)`: Reranks documents against a query. -- `evaluate(...)`: Evaluates typed questions against shared state. +- `decide(...)`: Decides typed questions about shared state. - `aclose()`: Closes provider-owned resources. ### GatewayParams diff --git a/docs/ai-python/content/docs/reference/ai.providers/index.mdx b/docs/ai-python/content/docs/reference/ai.providers/index.mdx index 165cb2c8..9ecd7e04 100644 --- a/docs/ai-python/content/docs/reference/ai.providers/index.mdx +++ b/docs/ai-python/content/docs/reference/ai.providers/index.mdx @@ -31,7 +31,7 @@ provider-specific namespaces. - `embed`: Embed text values. - `transcribe`: Transcribe audio. - `rerank`: Rerank documents against a query. -- `evaluate`: Evaluate typed questions against shared state. +- `decide`: Decide typed questions about shared state. Application code normally uses `ai.stream`, `ai.experimental_generate`, or [`ai.ops`](/docs/reference/ops) instead of calling provider methods directly. diff --git a/docs/ai-python/content/docs/reference/ops.mdx b/docs/ai-python/content/docs/reference/ops.mdx index d4df3a3d..04ba59d3 100644 --- a/docs/ai-python/content/docs/reference/ops.mdx +++ b/docs/ai-python/content/docs/reference/ops.mdx @@ -2,7 +2,7 @@ title: "ai.ops" description: Reference for dedicated model operations. type: reference -summary: Reference for image, video, speech, embedding, transcription, reranking, and evaluation APIs. +summary: Reference for image, video, speech, embedding, transcription, reranking, and decision APIs. --- `ai.ops` contains model operations that return standalone values instead of @@ -239,13 +239,13 @@ provider. - `index`: Position of the document in the original input list. - `score`: Relevance score for the query. -## experimental.evaluate +## experimental.decide -Evaluation questions, answers, `EvaluationInput`, and `EvaluationParams` live +Decision questions, answers, `DecisionInput`, and `DecideParams` live under `ai.ops.experimental`. ```python -await ai.ops.experimental.evaluate( +await ai.ops.experimental.decide( model, state, questions, @@ -257,25 +257,25 @@ await ai.ops.experimental.evaluate( Arguments: -- `model`: Evaluation model. +- `model`: Decision model. - `state`: Shared JSON-compatible string, object, or array. - `questions`: A nonempty Pydantic model or mapping whose fields contain - `ChoiceQuestion`, `ScoreQuestion`, or `NoulQuestion` values. + `ChoiceQuestion`, `ScoreQuestion`, or `PredicateQuestion` values. - `output_type`: Required with a Pydantic question model. It must be a Pydantic model class with the same fields and corresponding answer types. Omit it with mapped questions. -- `params`: Optional `EvaluationParams`. +- `params`: Optional `DecideParams`. Pydantic questions return `Item[Answers]`, whose value is an instance of `output_type`. Mapped questions return -`Item[dict[str, ChoiceAnswer | ScoreAnswer | NoulAnswer]]`: +`Item[dict[str, ChoiceAnswer | ScoreAnswer | PredicateAnswer]]`: ```python -result = await ai.ops.experimental.evaluate( +result = await ai.ops.experimental.decide( model, state, { - question_id: ai.ops.experimental.NoulQuestion(instructions=instructions) + question_id: ai.ops.experimental.PredicateQuestion(instructions=instructions) for question_id, instructions in dynamic_questions.items() }, ) @@ -295,17 +295,21 @@ Question and answer types: index `0`. `ScoreAnswer` contains a fractional `score` and an optional complete `probabilities` distribution keyed by string indices. With a distribution, `score` is its probability-weighted mean. -- `NoulQuestion`: Estimates whether its instructions are true. Its optional - `criteria` mapping can describe `"true"` and `"false"`. `NoulAnswer.noul` +- `PredicateQuestion`: Estimates whether its instructions are true. Its optional + `criteria` mapping can describe `"true"` and `"false"`. `PredicateAnswer.probability` is the estimated probability of true, not confidence in either outcome. `ChoiceAnswer.confidence` and `ScoreAnswer.confidence` expose a value from 0 to 1 when the provider supplies it, or `None` otherwise. For TypeSafe through AI Gateway, these values come from `provider_metadata["typesafe"]["confidence"]`, which is also -preserved on the result. Noul answers have no separate confidence field. +preserved on the result. Predicate answers have no separate confidence field. Instructions accept strings, JSON objects, or JSON arrays. Criteria descriptions accept the same values or `None`. Nested keys are model input and have no SDK-defined meaning. -`EvaluationParams` contains `provider_options`, keyed by provider name. +`DecideParams` contains `provider_options`, keyed by provider name. + +The old `evaluate`, `EvaluationInput`, `EvaluationParams`, and `Noul*` names, +`type="noul"`, and `PredicateAnswer.noul` still work but emit a +`DeprecationWarning`. diff --git a/docs/ai-python/content/docs/reference/telemetry.mdx b/docs/ai-python/content/docs/reference/telemetry.mdx index a7311c37..7889055d 100644 --- a/docs/ai-python/content/docs/reference/telemetry.mdx +++ b/docs/ai-python/content/docs/reference/telemetry.mdx @@ -95,7 +95,7 @@ Built-in data types: | `GenerateVideoSpanData` | `generate_video` | One video generation operation. | | `RerankSpanData` | `rerank` | One reranking operation. | | `TranscribeSpanData` | `transcribe` | One transcription operation. | -| `EvaluateSpanData` | `evaluate` | One typed evaluation operation. | +| `DecideSpanData` | `decide` | One typed decision operation. | | `ToolExecutionSpanData` | `tool_execution` | One tool execution, from dispatch to result. | | `HookSpanData` | `hook` | One hook suspension, from deferred until resolved or cancelled. | | `CustomSpanData` | `custom` | A user span made with `span("name")`; its attributes live in `attrs`. | @@ -398,7 +398,7 @@ The adapter names spans and sets attributes per the `gen_ai` conventions: | `AiGenerateSpanData` | `chat {model}` | `chat` | | `EmbedSpanData` | `embeddings {model}` | `embeddings` | | `GenerateAudioSpanData`, `GenerateImageSpanData`, `GenerateVideoSpanData` | `generate_content {model}` | `generate_content` | -| `TranscribeSpanData`, `RerankSpanData`, `EvaluateSpanData` | `{kind} {model}` | The span data kind | +| `TranscribeSpanData`, `RerankSpanData`, `DecideSpanData` | `{kind} {model}` | The span data kind | | `ToolExecutionSpanData` | `execute_tool {tool_name}` | `execute_tool` | | Other | The span's own name | — | diff --git a/examples/models/gateway/evaluation.py b/examples/models/gateway/decisions.py similarity index 81% rename from examples/models/gateway/evaluation.py rename to examples/models/gateway/decisions.py index 8eaaf591..b6aec011 100644 --- a/examples/models/gateway/evaluation.py +++ b/examples/models/gateway/decisions.py @@ -1,11 +1,11 @@ -"""Evaluate plain text and structured data with Jev through AI Gateway. +"""Ask decision questions about text and data with Jev through AI Gateway. Set ``AI_GATEWAY_API_KEY`` and run: - uv run python examples/models/gateway/evaluation.py + uv run python examples/models/gateway/decisions.py The two calls below are independent examples. The first asks one question about -plain text. The second asks choice, score, and Noul questions about shared +plain text. The second asks choice, score, and predicate questions about shared structured state in one request. """ @@ -17,23 +17,23 @@ class RefundQuestions(pydantic.BaseModel): - requests_refund: ai.ops.experimental.NoulQuestion + requests_refund: ai.ops.experimental.PredicateQuestion class RefundAnswers(pydantic.BaseModel): - requests_refund: ai.ops.experimental.NoulAnswer + requests_refund: ai.ops.experimental.PredicateAnswer class TicketQuestions(pydantic.BaseModel): queue: ai.ops.experimental.ChoiceQuestion urgency: ai.ops.experimental.ScoreQuestion - refund_warranted: ai.ops.experimental.NoulQuestion + refund_warranted: ai.ops.experimental.PredicateQuestion class TicketAnswers(pydantic.BaseModel): queue: ai.ops.experimental.ChoiceAnswer urgency: ai.ops.experimental.ScoreAnswer - refund_warranted: ai.ops.experimental.NoulAnswer + refund_warranted: ai.ops.experimental.PredicateAnswer async def main() -> None: @@ -42,13 +42,13 @@ async def main() -> None: print("Set AI_GATEWAY_API_KEY to run this example.") return - # Ask a single Noul question about plain text. Noul answers are + # Ask a single predicate question about plain text. Predicate answers are # probabilities rather than only true or false. - refund_result = await ai.ops.experimental.evaluate( + refund_result = await ai.ops.experimental.decide( model, "Please refund the duplicate charge on my account.", RefundQuestions( - requests_refund=ai.ops.experimental.NoulQuestion( + requests_refund=ai.ops.experimental.PredicateQuestion( instructions="Is the customer asking for a refund?", ) ), @@ -57,12 +57,12 @@ async def main() -> None: print( "refund requested:", - f"{refund_result.value.requests_refund.noul:.0%}", + f"{refund_result.value.requests_refund.probability:.0%}", ) # Ask several question types about the same structured state. This is useful # when related decisions should use exactly the same source information. - ticket: ai.ops.experimental.EvaluationInput = { + ticket: ai.ops.experimental.DecisionInput = { "message": ( "I was charged $240 twice for the same renewal. The service works, " "but please refund the duplicate charge." @@ -88,7 +88,7 @@ async def main() -> None: "service_status": "operational", } - result = await ai.ops.experimental.evaluate( + result = await ai.ops.experimental.decide( model, ticket, TicketQuestions( @@ -110,7 +110,7 @@ async def main() -> None: "Critical: security incident, outage, or ongoing loss", ], ), - refund_warranted=ai.ops.experimental.NoulQuestion( + refund_warranted=ai.ops.experimental.PredicateQuestion( instructions="Does the evidence warrant a refund?", criteria={ "true": "A duplicate settlement or billing error is shown", @@ -120,7 +120,7 @@ async def main() -> None: ), output_type=TicketAnswers, # Provider options are optional and apply only to this request. - params=ai.ops.experimental.EvaluationParams( + params=ai.ops.experimental.DecideParams( provider_options={ "gateway": { "zeroDataRetention": True, @@ -141,7 +141,7 @@ async def main() -> None: print( "\nrefund warranted:", - f"{answers.refund_warranted.noul:.0%}", + f"{answers.refund_warranted.probability:.0%}", ) # Every operation also exposes framework and provider metadata. diff --git a/examples/models/gateway/jev_python_ast.py b/examples/models/gateway/jev_python_ast.py index 612bd6b0..108d49a2 100644 --- a/examples/models/gateway/jev_python_ast.py +++ b/examples/models/gateway/jev_python_ast.py @@ -775,7 +775,7 @@ async def generate( status(f"Filling {'.'.join(map(str, path))} ({kind})") started = time.perf_counter() try: - result = await ai.ops.experimental.evaluate( + result = await ai.ops.experimental.decide( model, state.model_dump(), questions, output_type=Answers ) break diff --git a/skills/ai/SKILL.md b/skills/ai/SKILL.md index 265bc4ab..19586f7f 100644 --- a/skills/ai/SKILL.md +++ b/skills/ai/SKILL.md @@ -1,6 +1,6 @@ --- name: ai -description: AI SDK for Python (the `ai` package). Use for Python model calls (LLMs, image, video, speech, embedding, transcription, reranking, or evaluation), agents, model interaction tests, tool calling, subagents, approvals, durable execution, telemetry, AI SDK UI backends, and custom providers. +description: AI SDK for Python (the `ai` package). Use for Python model calls (LLMs, image, video, speech, embedding, transcription, reranking, or decisions), agents, model interaction tests, tool calling, subagents, approvals, durable execution, telemetry, AI SDK UI backends, and custom providers. metadata: sdk-version: "0.5.0" --- @@ -71,7 +71,7 @@ read the listed local notes before writing code. | Provider clients, options, discovery | `basics/providers.md` | — | | Structured output, complex streams | `basics/streaming.md` | — | | Buffered language-model calls | `basics/streaming.md` | — | -| Images, video, speech, embeddings, transcription, reranking, evaluation | `basics/model-operations.md` | — | +| Images, video, speech, embeddings, transcription, reranking, decisions | `basics/model-operations.md` | — | | Events and serialization | `basics/messages-and-events.md` | — | | Advanced tools, streaming, aggregation | `basics/tools.md` | [streaming-tools.md](references/streaming-tools.md) | | Advanced agent behavior | `basics/agents.md` | — | diff --git a/src/ai/experimental_telemetry/__init__.py b/src/ai/experimental_telemetry/__init__.py index ceb74415..a6313427 100644 --- a/src/ai/experimental_telemetry/__init__.py +++ b/src/ai/experimental_telemetry/__init__.py @@ -5,6 +5,9 @@ See :mod:`.span` for the full story. """ +import warnings +from typing import TYPE_CHECKING, Any + from .span import ( FIRST_TOKEN, HOOK_CANCELLED, @@ -17,9 +20,9 @@ AiGenerateSpanData, AiStreamSpanData, CustomSpanData, + DecideSpanData, DictSink, EmbedSpanData, - EvaluateSpanData, GenerateAudioSpanData, GenerateImageSpanData, GenerateVideoSpanData, @@ -60,9 +63,9 @@ "AiGenerateSpanData", "AiStreamSpanData", "CustomSpanData", + "DecideSpanData", "DictSink", "EmbedSpanData", - "EvaluateSpanData", "GenerateAudioSpanData", "GenerateImageSpanData", "GenerateVideoSpanData", @@ -90,3 +93,19 @@ "use_span", "use_time", ] + +if TYPE_CHECKING: + EvaluateSpanData = DecideSpanData +else: + + def __getattr__(name: str) -> Any: + # Deprecated names, kept as aliases of their replacements. + if name == "EvaluateSpanData": + warnings.warn( + "ai.experimental_telemetry.EvaluateSpanData is deprecated, " + "use ai.experimental_telemetry.DecideSpanData", + DeprecationWarning, + stacklevel=2, + ) + return DecideSpanData + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/ai/experimental_telemetry/otel.py b/src/ai/experimental_telemetry/otel.py index d3775d04..68a0551a 100644 --- a/src/ai/experimental_telemetry/otel.py +++ b/src/ai/experimental_telemetry/otel.py @@ -70,8 +70,8 @@ def _semconv_name(sp: telemetry.Span) -> str: (telemetry.AiStreamSpanData() | telemetry.AiGenerateSpanData()) as d ): return f"chat {d.model}" - case telemetry.EvaluateSpanData() as d: - return f"evaluate {d.model}" + case telemetry.DecideSpanData() as d: + return f"decide {d.model}" case telemetry.EmbedSpanData() as d: return f"embeddings {d.model}" case ( @@ -101,7 +101,7 @@ def _semconv_kind(sp: telemetry.Span) -> opentelemetry.trace.SpanKind: case ( telemetry.AiStreamSpanData() | telemetry.AiGenerateSpanData() - | telemetry.EvaluateSpanData() + | telemetry.DecideSpanData() | telemetry.EmbedSpanData() | telemetry.GenerateAudioSpanData() | telemetry.GenerateImageSpanData() @@ -348,8 +348,8 @@ def _attributes(sp: telemetry.Span, *, capture_content: bool) -> dict[str, Any]: error=sp.error is not None, finish_reason=d.finish_reason, ) - case telemetry.EvaluateSpanData() as d: - attrs["gen_ai.operation.name"] = "evaluate" + case telemetry.DecideSpanData() as d: + attrs["gen_ai.operation.name"] = "decide" if d.provider is not None: attrs["gen_ai.provider.name"] = d.provider attrs["gen_ai.request.model"] = d.model diff --git a/src/ai/experimental_telemetry/span.py b/src/ai/experimental_telemetry/span.py index 3b5d983c..4e913980 100644 --- a/src/ai/experimental_telemetry/span.py +++ b/src/ai/experimental_telemetry/span.py @@ -267,10 +267,10 @@ class AiGenerateSpanData(pydantic.BaseModel): response_model: str | None = None -class EvaluateSpanData(pydantic.BaseModel): - """One typed evaluation operation.""" +class DecideSpanData(pydantic.BaseModel): + """One typed decision operation.""" - kind: Literal["evaluate"] = "evaluate" + kind: Literal["decide"] = "decide" model: str provider: str | None = None question_count: int @@ -442,7 +442,7 @@ def from_exception(cls, exc: BaseException) -> SpanError: | LoopTurnSpanData | AiStreamSpanData | AiGenerateSpanData - | EvaluateSpanData + | DecideSpanData | EmbedSpanData | GenerateAudioSpanData | GenerateImageSpanData diff --git a/src/ai/experimental_telemetry/utils/console.py b/src/ai/experimental_telemetry/utils/console.py index 73dacd9d..82d2c6bf 100644 --- a/src/ai/experimental_telemetry/utils/console.py +++ b/src/ai/experimental_telemetry/utils/console.py @@ -43,8 +43,8 @@ def _label(sp: telemetry.Span) -> str: else "" ) return f"generate {d.model}{tokens}" - case telemetry.EvaluateSpanData() as d: - return f"evaluate {d.model}" + case telemetry.DecideSpanData() as d: + return f"decide {d.model}" case telemetry.EmbedSpanData() as d: tokens = ( f" in:{d.usage.input_tokens} out:{d.usage.output_tokens} tok" diff --git a/src/ai/ops/experimental/__init__.py b/src/ai/ops/experimental/__init__.py index a3d23f48..36a66697 100644 --- a/src/ai/ops/experimental/__init__.py +++ b/src/ai/ops/experimental/__init__.py @@ -1,27 +1,62 @@ """Experimental model operations.""" -from .evaluation import ( +import warnings +from typing import TYPE_CHECKING, Any + +from .decisions import ( ChoiceAnswer, ChoiceQuestion, - EvaluationInput, - EvaluationParams, - NoulAnswer, - NoulCriteria, - NoulQuestion, + DecideParams, + DecisionInput, + PredicateAnswer, + PredicateCriteria, + PredicateQuestion, ScoreAnswer, ScoreQuestion, - evaluate, + decide, ) +# Deprecated names, kept as aliases of their replacements. +_DEPRECATED = { + "evaluate": "decide", + "EvaluationInput": "DecisionInput", + "EvaluationParams": "DecideParams", + "NoulAnswer": "PredicateAnswer", + "NoulCriteria": "PredicateCriteria", + "NoulQuestion": "PredicateQuestion", +} + +if TYPE_CHECKING: + evaluate = decide + EvaluationInput = DecisionInput + EvaluationParams = DecideParams + NoulAnswer = PredicateAnswer + NoulCriteria = PredicateCriteria + NoulQuestion = PredicateQuestion +else: + + def __getattr__(name: str) -> Any: + if name in _DEPRECATED: + new = _DEPRECATED[name] + warnings.warn( + f"ai.ops.experimental.{name} is deprecated, " + f"use ai.ops.experimental.{new}", + DeprecationWarning, + stacklevel=2, + ) + return globals()[new] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + __all__ = [ "ChoiceAnswer", "ChoiceQuestion", - "EvaluationInput", - "EvaluationParams", - "NoulAnswer", - "NoulCriteria", - "NoulQuestion", + "DecideParams", + "DecisionInput", + "PredicateAnswer", + "PredicateCriteria", + "PredicateQuestion", "ScoreAnswer", "ScoreQuestion", - "evaluate", + "decide", ] diff --git a/src/ai/ops/experimental/evaluation.py b/src/ai/ops/experimental/decisions.py similarity index 70% rename from src/ai/ops/experimental/evaluation.py rename to src/ai/ops/experimental/decisions.py index e77f56bc..018c6654 100644 --- a/src/ai/ops/experimental/evaluation.py +++ b/src/ai/ops/experimental/decisions.py @@ -1,4 +1,4 @@ -"""Typed evaluation of shared state via dedicated evaluation models. +"""Typed decisions about shared state via dedicated decision models. :: @@ -6,16 +6,16 @@ import pydantic class Questions(pydantic.BaseModel): - requests_refund: ai.ops.experimental.NoulQuestion + requests_refund: ai.ops.experimental.PredicateQuestion class Answers(pydantic.BaseModel): - requests_refund: ai.ops.experimental.NoulAnswer + requests_refund: ai.ops.experimental.PredicateAnswer - result = await ai.ops.experimental.evaluate( + result = await ai.ops.experimental.decide( ai.get_model("typesafe-ai/jev"), {"message": "Please refund the duplicate charge."}, Questions( - requests_refund=ai.ops.experimental.NoulQuestion( + requests_refund=ai.ops.experimental.PredicateQuestion( instructions="Is the customer requesting a refund?", ), ), @@ -27,6 +27,7 @@ class Answers(pydantic.BaseModel): from __future__ import annotations import dataclasses +import warnings from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING, Annotated, Any, Literal, TypedDict, overload @@ -39,16 +40,16 @@ class Answers(pydantic.BaseModel): from ...models.core import model as model_ -type EvaluationInput = ( +type DecisionInput = ( str | dict[str, pydantic.JsonValue] | list[pydantic.JsonValue] ) -class NoulCriteria(TypedDict, total=False): - """Optional descriptions of when a Noul answer is true or false.""" +class PredicateCriteria(TypedDict, total=False): + """Optional descriptions of when a predicate answer is true or false.""" - true: EvaluationInput | None - false: EvaluationInput | None + true: DecisionInput | None + false: DecisionInput | None _QUESTION_CONFIG = pydantic.ConfigDict( @@ -62,9 +63,9 @@ class NoulCriteria(TypedDict, total=False): class ChoiceQuestion(pydantic.BaseModel): """Choose one key from a nonempty map of options.""" - instructions: EvaluationInput + instructions: DecisionInput criteria: Annotated[ - dict[str, EvaluationInput | None], + dict[str, DecisionInput | None], pydantic.Field(min_length=1), ] type: Literal["choice"] = "choice" @@ -75,9 +76,9 @@ class ChoiceQuestion(pydantic.BaseModel): class ScoreQuestion(pydantic.BaseModel): """Score state against at least two ordered rubric levels.""" - instructions: EvaluationInput + instructions: DecisionInput criteria: Annotated[ - list[EvaluationInput | None], + list[DecisionInput | None], pydantic.Field(min_length=2), ] type: Literal["score"] = "score" @@ -85,15 +86,28 @@ class ScoreQuestion(pydantic.BaseModel): model_config = _QUESTION_CONFIG -class NoulQuestion(pydantic.BaseModel): +class PredicateQuestion(pydantic.BaseModel): """Estimate the probability that a statement about state is true.""" - instructions: EvaluationInput - criteria: NoulCriteria | None = None - type: Literal["noul"] = "noul" + instructions: DecisionInput + criteria: PredicateCriteria | None = None + type: Literal["predicate"] = "predicate" model_config = _QUESTION_CONFIG + @pydantic.model_validator(mode="before") + @classmethod + def _accept_noul(cls, data: Any) -> Any: + # Deprecated: accept the old "noul" type name. + if isinstance(data, dict) and data.get("type") == "noul": + warnings.warn( + 'type="noul" is deprecated, use type="predicate"', + DeprecationWarning, + stacklevel=2, + ) + data = {**data, "type": "predicate"} + return data + Probability = Annotated[ pydantic.StrictFloat, @@ -127,18 +141,48 @@ class ScoreAnswer(pydantic.BaseModel): model_config = pydantic.ConfigDict(frozen=True) -class NoulAnswer(pydantic.BaseModel): +class PredicateAnswer(pydantic.BaseModel): """Model-estimated probability that the answer is true.""" - type: Literal["noul"] = "noul" - noul: Probability + type: Literal["predicate"] = "predicate" + probability: Probability model_config = pydantic.ConfigDict(frozen=True) + @pydantic.model_validator(mode="before") + @classmethod + def _accept_noul(cls, data: Any) -> Any: + # Deprecated: accept the old "noul" type name and answer field. + if isinstance(data, dict) and ( + data.get("type") == "noul" or "noul" in data + ): + warnings.warn( + 'type="noul" and the noul field are deprecated, ' + 'use type="predicate" and probability', + DeprecationWarning, + stacklevel=2, + ) + data = {**data} + if data.get("type") == "noul": + data["type"] = "predicate" + if "noul" in data: + data["probability"] = data.pop("noul") + return data + + @property + def noul(self) -> float: + """Deprecated alias for :attr:`probability`.""" + warnings.warn( + "PredicateAnswer.noul is deprecated, use probability", + DeprecationWarning, + stacklevel=2, + ) + return self.probability + @dataclasses.dataclass(frozen=True, kw_only=True) -class EvaluationParams: - """Parameters for evaluation.""" +class DecideParams: + """Parameters for decisions.""" provider_options: Mapping[str, Any] = dataclasses.field( default_factory=dict @@ -146,56 +190,56 @@ class EvaluationParams: """Provider-specific options, keyed by provider name.""" -type _Question = ChoiceQuestion | ScoreQuestion | NoulQuestion -type _Answer = ChoiceAnswer | ScoreAnswer | NoulAnswer +type _Question = ChoiceQuestion | ScoreQuestion | PredicateQuestion +type _Answer = ChoiceAnswer | ScoreAnswer | PredicateAnswer -_QUESTION_TYPES = (ChoiceQuestion, ScoreQuestion, NoulQuestion) +_QUESTION_TYPES = (ChoiceQuestion, ScoreQuestion, PredicateQuestion) _ANSWER_TYPES: dict[type[_Question], type[pydantic.BaseModel]] = { ChoiceQuestion: ChoiceAnswer, ScoreQuestion: ScoreAnswer, - NoulQuestion: NoulAnswer, + PredicateQuestion: PredicateAnswer, } -_INPUT_ADAPTER: pydantic.TypeAdapter[EvaluationInput] = pydantic.TypeAdapter( - EvaluationInput, +_INPUT_ADAPTER: pydantic.TypeAdapter[DecisionInput] = pydantic.TypeAdapter( + DecisionInput, config=pydantic.ConfigDict(allow_inf_nan=False, strict=True), ) @overload -async def evaluate[AnswerT: pydantic.BaseModel]( +async def decide[AnswerT: pydantic.BaseModel]( model: model_.Model, - state: EvaluationInput, + state: DecisionInput, questions: pydantic.BaseModel, *, output_type: type[AnswerT], - params: EvaluationParams | None = None, + params: DecideParams | None = None, ) -> items.Item[AnswerT]: ... @overload -async def evaluate( +async def decide( model: model_.Model, - state: EvaluationInput, + state: DecisionInput, questions: Mapping[str, pydantic.BaseModel], *, output_type: None = None, - params: EvaluationParams | None = None, + params: DecideParams | None = None, ) -> items.Item[dict[str, _Answer]]: ... -async def evaluate( +async def decide( model: model_.Model, - state: EvaluationInput, + state: DecisionInput, questions: pydantic.BaseModel | Mapping[str, pydantic.BaseModel], *, output_type: type[pydantic.BaseModel] | None = None, - params: EvaluationParams | None = None, + params: DecideParams | None = None, ) -> items.Item[Any]: - """Evaluate questions against one shared state. + """Decide questions about one shared state. Pass matching Pydantic question and output models for a statically typed value, or pass a question mapping to receive an answer mapping. Each choice, - score, or Noul question is validated against its corresponding answer. + score, or predicate question is validated against its corresponding answer. Experimental: not part of the stable API, may change or be removed. """ @@ -253,8 +297,8 @@ async def evaluate( normalized[name] = question # Start telemetry after local validation, immediately around provider work. - params = params or EvaluationParams() - data = telemetry.EvaluateSpanData( + params = params or DecideParams() + data = telemetry.DecideSpanData( model=model.id, provider=model.provider.name, question_count=len(normalized), @@ -262,7 +306,7 @@ async def evaluate( async with telemetry.span(data) as sp: # Providers operate on one normalized question mapping and return raw # answer data. - raw_item = await model.provider.evaluate( + raw_item = await model.provider.decide( model, state, normalized, @@ -289,7 +333,7 @@ async def evaluate( elif isinstance(question, ScoreQuestion): answers[name] = ScoreAnswer.model_validate(raw_answer) else: - answers[name] = NoulAnswer.model_validate(raw_answer) + answers[name] = PredicateAnswer.model_validate(raw_answer) value = answers # Rewrap the validated value without dropping operation metadata. diff --git a/src/ai/providers/ai_gateway/protocol/v4.py b/src/ai/providers/ai_gateway/protocol/v4.py index 2e7bf1e1..480163ad 100644 --- a/src/ai/providers/ai_gateway/protocol/v4.py +++ b/src/ai/providers/ai_gateway/protocol/v4.py @@ -528,29 +528,29 @@ async def stream( # --------------------------------------------------------------------------- -# Evaluation +# Decisions # --------------------------------------------------------------------------- -async def evaluate( +async def decide( gateway: gateway_client.GatewayClient, model: models.Model, - state: ops.experimental.EvaluationInput, + state: ops.experimental.DecisionInput, questions: Mapping[ str, ops.experimental.ChoiceQuestion | ops.experimental.ScoreQuestion - | ops.experimental.NoulQuestion, + | ops.experimental.PredicateQuestion, ], *, - params: ops.experimental.EvaluationParams, + params: ops.experimental.DecideParams, ) -> ops.items.Item[dict[str, Any]]: - """Hit ``/evaluation-model`` and return raw evaluation answers.""" + """Hit ``/evaluation-model`` and return raw decision answers.""" wire_questions: dict[str, dict[str, Any]] = {} for question_id, question in questions.items(): wire_question = question.model_dump(mode="json", by_alias=True) - if isinstance(question, ops.experimental.NoulQuestion): - # AI Gateway v4 calls the Noul primitive "Boolean". + if isinstance(question, ops.experimental.PredicateQuestion): + # AI Gateway v4 calls the predicate primitive "Boolean". wire_question["type"] = "boolean" if question.criteria is None: wire_question.pop("criteria") @@ -587,9 +587,7 @@ async def evaluate( if not isinstance(answer, dict): continue if answer.get("type") == "boolean": - answer["type"] = "noul" - if "probability" in answer: - answer["noul"] = answer.pop("probability") + answer["type"] = "predicate" if ( answer.get("type") in ("choice", "score") and question_id in confidence @@ -807,23 +805,23 @@ def stream( params=params, ) - async def evaluate( + async def decide( self, client: gateway_client.GatewayClient, model: models.Model, - state: ops.experimental.EvaluationInput, + state: ops.experimental.DecisionInput, questions: Mapping[ str, ops.experimental.ChoiceQuestion | ops.experimental.ScoreQuestion - | ops.experimental.NoulQuestion, + | ops.experimental.PredicateQuestion, ], *, - params: ops.experimental.EvaluationParams, + params: ops.experimental.DecideParams, provider: str, ) -> ops.items.Item[dict[str, Any]]: _ = provider - return await evaluate( + return await decide( client, model, state, diff --git a/src/ai/providers/base.py b/src/ai/providers/base.py index 838c8e2a..4cc826e2 100644 --- a/src/ai/providers/base.py +++ b/src/ai/providers/base.py @@ -33,7 +33,7 @@ transcriptions, videos, ) - from ..ops.experimental import evaluation + from ..ops.experimental import decisions from ..types import events from ..types import messages as messages_ from ..types import tools as tools_ @@ -179,24 +179,24 @@ async def generate( f"protocol {type(self).__name__!r} does not support generate()" ) - async def evaluate( + async def decide( self, client: ClientT, model: model_.Model, - state: evaluation.EvaluationInput, + state: decisions.DecisionInput, questions: Mapping[ str, - evaluation.ChoiceQuestion - | evaluation.ScoreQuestion - | evaluation.NoulQuestion, + decisions.ChoiceQuestion + | decisions.ScoreQuestion + | decisions.PredicateQuestion, ], *, - params: evaluation.EvaluationParams, + params: decisions.DecideParams, provider: str, ) -> items.Item[dict[str, Any]]: - """Evaluate typed questions against shared state using *client*.""" + """Decide typed questions about shared state using *client*.""" raise NotImplementedError( - f"protocol {type(self).__name__!r} does not support evaluate()" + f"protocol {type(self).__name__!r} does not support decide()" ) async def generate_image( @@ -530,22 +530,22 @@ async def generate( provider=self.name, ) - async def evaluate( + async def decide( self, model: model_.Model, - state: evaluation.EvaluationInput, + state: decisions.DecisionInput, questions: Mapping[ str, - evaluation.ChoiceQuestion - | evaluation.ScoreQuestion - | evaluation.NoulQuestion, + decisions.ChoiceQuestion + | decisions.ScoreQuestion + | decisions.PredicateQuestion, ], *, - params: evaluation.EvaluationParams, + params: decisions.DecideParams, ) -> items.Item[dict[str, Any]]: - """Evaluate typed questions against shared state with this provider.""" + """Decide typed questions about shared state with this provider.""" selected_protocol = model.protocol or self.protocol - return await selected_protocol.evaluate( + return await selected_protocol.decide( self.client, model, state, diff --git a/src/ai/providers/typesafe/__init__.py b/src/ai/providers/typesafe/__init__.py index 808fd0f7..b1df1bb6 100644 --- a/src/ai/providers/typesafe/__init__.py +++ b/src/ai/providers/typesafe/__init__.py @@ -6,9 +6,9 @@ from ai.providers.typesafe import TypeSafeProvider model = ai.Model(id="jev-latest", provider=TypeSafeProvider()) - result = await ai.ops.experimental.evaluate(model, state, questions) + result = await ai.ops.experimental.decide(model, state, questions) -The provider supports evaluation only. The optional upstream TypeSafe SDK is +The provider supports decisions only. The optional upstream TypeSafe SDK is loaded lazily when the provider creates or uses an SDK client. """ diff --git a/src/ai/providers/typesafe/protocol.py b/src/ai/providers/typesafe/protocol.py index 795fc817..40ec30a4 100644 --- a/src/ai/providers/typesafe/protocol.py +++ b/src/ai/providers/typesafe/protocol.py @@ -21,33 +21,36 @@ class TypeSafeSystemOneProtocol(base.ProviderProtocol[TypeSafeClient]): - """TypeSafe ``/v1/systemone`` evaluation protocol.""" + """TypeSafe ``/v1/systemone`` decision protocol.""" protocol_class_id: Literal["typesafe-system-one"] = "typesafe-system-one" - async def evaluate( + async def decide( self, client: TypeSafeClient, model: model_.Model, - state: ops.experimental.EvaluationInput, + state: ops.experimental.DecisionInput, questions: Mapping[ str, ops.experimental.ChoiceQuestion | ops.experimental.ScoreQuestion - | ops.experimental.NoulQuestion, + | ops.experimental.PredicateQuestion, ], *, - params: ops.experimental.EvaluationParams, + params: ops.experimental.DecideParams, provider: str, ) -> ops.items.Item[dict[str, Any]]: - """Hit ``/v1/systemone`` and return raw evaluation answers.""" + """Hit ``/v1/systemone`` and return raw decision answers.""" typesafe = _sdk.import_sdk(provider=provider) - # The native wire format matches the question models, except that an - # absent Noul criteria is omitted rather than sent as null. + # The native wire format matches the question models, except that + # TypeSafe calls the predicate primitive "noul" and an absent + # predicate criteria is omitted rather than sent as null. wire_questions: dict[str, Any] = {} for question_id, question in questions.items(): wire_question = question.model_dump(mode="json") + if wire_question["type"] == "predicate": + wire_question["type"] = "noul" if wire_question.get("criteria", ...) is None: del wire_question["criteria"] wire_questions[question_id] = wire_question @@ -78,6 +81,10 @@ async def evaluate( data = answer.model_dump(mode="json") if "legend" in data: legends[question_id] = data.pop("legend") + if data.get("type") == "noul": + data["type"] = "predicate" + if "noul" in data: + data["probability"] = data.pop("noul") answers[question_id] = data typesafe_metadata: dict[str, Any] = {"model": response.model} diff --git a/tests/experimental_telemetry/test_otel.py b/tests/experimental_telemetry/test_otel.py index dc13f533..bca7d3c7 100644 --- a/tests/experimental_telemetry/test_otel.py +++ b/tests/experimental_telemetry/test_otel.py @@ -653,8 +653,8 @@ def test_generate_image_span_operation_name() -> None: assert "gen_ai.embeddings.dimension.count" not in attrs -def test_evaluate_span_operation_name() -> None: - data = ai.experimental_telemetry.EvaluateSpanData( +def test_decide_span_operation_name() -> None: + data = ai.experimental_telemetry.DecideSpanData( question_count=3, model="typesafe-ai/jev", provider="ai-gateway", @@ -663,10 +663,10 @@ def test_evaluate_span_operation_name() -> None: sp = ai.experimental_telemetry.Span( name="ops", data=data, id="span-1", trace_id="trace-1" ) - assert otel._semconv_name(sp) == "evaluate typesafe-ai/jev" + assert otel._semconv_name(sp) == "decide typesafe-ai/jev" assert otel._semconv_kind(sp) == SpanKind.CLIENT attrs = otel._attributes(sp, capture_content=False) - assert attrs["gen_ai.operation.name"] == "evaluate" + assert attrs["gen_ai.operation.name"] == "decide" assert attrs["gen_ai.provider.name"] == "ai-gateway" assert attrs["gen_ai.request.model"] == "typesafe-ai/jev" assert attrs["gen_ai.usage.input_tokens"] == 9 diff --git a/tests/ops/experimental/test_evaluation.py b/tests/ops/experimental/test_decisions.py similarity index 54% rename from tests/ops/experimental/test_evaluation.py rename to tests/ops/experimental/test_decisions.py index 7aeb6d15..7c3adece 100644 --- a/tests/ops/experimental/test_evaluation.py +++ b/tests/ops/experimental/test_decisions.py @@ -1,4 +1,4 @@ -"""Tests for ``ai.ops.experimental.evaluate`` dispatch and input validation.""" +"""Tests for ``ai.ops.experimental.decide`` dispatch and input validation.""" from __future__ import annotations @@ -13,43 +13,43 @@ from ... import conftest -type EvaluationQuestion = ( +type DecisionQuestion = ( ops.experimental.ChoiceQuestion | ops.experimental.ScoreQuestion - | ops.experimental.NoulQuestion + | ops.experimental.PredicateQuestion ) class Questions(pydantic.BaseModel): department: ops.experimental.ChoiceQuestion severity: ops.experimental.ScoreQuestion - refund: ops.experimental.NoulQuestion + refund: ops.experimental.PredicateQuestion class Answers(pydantic.BaseModel): department: ops.experimental.ChoiceAnswer severity: ops.experimental.ScoreAnswer - refund: ops.experimental.NoulAnswer + refund: ops.experimental.PredicateAnswer -class EvaluationProvider(models.Provider): - provider_class_id: Literal["test-evaluation-provider"] = ( - "test-evaluation-provider" +class DecisionProvider(models.Provider): + provider_class_id: Literal["test-decision-provider"] = ( + "test-decision-provider" ) - name: str = "mock-evaluation" + name: str = "mock-decision" default_base_url: str = "http://mock.test" api_key_env: str | None = None async def list_models(self) -> list[str]: return [] - async def evaluate( + async def decide( self, model: models.Model, - state: ops.experimental.EvaluationInput, - questions: Mapping[str, EvaluationQuestion], + state: ops.experimental.DecisionInput, + questions: Mapping[str, DecisionQuestion], *, - params: ops.experimental.EvaluationParams, + params: ops.experimental.DecideParams, ) -> ops.Item[dict[str, Any]]: assert state == {"message": "refund me"} assert set(questions) == {"department", "severity", "refund"} @@ -68,7 +68,7 @@ async def evaluate( "score": 1.5, "probabilities": {"0": 0.0, "1": 0.5, "2": 0.5}, }, - "refund": {"type": "noul", "noul": 0.98}, + "refund": {"type": "predicate", "probability": 0.98}, }, usage=ai.types.usage.Usage(input_tokens=12), metadata={"rounding": {"probabilityDecimals": 2}}, @@ -76,11 +76,11 @@ async def evaluate( ) -class StaticEvaluationProvider(models.Provider): - provider_class_id: Literal["test-static-evaluation-provider"] = ( - "test-static-evaluation-provider" +class StaticDecisionProvider(models.Provider): + provider_class_id: Literal["test-static-decision-provider"] = ( + "test-static-decision-provider" ) - name: str = "mock-static-evaluation" + name: str = "mock-static-decision" default_base_url: str = "http://mock.test" api_key_env: str | None = None answers: dict[str, Any] @@ -88,13 +88,13 @@ class StaticEvaluationProvider(models.Provider): async def list_models(self) -> list[str]: return [] - async def evaluate( + async def decide( self, model: models.Model, - state: ops.experimental.EvaluationInput, - questions: Mapping[str, EvaluationQuestion], + state: ops.experimental.DecisionInput, + questions: Mapping[str, DecisionQuestion], *, - params: ops.experimental.EvaluationParams, + params: ops.experimental.DecideParams, ) -> ops.Item[dict[str, Any]]: return ops.Item(value=self.answers) @@ -112,24 +112,22 @@ def questions() -> Questions: instructions={"task": "Rate severity"}, criteria=["Cosmetic", "Workaround exists", "Blocking"], ), - refund=ops.experimental.NoulQuestion( + refund=ops.experimental.PredicateQuestion( instructions="Is the customer requesting a refund?", criteria={"true": "Refund requested", "false": None}, ), ) -async def test_evaluate_dispatch_and_span(recorder: conftest.Recorder) -> None: - model = models.Model( - id="mock-evaluation-model", provider=EvaluationProvider() - ) +async def test_decide_dispatch_and_span(recorder: conftest.Recorder) -> None: + model = models.Model(id="mock-decision-model", provider=DecisionProvider()) - result = await ops.experimental.evaluate( + result = await ops.experimental.decide( model, {"message": "refund me"}, questions(), output_type=Answers, - params=ops.experimental.EvaluationParams( + params=ops.experimental.DecideParams( provider_options={"gateway": {"zeroDataRetention": True}} ), ) @@ -137,32 +135,30 @@ async def test_evaluate_dispatch_and_span(recorder: conftest.Recorder) -> None: assert_type(result, ops.Item[Answers]) assert isinstance(result.value, Answers) assert result.value.department.choice == "billing" - assert result.value.refund.noul == 0.98 + assert result.value.refund.probability == 0.98 assert result.metadata == {"rounding": {"probabilityDecimals": 2}} assert result.provider_metadata == {"test": {"request_id": "req_1"}} (span,) = recorder.ended - assert isinstance(span.data, ai.experimental_telemetry.EvaluateSpanData) + assert isinstance(span.data, ai.experimental_telemetry.DecideSpanData) assert span.data.question_count == 3 assert span.data.answer_count == 3 assert span.data.usage == ai.types.usage.Usage(input_tokens=12) -async def test_evaluate_dynamic_questions() -> None: - model = models.Model( - id="mock-evaluation-model", provider=EvaluationProvider() - ) +async def test_decide_dynamic_questions() -> None: + model = models.Model(id="mock-decision-model", provider=DecisionProvider()) dynamic_questions = { "department": questions().department, "severity": questions().severity, "refund": questions().refund, } - result = await ops.experimental.evaluate( + result = await ops.experimental.decide( model, {"message": "refund me"}, dynamic_questions, - params=ops.experimental.EvaluationParams( + params=ops.experimental.DecideParams( provider_options={"gateway": {"zeroDataRetention": True}} ), ) @@ -174,7 +170,7 @@ async def test_evaluate_dynamic_questions() -> None: str, ops.experimental.ChoiceAnswer | ops.experimental.ScoreAnswer - | ops.experimental.NoulAnswer, + | ops.experimental.PredicateAnswer, ] ], ) @@ -182,31 +178,31 @@ async def test_evaluate_dynamic_questions() -> None: assert result.value["department"].choice == "billing" assert isinstance(result.value["severity"], ops.experimental.ScoreAnswer) assert result.value["severity"].score == 1.5 - assert isinstance(result.value["refund"], ops.experimental.NoulAnswer) - assert result.value["refund"].noul == 0.98 + assert isinstance(result.value["refund"], ops.experimental.PredicateAnswer) + assert result.value["refund"].probability == 0.98 assert result.metadata == {"rounding": {"probabilityDecimals": 2}} -async def test_evaluate_raises_not_implemented() -> None: +async def test_decide_raises_not_implemented() -> None: provider = ai.get_provider("openai", api_key="[redacted]") - model = ai.Model(id="evaluation-test", provider=provider) + model = ai.Model(id="decision-test", provider=provider) - class NoulQuestions(pydantic.BaseModel): - answer: ops.experimental.NoulQuestion + class PredicateQuestions(pydantic.BaseModel): + answer: ops.experimental.PredicateQuestion - class NoulAnswers(pydantic.BaseModel): - answer: ops.experimental.NoulAnswer + class PredicateAnswers(pydantic.BaseModel): + answer: ops.experimental.PredicateAnswer - with pytest.raises(NotImplementedError, match="evaluate"): - await ops.experimental.evaluate( + with pytest.raises(NotImplementedError, match="decide"): + await ops.experimental.decide( model, "state", - NoulQuestions( - answer=ops.experimental.NoulQuestion( + PredicateQuestions( + answer=ops.experimental.PredicateQuestion( instructions="Is this valid?" ) ), - output_type=NoulAnswers, + output_type=PredicateAnswers, ) @@ -222,87 +218,115 @@ def test_score_question_requires_two_levels() -> None: ) -def test_noul_question_rejects_unknown_criteria() -> None: +def test_predicate_question_rejects_unknown_criteria() -> None: with pytest.raises(pydantic.ValidationError): - ops.experimental.NoulQuestion( + ops.experimental.PredicateQuestion( instructions="Decide", - criteria=cast("ops.experimental.NoulCriteria", {"maybe": "Maybe"}), + criteria=cast( + "ops.experimental.PredicateCriteria", {"maybe": "Maybe"} + ), ) +@pytest.mark.parametrize( + ("old", "new"), + [ + ("evaluate", "decide"), + ("EvaluationInput", "DecisionInput"), + ("EvaluationParams", "DecideParams"), + ("NoulAnswer", "PredicateAnswer"), + ("NoulCriteria", "PredicateCriteria"), + ("NoulQuestion", "PredicateQuestion"), + ], +) +def test_deprecated_aliases(old: str, new: str) -> None: + with pytest.warns(DeprecationWarning, match=new): + alias = getattr(ops.experimental, old) + assert alias is getattr(ops.experimental, new) + + +def test_deprecated_noul_type() -> None: + with pytest.warns(DeprecationWarning, match="predicate"): + question = ops.experimental.PredicateQuestion.model_validate( + {"type": "noul", "instructions": "Is this valid?"} + ) + assert question.type == "predicate" + + with pytest.warns(DeprecationWarning, match="probability"): + answer = ops.experimental.PredicateAnswer.model_validate( + {"type": "noul", "noul": 0.5} + ) + assert answer == ops.experimental.PredicateAnswer(probability=0.5) + + with pytest.warns(DeprecationWarning, match="probability"): + assert answer.noul == 0.5 + + def test_question_rejects_non_json_instructions() -> None: with pytest.raises(pydantic.ValidationError): - ops.experimental.NoulQuestion( - instructions=cast("ops.experimental.EvaluationInput", object()), + ops.experimental.PredicateQuestion( + instructions=cast("ops.experimental.DecisionInput", object()), ) @pytest.mark.parametrize("state", [1, {"value": float("nan")}]) -async def test_evaluate_validates_state(state: Any) -> None: - model = models.Model( - id="mock-evaluation-model", provider=EvaluationProvider() - ) +async def test_decide_validates_state(state: Any) -> None: + model = models.Model(id="mock-decision-model", provider=DecisionProvider()) with pytest.raises(pydantic.ValidationError): - await ops.experimental.evaluate( + await ops.experimental.decide( model, - cast("ops.experimental.EvaluationInput", state), + cast("ops.experimental.DecisionInput", state), questions(), output_type=Answers, ) -async def test_evaluate_requires_question_values() -> None: - model = models.Model( - id="mock-evaluation-model", provider=EvaluationProvider() - ) +async def test_decide_requires_question_values() -> None: + model = models.Model(id="mock-decision-model", provider=DecisionProvider()) with pytest.raises(TypeError, match="must contain a question"): - await ops.experimental.evaluate( + await ops.experimental.decide( model, "state", cast( - "Mapping[str, EvaluationQuestion]", + "Mapping[str, DecisionQuestion]", {"answer": "invalid"}, ), ) -async def test_evaluate_requires_questions() -> None: +async def test_decide_requires_questions() -> None: class EmptyQuestions(pydantic.BaseModel): pass class EmptyAnswers(pydantic.BaseModel): pass - model = models.Model( - id="mock-evaluation-model", provider=EvaluationProvider() - ) + model = models.Model(id="mock-decision-model", provider=DecisionProvider()) with pytest.raises(ValueError, match="questions must not be empty"): - await ops.experimental.evaluate( + await ops.experimental.decide( model, "state", EmptyQuestions(), output_type=EmptyAnswers, ) - empty: dict[str, EvaluationQuestion] = {} + empty: dict[str, DecisionQuestion] = {} with pytest.raises(ValueError, match="questions must not be empty"): - await ops.experimental.evaluate(model, "state", empty) + await ops.experimental.decide(model, "state", empty) -async def test_evaluate_rejects_mixed_modes() -> None: - model = models.Model( - id="mock-evaluation-model", provider=EvaluationProvider() - ) - evaluate = cast("Any", ops.experimental.evaluate) +async def test_decide_rejects_mixed_modes() -> None: + model = models.Model(id="mock-decision-model", provider=DecisionProvider()) + decide = cast("Any", ops.experimental.decide) with pytest.raises(TypeError, match="required for Pydantic"): - await evaluate(model, "state", questions()) + await decide(model, "state", questions()) with pytest.raises(TypeError, match="cannot be used with mapped"): - await evaluate( + await decide( model, "state", {"refund": questions().refund}, @@ -310,36 +334,32 @@ async def test_evaluate_rejects_mixed_modes() -> None: ) -async def test_evaluate_requires_question_fields() -> None: +async def test_decide_requires_question_fields() -> None: class InvalidQuestions(pydantic.BaseModel): answer: str - class NoulAnswers(pydantic.BaseModel): - answer: ops.experimental.NoulAnswer + class PredicateAnswers(pydantic.BaseModel): + answer: ops.experimental.PredicateAnswer - model = models.Model( - id="mock-evaluation-model", provider=EvaluationProvider() - ) + model = models.Model(id="mock-decision-model", provider=DecisionProvider()) with pytest.raises(TypeError, match="must contain a question"): - await ops.experimental.evaluate( + await ops.experimental.decide( model, "state", InvalidQuestions(answer="invalid"), - output_type=NoulAnswers, + output_type=PredicateAnswers, ) -async def test_evaluate_requires_matching_output_fields() -> None: +async def test_decide_requires_matching_output_fields() -> None: class MissingAnswers(pydantic.BaseModel): department: ops.experimental.ChoiceAnswer - model = models.Model( - id="mock-evaluation-model", provider=EvaluationProvider() - ) + model = models.Model(id="mock-decision-model", provider=DecisionProvider()) with pytest.raises(TypeError, match="fields must match"): - await ops.experimental.evaluate( + await ops.experimental.decide( model, "state", questions(), @@ -347,18 +367,16 @@ class MissingAnswers(pydantic.BaseModel): ) -async def test_evaluate_requires_matching_answer_types() -> None: +async def test_decide_requires_matching_answer_types() -> None: class WrongAnswers(pydantic.BaseModel): department: ops.experimental.ChoiceAnswer - severity: ops.experimental.NoulAnswer - refund: ops.experimental.NoulAnswer + severity: ops.experimental.PredicateAnswer + refund: ops.experimental.PredicateAnswer - model = models.Model( - id="mock-evaluation-model", provider=EvaluationProvider() - ) + model = models.Model(id="mock-decision-model", provider=DecisionProvider()) with pytest.raises(TypeError, match="ScoreAnswer"): - await ops.experimental.evaluate( + await ops.experimental.decide( model, "state", questions(), @@ -366,43 +384,43 @@ class WrongAnswers(pydantic.BaseModel): ) -async def test_evaluate_validates_provider_output() -> None: +async def test_decide_validates_provider_output() -> None: model = models.Model( - id="mock-evaluation-model", - provider=StaticEvaluationProvider( + id="mock-decision-model", + provider=StaticDecisionProvider( answers={ "department": {"type": "choice", "choice": "billing"}, "severity": {"type": "score", "score": 1.5}, - "refund": {"type": "noul", "noul": 2.0}, + "refund": {"type": "predicate", "probability": 2.0}, } ), ) with pytest.raises(pydantic.ValidationError): - await ops.experimental.evaluate( + await ops.experimental.decide( model, "state", questions(), output_type=Answers, ) - dynamic_questions: dict[str, EvaluationQuestion] = { + dynamic_questions: dict[str, DecisionQuestion] = { "department": questions().department, "severity": questions().severity, "refund": questions().refund, } with pytest.raises(pydantic.ValidationError): - await ops.experimental.evaluate(model, "state", dynamic_questions) + await ops.experimental.decide(model, "state", dynamic_questions) -async def test_evaluate_requires_matching_answer_fields() -> None: +async def test_decide_requires_matching_answer_fields() -> None: model = models.Model( - id="mock-evaluation-model", - provider=StaticEvaluationProvider(answers={}), + id="mock-decision-model", + provider=StaticDecisionProvider(answers={}), ) with pytest.raises(ValueError, match="answer fields must match"): - await ops.experimental.evaluate( + await ops.experimental.decide( model, "state", questions(), @@ -410,17 +428,15 @@ async def test_evaluate_requires_matching_answer_fields() -> None: ) -async def test_evaluate_rejects_cyclic_state() -> None: +async def test_decide_rejects_cyclic_state() -> None: state: list[Any] = [] state.append(state) - model = models.Model( - id="mock-evaluation-model", provider=EvaluationProvider() - ) + model = models.Model(id="mock-decision-model", provider=DecisionProvider()) with pytest.raises(pydantic.ValidationError): - await ops.experimental.evaluate( + await ops.experimental.decide( model, - cast("ops.experimental.EvaluationInput", state), + cast("ops.experimental.DecisionInput", state), questions(), output_type=Answers, ) diff --git a/tests/providers/ai_gateway/protocol/test_v4_evaluation.py b/tests/providers/ai_gateway/protocol/test_v4_decisions.py similarity index 88% rename from tests/providers/ai_gateway/protocol/test_v4_evaluation.py rename to tests/providers/ai_gateway/protocol/test_v4_decisions.py index ca30a9ff..e305ccb3 100644 --- a/tests/providers/ai_gateway/protocol/test_v4_evaluation.py +++ b/tests/providers/ai_gateway/protocol/test_v4_decisions.py @@ -20,21 +20,21 @@ class Questions(pydantic.BaseModel): department: ops.experimental.ChoiceQuestion severity: ops.experimental.ScoreQuestion - refund: ops.experimental.NoulQuestion + refund: ops.experimental.PredicateQuestion class Answers(pydantic.BaseModel): department: ops.experimental.ChoiceAnswer severity: ops.experimental.ScoreAnswer - refund: ops.experimental.NoulAnswer + refund: ops.experimental.PredicateAnswer -class NoulQuestions(pydantic.BaseModel): - answer: ops.experimental.NoulQuestion +class PredicateQuestions(pydantic.BaseModel): + answer: ops.experimental.PredicateQuestion -class NoulAnswers(pydantic.BaseModel): - answer: ops.experimental.NoulAnswer +class PredicateAnswers(pydantic.BaseModel): + answer: ops.experimental.PredicateAnswer def questions() -> Questions: @@ -47,14 +47,14 @@ def questions() -> Questions: instructions="How severe is this?", criteria=["Cosmetic", "Workaround exists", "Blocking"], ), - refund=ops.experimental.NoulQuestion( + refund=ops.experimental.PredicateQuestion( instructions="Is a refund requested?", criteria={"true": "A refund is requested", "false": None}, ), ) -async def test_evaluate_request_and_response() -> None: +async def test_decide_request_and_response() -> None: captured_body: dict[str, Any] = {} captured_headers: dict[str, str] = {} captured_url: list[str] = [] @@ -95,7 +95,7 @@ def handler(request: httpx.Request) -> httpx.Response: }, ) - result = await ops.experimental.evaluate( + result = await ops.experimental.decide( mock_model( httpx.MockTransport(handler), api_key="sk-test", @@ -104,7 +104,7 @@ def handler(request: httpx.Request) -> httpx.Response: {"message": "Please refund the duplicate charge."}, questions(), output_type=Answers, - params=ops.experimental.EvaluationParams( + params=ops.experimental.DecideParams( provider_options={ "gateway": { "zeroDataRetention": True, @@ -162,7 +162,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert result.value.severity.score == 1.5 assert result.value.department.confidence == 0.91 assert result.value.severity.confidence == 0.87 - assert result.value.refund.noul == 0.98 + assert result.value.refund.probability == 0.98 assert result.metadata == { "rounding": { "probabilityDecimals": 2, @@ -192,7 +192,7 @@ def handler(request: httpx.Request) -> httpx.Response: {"typesafe": {"confidence": {"department": 1.0, "severity": 0.5}}}, ], ) -async def test_evaluate_dynamic_confidence_and_noul( +async def test_decide_dynamic_confidence_and_predicate( provider_metadata: dict[str, Any] | None, ) -> None: def handler(request: httpx.Request) -> httpx.Response: @@ -214,13 +214,13 @@ def handler(request: httpx.Request) -> httpx.Response: ) question_values = questions() - result = await ops.experimental.evaluate( + result = await ops.experimental.decide( mock_model(httpx.MockTransport(handler), model_id=_MODEL_ID), "state", { "department": question_values.department, "severity": question_values.severity, - "refund": ops.experimental.NoulQuestion( + "refund": ops.experimental.PredicateQuestion( instructions="Refund requested?" ), }, @@ -233,15 +233,15 @@ def handler(request: httpx.Request) -> httpx.Response: refund = result.value["refund"] assert isinstance(department, ops.experimental.ChoiceAnswer) assert isinstance(severity, ops.experimental.ScoreAnswer) - assert isinstance(refund, ops.experimental.NoulAnswer) + assert isinstance(refund, ops.experimental.PredicateAnswer) assert department.confidence == confidence.get("department") assert severity.confidence == confidence.get("severity") - assert refund.model_dump() == {"type": "noul", "noul": 0.98} + assert refund.model_dump() == {"type": "predicate", "probability": 0.98} assert result.provider_metadata == provider_metadata @pytest.mark.parametrize("confidence", [-0.1, 1.1, "0.5", True]) -async def test_evaluate_rejects_invalid_confidence(confidence: Any) -> None: +async def test_decide_rejects_invalid_confidence(confidence: Any) -> None: def handler(request: httpx.Request) -> httpx.Response: return httpx.Response( 200, @@ -254,7 +254,7 @@ def handler(request: httpx.Request) -> httpx.Response: ) with pytest.raises(pydantic.ValidationError, match="confidence"): - await ops.experimental.evaluate( + await ops.experimental.decide( mock_model(httpx.MockTransport(handler), model_id=_MODEL_ID), "state", { @@ -265,7 +265,7 @@ def handler(request: httpx.Request) -> httpx.Response: ) -async def test_evaluate_maps_all_warning_types() -> None: +async def test_decide_maps_all_warning_types() -> None: def handler(request: httpx.Request) -> httpx.Response: return httpx.Response( 200, @@ -291,7 +291,7 @@ def handler(request: httpx.Request) -> httpx.Response: }, ) - result = await ops.experimental.evaluate( + result = await ops.experimental.decide( mock_model(httpx.MockTransport(handler), model_id=_MODEL_ID), "state", { @@ -312,7 +312,7 @@ def handler(request: httpx.Request) -> httpx.Response: ] -async def test_evaluate_maps_authentication_error() -> None: +async def test_decide_maps_authentication_error() -> None: def handler(request: httpx.Request) -> httpx.Response: return httpx.Response( 401, @@ -325,13 +325,13 @@ def handler(request: httpx.Request) -> httpx.Response: ) with pytest.raises(ai.ProviderAuthenticationError): - await ops.experimental.evaluate( + await ops.experimental.decide( mock_model(httpx.MockTransport(handler), model_id=_MODEL_ID), "state", - NoulQuestions( - answer=ops.experimental.NoulQuestion( + PredicateQuestions( + answer=ops.experimental.PredicateQuestion( instructions="Is this true?" ) ), - output_type=NoulAnswers, + output_type=PredicateAnswers, ) diff --git a/tests/providers/typesafe/test_protocol.py b/tests/providers/typesafe/test_protocol.py index 76f564dc..f5ae24e6 100644 --- a/tests/providers/typesafe/test_protocol.py +++ b/tests/providers/typesafe/test_protocol.py @@ -9,7 +9,7 @@ import typesafe_sdk import ai -from ai.ops import experimental as evaluation +from ai.ops import experimental as decisions from ai.providers.typesafe import TypeSafeProvider @@ -34,18 +34,18 @@ def _response(answers: dict[str, Any]) -> dict[str, Any]: class Questions(pydantic.BaseModel): - urgent: evaluation.NoulQuestion - department: evaluation.ChoiceQuestion - frustration: evaluation.ScoreQuestion + urgent: decisions.PredicateQuestion + department: decisions.ChoiceQuestion + frustration: decisions.ScoreQuestion class Answers(pydantic.BaseModel): - urgent: evaluation.NoulAnswer - department: evaluation.ChoiceAnswer - frustration: evaluation.ScoreAnswer + urgent: decisions.PredicateAnswer + department: decisions.ChoiceAnswer + frustration: decisions.ScoreAnswer -async def test_evaluate_sends_native_request_and_parses_answers() -> None: +async def test_decide_sends_native_request_and_parses_answers() -> None: requests: list[httpx2.Request] = [] def handler(request: httpx2.Request) -> httpx2.Response: @@ -72,18 +72,18 @@ def handler(request: httpx2.Request) -> httpx2.Response: ), ) - result = await evaluation.evaluate( + result = await decisions.decide( _model(handler), "Help! My payouts have been failing for 3 days.", Questions( - urgent=evaluation.NoulQuestion( + urgent=decisions.PredicateQuestion( instructions="Does this convey urgency?" ), - department=evaluation.ChoiceQuestion( + department=decisions.ChoiceQuestion( instructions="Which team should handle this?", criteria={"billing": "Payments", "technical": None}, ), - frustration=evaluation.ScoreQuestion( + frustration=decisions.ScoreQuestion( instructions="How frustrated is the customer?", criteria=["Calm", "Frustrated"], ), @@ -115,7 +115,7 @@ def handler(request: httpx2.Request) -> httpx2.Response: }, } - assert result.value.urgent.noul == 0.95 + assert result.value.urgent.probability == 0.95 assert result.value.department.choice == "billing" assert result.value.department.confidence == 0.81 assert result.value.frustration.score == 1.05 @@ -131,7 +131,7 @@ def handler(request: httpx2.Request) -> httpx2.Response: } -async def test_evaluate_sends_noul_criteria_and_provider_options() -> None: +async def test_decide_sends_predicate_criteria_and_provider_options() -> None: bodies: list[dict[str, Any]] = [] def handler(request: httpx2.Request) -> httpx2.Response: @@ -140,16 +140,16 @@ def handler(request: httpx2.Request) -> httpx2.Response: 200, json=_response({"urgent": {"type": "noul", "noul": 0.1}}) ) - result = await evaluation.evaluate( + result = await decisions.decide( _model(handler), {"message": "fine, thanks"}, { - "urgent": evaluation.NoulQuestion( + "urgent": decisions.PredicateQuestion( instructions="Urgent?", criteria={"true": "Time-sensitive", "false": "Not urgent"}, ) }, - params=evaluation.EvaluationParams( + params=decisions.DecideParams( provider_options={"typesafe": {"effort": "high"}} ), ) @@ -157,23 +157,26 @@ def handler(request: httpx2.Request) -> httpx2.Response: [body] = bodies assert body["state"] == {"message": "fine, thanks"} assert body["effort"] == "high" + assert body["questions"]["urgent"]["type"] == "noul" assert body["questions"]["urgent"]["criteria"] == { "true": "Time-sensitive", "false": "Not urgent", } - assert result.value == {"urgent": evaluation.NoulAnswer(noul=0.1)} + assert result.value == { + "urgent": decisions.PredicateAnswer(probability=0.1) + } assert result.provider_metadata == {"typesafe": {"model": "jev-1.13.0"}} -async def test_evaluate_rejects_non_mapping_provider_options() -> None: +async def test_decide_rejects_non_mapping_provider_options() -> None: model = _model(lambda request: httpx2.Response(500)) with pytest.raises(TypeError, match="must be a mapping"): - await evaluation.evaluate( + await decisions.decide( model, "state", - {"q": evaluation.NoulQuestion(instructions="?")}, - params=evaluation.EvaluationParams( + {"q": decisions.PredicateQuestion(instructions="?")}, + params=decisions.DecideParams( provider_options={"typesafe": "high"} ), ) @@ -189,7 +192,7 @@ async def test_evaluate_rejects_non_mapping_provider_options() -> None: (500, ai.ProviderInternalServerError), ], ) -async def test_evaluate_maps_status_errors( +async def test_decide_maps_status_errors( status: int, error_type: type[ai.ProviderAPIError] ) -> None: def handler(request: httpx2.Request) -> httpx2.Response: @@ -200,10 +203,10 @@ def handler(request: httpx2.Request) -> httpx2.Response: ) with pytest.raises(error_type) as exc_info: - await evaluation.evaluate( + await decisions.decide( _model(handler), "state", - {"q": evaluation.NoulQuestion(instructions="?")}, + {"q": decisions.PredicateQuestion(instructions="?")}, ) exc = exc_info.value @@ -215,29 +218,29 @@ def handler(request: httpx2.Request) -> httpx2.Response: assert exc.http_context.status_code == status -async def test_evaluate_maps_connection_errors() -> None: +async def test_decide_maps_connection_errors() -> None: def handler(request: httpx2.Request) -> httpx2.Response: raise httpx2.ConnectError("boom", request=request) with pytest.raises(ai.ProviderConnectionError) as exc_info: - await evaluation.evaluate( + await decisions.decide( _model(handler), "state", - {"q": evaluation.NoulQuestion(instructions="?")}, + {"q": decisions.PredicateQuestion(instructions="?")}, ) assert exc_info.value.is_retryable -async def test_evaluate_maps_invalid_responses() -> None: +async def test_decide_maps_invalid_responses() -> None: def handler(request: httpx2.Request) -> httpx2.Response: return httpx2.Response(200, json={"model": "jev-1.13.0"}) with pytest.raises(ai.errors.ProviderResponseError) as exc_info: - await evaluation.evaluate( + await decisions.decide( _model(handler), "state", - {"q": evaluation.NoulQuestion(instructions="?")}, + {"q": decisions.PredicateQuestion(instructions="?")}, ) assert not exc_info.value.is_retryable