diff --git a/langfuse/api/commons/types/base_score.py b/langfuse/api/commons/types/base_score.py index 44e09033c..f46240dae 100644 --- a/langfuse/api/commons/types/base_score.py +++ b/langfuse/api/commons/types/base_score.py @@ -61,7 +61,9 @@ class BaseScore(UniversalBaseModel): Comment on the score """ - metadata: typing.Any = pydantic.Field() + metadata: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field( + default=None + ) """ Metadata associated with the score """ diff --git a/langfuse/api/commons/types/base_score_v1.py b/langfuse/api/commons/types/base_score_v1.py index 881b10a3b..13845d997 100644 --- a/langfuse/api/commons/types/base_score_v1.py +++ b/langfuse/api/commons/types/base_score_v1.py @@ -41,7 +41,9 @@ class BaseScoreV1(UniversalBaseModel): Comment on the score """ - metadata: typing.Any = pydantic.Field() + metadata: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field( + default=None + ) """ Metadata associated with the score """ diff --git a/langfuse/api/commons/types/observation.py b/langfuse/api/commons/types/observation.py index 91bd50643..76c019b0d 100644 --- a/langfuse/api/commons/types/observation.py +++ b/langfuse/api/commons/types/observation.py @@ -77,7 +77,9 @@ class Observation(UniversalBaseModel): The version of the observation """ - metadata: typing.Any = pydantic.Field() + metadata: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field( + default=None + ) """ Additional metadata of the observation """ diff --git a/langfuse/api/commons/types/observation_v2.py b/langfuse/api/commons/types/observation_v2.py index f034d8ace..9c6d9e3e6 100644 --- a/langfuse/api/commons/types/observation_v2.py +++ b/langfuse/api/commons/types/observation_v2.py @@ -152,7 +152,9 @@ class ObservationV2(UniversalBaseModel): The output data of the observation """ - metadata: typing.Optional[typing.Any] = pydantic.Field(default=None) + metadata: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field( + default=None + ) """ Additional metadata of the observation """ diff --git a/langfuse/api/commons/types/score.py b/langfuse/api/commons/types/score.py index 7b31ae39a..e0e98c9ae 100644 --- a/langfuse/api/commons/types/score.py +++ b/langfuse/api/commons/types/score.py @@ -54,7 +54,7 @@ class Score_Numeric(Base): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -100,7 +100,7 @@ class Score_Categorical(Base): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -146,7 +146,7 @@ class Score_Boolean(Base): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -192,7 +192,7 @@ class Score_Correction(Base): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -237,7 +237,7 @@ class Score_Text(Base): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None diff --git a/langfuse/api/commons/types/score_v1.py b/langfuse/api/commons/types/score_v1.py index e17e61b89..32e0a6b4e 100644 --- a/langfuse/api/commons/types/score_v1.py +++ b/langfuse/api/commons/types/score_v1.py @@ -35,7 +35,7 @@ class ScoreV1_Numeric(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -73,7 +73,7 @@ class ScoreV1_Categorical(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -111,7 +111,7 @@ class ScoreV1_Boolean(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -148,7 +148,7 @@ class ScoreV1_Text(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None diff --git a/langfuse/api/commons/types/trace.py b/langfuse/api/commons/types/trace.py index 6eafa9fcd..1b489c03c 100644 --- a/langfuse/api/commons/types/trace.py +++ b/langfuse/api/commons/types/trace.py @@ -59,9 +59,11 @@ class Trace(UniversalBaseModel): The user identifier associated with the trace """ - metadata: typing.Optional[typing.Any] = pydantic.Field(default=None) + metadata: typing.Optional[typing.Dict[str, typing.Any]] = pydantic.Field( + default=None + ) """ - The metadata associated with the trace. Can be any JSON. + The metadata associated with the trace. Values can be any JSON; non-object metadata sent at ingestion is returned under the `metadata` key. """ tags: typing.List[str] = pydantic.Field() diff --git a/langfuse/api/scores/types/get_scores_response_data.py b/langfuse/api/scores/types/get_scores_response_data.py index f85cd471c..84b8abcc4 100644 --- a/langfuse/api/scores/types/get_scores_response_data.py +++ b/langfuse/api/scores/types/get_scores_response_data.py @@ -45,7 +45,7 @@ class GetScoresResponseData_Numeric(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -92,7 +92,7 @@ class GetScoresResponseData_Categorical(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -139,7 +139,7 @@ class GetScoresResponseData_Boolean(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -186,7 +186,7 @@ class GetScoresResponseData_Correction(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None @@ -232,7 +232,7 @@ class GetScoresResponseData_Text(UniversalBaseModel): typing.Optional[str], FieldMetadata(alias="authorUserId") ] = None comment: typing.Optional[str] = None - metadata: typing.Any + metadata: typing.Optional[typing.Dict[str, typing.Any]] = None config_id: typing_extensions.Annotated[ typing.Optional[str], FieldMetadata(alias="configId") ] = None diff --git a/tests/unit/test_api_metadata_parsing.py b/tests/unit/test_api_metadata_parsing.py new file mode 100644 index 000000000..6be15dfba --- /dev/null +++ b/tests/unit/test_api_metadata_parsing.py @@ -0,0 +1,166 @@ +"""Response parsing of read-side ``metadata`` on generated API models. + +The server returns read-side metadata as a JSON object, ``null``, or omits it. +These tests pin that contract so a future regeneration that makes the field +required or narrows its values is caught. +""" + +import typing + +import pydantic +import pytest + +from langfuse.api import ( + BaseScore, + BaseScoreV1, + GetScoresResponseData, + Observation, + ObservationV2, + Score, + ScoreV1, + Trace, +) +from langfuse.api.core import parse_obj_as + +TIMESTAMP = "2026-01-01T00:00:00.000Z" + +SCORE_BASE: typing.Dict[str, typing.Any] = { + "id": "score-1", + "name": "quality", + "source": "API", + "timestamp": TIMESTAMP, + "createdAt": TIMESTAMP, + "updatedAt": TIMESTAMP, + "environment": "default", +} +SCORE_V1_BASE = {**SCORE_BASE, "traceId": "trace-1"} + +SCORE_VARIANTS: typing.Dict[str, typing.Dict[str, typing.Any]] = { + "NUMERIC": {"value": 0.5}, + "CATEGORICAL": {"value": 1, "stringValue": "good"}, + "BOOLEAN": {"value": 1, "stringValue": "True"}, + "CORRECTION": {"value": 0, "stringValue": "corrected output"}, + "TEXT": {"stringValue": "free text"}, +} +SCORE_V1_DATA_TYPES = ["NUMERIC", "CATEGORICAL", "BOOLEAN", "TEXT"] + + +def _score_payload( + base: typing.Dict[str, typing.Any], data_type: str +) -> typing.Dict[str, typing.Any]: + return {**base, "dataType": data_type, **SCORE_VARIANTS[data_type]} + + +CASES: typing.List[typing.Any] = [ + pytest.param( + Trace, + { + "id": "trace-1", + "timestamp": TIMESTAMP, + "tags": [], + "public": False, + "environment": "default", + }, + id="Trace", + ), + pytest.param( + Observation, + { + "id": "obs-1", + "type": "SPAN", + "startTime": TIMESTAMP, + "modelParameters": {}, + "input": None, + "output": None, + "usage": {"input": 0, "output": 0, "total": 0}, + "level": "DEFAULT", + "usageDetails": {}, + "costDetails": {}, + "environment": "default", + }, + id="Observation", + ), + pytest.param( + ObservationV2, + { + "id": "obs-1", + "startTime": TIMESTAMP, + "projectId": "project-1", + "type": "SPAN", + }, + id="ObservationV2", + ), + pytest.param(BaseScore, SCORE_BASE, id="BaseScore"), + pytest.param(BaseScoreV1, SCORE_V1_BASE, id="BaseScoreV1"), + *[ + pytest.param(Score, _score_payload(SCORE_BASE, dt), id=f"Score-{dt}") + for dt in SCORE_VARIANTS + ], + *[ + pytest.param( + GetScoresResponseData, + _score_payload(SCORE_BASE, dt), + id=f"GetScoresResponseData-{dt}", + ) + for dt in SCORE_VARIANTS + ], + *[ + pytest.param(ScoreV1, _score_payload(SCORE_V1_BASE, dt), id=f"ScoreV1-{dt}") + for dt in SCORE_V1_DATA_TYPES + ], +] + +NESTED_METADATA = { + "str": "value", + "int": 1, + "float": 1.5, + "bool": True, + "null": None, + "list": [1, "two", {"three": 3}], + "nested": {"deeper": {"key": ["a", "b"]}}, +} + + +@pytest.mark.parametrize(("type_", "payload"), CASES) +def test_metadata_omitted_parses_as_none(type_, payload): + assert "metadata" not in payload + + parsed = parse_obj_as(type_, payload) + + assert parsed.metadata is None + + +@pytest.mark.parametrize(("type_", "payload"), CASES) +def test_metadata_null_parses_as_none(type_, payload): + parsed = parse_obj_as(type_, {**payload, "metadata": None}) + + assert parsed.metadata is None + + +@pytest.mark.parametrize(("type_", "payload"), CASES) +def test_metadata_empty_object_parses(type_, payload): + parsed = parse_obj_as(type_, {**payload, "metadata": {}}) + + assert parsed.metadata == {} + + +@pytest.mark.parametrize(("type_", "payload"), CASES) +def test_metadata_object_with_nested_json_values_parses(type_, payload): + parsed = parse_obj_as(type_, {**payload, "metadata": NESTED_METADATA}) + + assert parsed.metadata == NESTED_METADATA + + +@pytest.mark.parametrize(("type_", "payload"), CASES) +@pytest.mark.parametrize( + "metadata", + ["a string", 1, 1.5, True, ["a", "list"]], + ids=["str", "int", "float", "bool", "list"], +) +def test_metadata_non_object_is_rejected(type_, payload, metadata): + with pytest.raises(pydantic.ValidationError) as exc_info: + parse_obj_as(type_, {**payload, "metadata": metadata}) + + # The discriminated unions prefix the location with the variant tag. + assert exc_info.value.errors() + assert all("metadata" in error["loc"] for error in exc_info.value.errors())