diff --git a/.github/langfuse-v4-dual-write.override.yml b/.github/langfuse-v4-dual-write.override.yml deleted file mode 100644 index 48ab6ff48..000000000 --- a/.github/langfuse-v4-dual-write.override.yml +++ /dev/null @@ -1,7 +0,0 @@ -services: - langfuse-worker: - environment: - LANGFUSE_MIGRATION_V4_WRITE_MODE: dual - langfuse-web: - environment: - LANGFUSE_MIGRATION_V4_WRITE_MODE: dual diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 104cda66f..ee355e49b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,8 +159,6 @@ jobs: LANGFUSE_SERVER_SHA="$(git ls-remote https://github.com/langfuse/langfuse.git HEAD | cut -f1)" curl -fsSL "https://raw.githubusercontent.com/langfuse/langfuse/${LANGFUSE_SERVER_SHA}/docker-compose.yml" \ -o ./langfuse-server/docker-compose.yml - cp ./.github/langfuse-v4-dual-write.override.yml \ - ./langfuse-server/docker-compose.override.yml echo "${LANGFUSE_SERVER_SHA}" - name: Run langfuse server diff --git a/langfuse/_client/attributes.py b/langfuse/_client/attributes.py index 43a85c2fd..e7027a192 100644 --- a/langfuse/_client/attributes.py +++ b/langfuse/_client/attributes.py @@ -32,8 +32,6 @@ class LangfuseOtelSpanAttributes: TRACE_TAGS = "langfuse.trace.tags" TRACE_PUBLIC = "langfuse.trace.public" TRACE_METADATA = "langfuse.trace.metadata" - TRACE_INPUT = "langfuse.trace.input" - TRACE_OUTPUT = "langfuse.trace.output" # Langfuse-observation attributes OBSERVATION_TYPE = "langfuse.observation.type" @@ -76,13 +74,9 @@ class LangfuseOtelSpanAttributes: def create_trace_attributes( *, - input: Optional[Any] = None, - output: Optional[Any] = None, public: Optional[bool] = None, ) -> dict: attributes = { - LangfuseOtelSpanAttributes.TRACE_INPUT: _serialize(input), - LangfuseOtelSpanAttributes.TRACE_OUTPUT: _serialize(output), LangfuseOtelSpanAttributes.TRACE_PUBLIC: public, } diff --git a/langfuse/_client/client.py b/langfuse/_client/client.py index 02d122862..d6191248a 100644 --- a/langfuse/_client/client.py +++ b/langfuse/_client/client.py @@ -38,7 +38,6 @@ _agnosticcontextmanager, ) from packaging.version import Version -from typing_extensions import deprecated from langfuse._client.attributes import ( LangfuseOtelSpanAttributes, @@ -213,7 +212,7 @@ class Langfuse: release (Optional[str]): Release version/hash of your application. Used for grouping analytics by release. media_upload_thread_count (Optional[int]): Number of background threads for handling media uploads. Defaults to 1. Can also be set via LANGFUSE_MEDIA_UPLOAD_THREAD_COUNT environment variable. sample_rate (Optional[float]): Sampling rate for traces (0.0 to 1.0). Defaults to 1.0 (100% of traces are sampled). Can also be set via LANGFUSE_SAMPLE_RATE environment variable. - mask (Optional[MaskFunction]): Function to mask sensitive data synchronously when Langfuse SDK attributes are created. This applies only to data set through Langfuse SDK APIs such as `start_observation()`, `update()`, and `set_trace_io()`. + mask (Optional[MaskFunction]): Function to mask sensitive data synchronously when Langfuse SDK attributes are created. This applies only to data set through Langfuse SDK APIs such as `start_observation()` and `update()`. mask_otel_spans (Optional[MaskOtelSpansFunction]): Synchronous export-stage hook for masking raw OpenTelemetry span attributes before this Langfuse client sends them to Langfuse. Use this for spans created by third-party OpenTelemetry instrumentations, or when you need to inspect final span attributes after export filtering and Langfuse media handling. It does not modify spans already exported through other OpenTelemetry exporters. The hook receives one OpenTelemetry export batch. A batch is not guaranteed to contain a complete trace, request, or Langfuse observation tree. The hook usually runs on the OpenTelemetry batch span processor worker thread; during `flush()` and shutdown it may run on the caller thread. Keep it synchronous, deterministic, and fast. @@ -1534,55 +1533,6 @@ def update_current_span( status_message=status_message, ) - @deprecated( - "Trace-level input/output is deprecated. " - "For trace attributes (user_id, session_id, tags, etc.), use propagate_attributes() instead. " - "This method will be removed in a future major version." - ) - def set_current_trace_io( - self, - *, - input: Optional[Any] = None, - output: Optional[Any] = None, - ) -> None: - """Set trace-level input and output for the current span's trace. - - .. deprecated:: - This is a legacy method for backward compatibility with Langfuse platform - features that still rely on trace-level input/output (e.g., legacy LLM-as-a-judge - evaluators). It will be removed in a future major version. - - For setting other trace attributes (user_id, session_id, metadata, tags, version), - use :func:`langfuse.propagate_attributes` (top-level import) instead. - - Args: - input: Input data to associate with the trace. - output: Output data to associate with the trace. - """ - if not self._tracing_enabled: - langfuse_logger.debug( - "Operation skipped: set_current_trace_io - Tracing is disabled or client is in no-op mode." - ) - return - - current_otel_span = self._get_current_otel_span() - - if current_otel_span is not None and current_otel_span.is_recording(): - span_class = self._get_span_class( - self._get_observation_type_from_otel_span(current_otel_span) - ) - span = span_class( - otel_span=current_otel_span, - langfuse_client=self, - environment=self._environment, - release=self._release, - ) - - span.set_trace_io( - input=input, - output=output, - ) - def set_current_trace_as_public(self) -> None: """Make the current trace publicly accessible via its URL. diff --git a/langfuse/_client/span.py b/langfuse/_client/span.py index 96879499c..9e2b5b718 100644 --- a/langfuse/_client/span.py +++ b/langfuse/_client/span.py @@ -36,7 +36,6 @@ if TYPE_CHECKING: from langfuse._client.client import Langfuse -from typing_extensions import deprecated from langfuse._client.attributes import ( LangfuseOtelSpanAttributes, @@ -226,53 +225,6 @@ def end(self, *, end_time: Optional[int] = None) -> "LangfuseObservationWrapper" return self - @deprecated( - "Trace-level input/output is deprecated. " - "For trace attributes (user_id, session_id, tags, etc.), use propagate_attributes() instead. " - "This method will be removed in a future major version." - ) - def set_trace_io( - self, - *, - input: Optional[Any] = None, - output: Optional[Any] = None, - ) -> "LangfuseObservationWrapper": - """Set trace-level input and output for the trace this span belongs to. - - .. deprecated:: - This is a legacy method for backward compatibility with Langfuse platform - features that still rely on trace-level input/output (e.g., legacy LLM-as-a-judge - evaluators). It will be removed in a future major version. - - For setting other trace attributes (user_id, session_id, metadata, tags, version), - use :func:`langfuse.propagate_attributes` (top-level import) instead. - - Args: - input: Input data to associate with the trace. - output: Output data to associate with the trace. - - Returns: - The span instance for method chaining. - """ - if not self._otel_span.is_recording(): - return self - - media_processed_input = self._process_media_and_apply_mask( - data=input, field="input", span=self._otel_span - ) - media_processed_output = self._process_media_and_apply_mask( - data=output, field="output", span=self._otel_span - ) - - attributes = create_trace_attributes( - input=media_processed_input, - output=media_processed_output, - ) - - self._otel_span.set_attributes(attributes) - - return self - def set_trace_as_public(self) -> "LangfuseObservationWrapper": """Make this trace publicly accessible via its URL. diff --git a/langfuse/_client/span_exporter.py b/langfuse/_client/span_exporter.py index 661a1a26c..efbb04261 100644 --- a/langfuse/_client/span_exporter.py +++ b/langfuse/_client/span_exporter.py @@ -27,7 +27,6 @@ _INPUT_MEDIA_ATTRIBUTE_KEYS = frozenset( { - LangfuseOtelSpanAttributes.TRACE_INPUT, LangfuseOtelSpanAttributes.OBSERVATION_INPUT, "ai.prompt.messages", "ai.prompt", @@ -54,7 +53,6 @@ _OUTPUT_MEDIA_ATTRIBUTE_KEYS = frozenset( { - LangfuseOtelSpanAttributes.TRACE_OUTPUT, LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT, "ai.response.text", "ai.result.text", diff --git a/tests/e2e/test_core_sdk.py b/tests/e2e/test_core_sdk.py index 614d6da41..5330932c3 100644 --- a/tests/e2e/test_core_sdk.py +++ b/tests/e2e/test_core_sdk.py @@ -1,3 +1,4 @@ +import json import os import time from asyncio import gather @@ -5,17 +6,20 @@ from time import sleep import pytest -from tenacity import Retrying, stop_after_delay, wait_fixed from langfuse import Langfuse, propagate_attributes from langfuse._client.resource_manager import LangfuseResourceManager from langfuse._utils import _get_timestamp -from tests.support.api_wrapper import LangfuseAPI from tests.support.utils import ( create_uuid, get_api, - wait_for_result, - wait_for_trace, + get_observations, + get_root_observation, + get_scores, + user_metadata, + wait_for_observations, + wait_for_root_observation, + wait_for_scores, ) @@ -38,31 +42,28 @@ async def update_generation(i, langfuse: Langfuse): # End the generation generation.end() + return generation.trace_id + # Create Langfuse client langfuse = Langfuse() # Run concurrent operations - await gather(*(update_generation(i, langfuse) for i in range(100))) + trace_ids = await gather(*(update_generation(i, langfuse) for i in range(100))) langfuse.flush() - # Allow time for all operations to be processed - sleep(10) - # Verify that all spans were created properly - api = get_api() - for i in range(100): - # Find the observations with the expected name - observations = api.legacy.observations_v1.get_many(name=str(i)).data + for i, trace_id in enumerate(trace_ids): + observations = wait_for_observations(trace_id, min_count=2) - # Find generation observations (there should be at least one) generation_obs = [obs for obs in observations if obs.type == "GENERATION"] - assert len(generation_obs) > 0 + assert len(generation_obs) == 1 # Verify metadata observation = generation_obs[0] assert observation.name == str(i) - assert observation.metadata["count"] == i + assert user_metadata(observation)["count"] == i + assert get_root_observation(observations).trace_name == str(i) def test_flush(): @@ -80,14 +81,10 @@ def test_flush(): # Flush all pending spans to the Langfuse API langfuse.flush() - # Allow time for API to process - sleep(2) - # Verify traces were sent by checking they exist in the API - api = get_api() for i, trace_id in enumerate(trace_ids): - trace = api.trace.get(trace_id) - assert trace.name == str(i) + root = wait_for_root_observation(trace_id) + assert root.trace_name == str(i) def test_invalid_score_data_does_not_raise_exception(): @@ -152,21 +149,20 @@ def test_create_session_score(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - score = get_api().scores.get_by_id(score_id) + scores = wait_for_scores(session_id=session_id, id=score_id) - # find the score by name (server may transform the id format) - assert score is not None + assert len(scores) == 1 + score = scores[0] assert score.value == 1 assert score.data_type == "NUMERIC" - assert score.session_id == session_id + assert score.subject.kind == "session" + assert score.subject.id == session_id def test_create_numeric_score(): langfuse = Langfuse() - api_wrapper = LangfuseAPI() # Create a span and set trace properties with langfuse.start_as_current_observation(name="test-span") as span: @@ -202,22 +198,21 @@ def test_create_numeric_score(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = api_wrapper.get_trace(trace_id) + wait_for_observations(trace_id, min_count=2) + scores = wait_for_scores(trace_id=trace_id, name="this-is-a-score") - # Find the score by name (server may transform the ID format) - score = next((s for s in trace["scores"] if s["name"] == "this-is-a-score"), None) - assert score is not None - assert score["value"] == 1 - assert score["dataType"] == "NUMERIC" - assert score["stringValue"] is None + assert len(scores) == 1 + score = scores[0] + assert score.id == score_id + assert score.value == 1 + assert score.data_type == "NUMERIC" + assert score.subject.kind == "trace" def test_create_boolean_score(): langfuse = Langfuse() - api_wrapper = LangfuseAPI() # Create a span and set trace properties with langfuse.start_as_current_observation(name="test-span") as span: @@ -231,7 +226,7 @@ def test_create_boolean_score(): # Ensure data is sent langfuse.flush() - api_wrapper.get_trace(trace_id) + wait_for_root_observation(trace_id) # Create a boolean score score_id = create_uuid() @@ -256,27 +251,17 @@ def test_create_boolean_score(): langfuse.flush() # Retrieve and verify - trace = api_wrapper.get_trace( - trace_id, - is_result_ready=lambda trace: any( - score["name"] == "this-is-a-score" for score in trace.get("scores", []) - ), - ) + scores = wait_for_scores(trace_id=trace_id, name="this-is-a-score") - # Find the score we created by name - created_score = next( - (s for s in trace["scores"] if s["name"] == "this-is-a-score"), None - ) - assert created_score is not None, "Score not found in trace" - assert created_score["id"] == score_id - assert created_score["dataType"] == "BOOLEAN" - assert created_score["value"] == 1 - assert created_score["stringValue"] == "True" + assert len(scores) == 1, "Score not found in trace" + created_score = scores[0] + assert created_score.id == score_id + assert created_score.data_type == "BOOLEAN" + assert created_score.value is True def test_create_categorical_score(): langfuse = Langfuse() - api_wrapper = LangfuseAPI() # Create a span and set trace properties with langfuse.start_as_current_observation(name="test-span") as span: @@ -290,7 +275,7 @@ def test_create_categorical_score(): # Ensure data is sent langfuse.flush() - api_wrapper.get_trace(trace_id) + wait_for_root_observation(trace_id) # Create a categorical score score_id = create_uuid() @@ -314,27 +299,17 @@ def test_create_categorical_score(): langfuse.flush() # Retrieve and verify - trace = api_wrapper.get_trace( - trace_id, - is_result_ready=lambda trace: any( - score["name"] == "this-is-a-score" for score in trace.get("scores", []) - ), - ) + scores = wait_for_scores(trace_id=trace_id, name="this-is-a-score") - # Find the score we created by name - created_score = next( - (s for s in trace["scores"] if s["name"] == "this-is-a-score"), None - ) - assert created_score is not None, "Score not found in trace" - assert created_score["id"] == score_id - assert created_score["dataType"] == "CATEGORICAL" - assert created_score["value"] == 0 - assert created_score["stringValue"] == "high score" + assert len(scores) == 1, "Score not found in trace" + created_score = scores[0] + assert created_score.id == score_id + assert created_score.data_type == "CATEGORICAL" + assert created_score.value == "high score" def test_create_text_score(): langfuse = Langfuse() - api_wrapper = LangfuseAPI() # Create a span and set trace properties with langfuse.start_as_current_observation(name="test-span") as span: @@ -372,30 +347,21 @@ def test_create_text_score(): # Ensure data is sent langfuse.flush() - # Retrieve and verify with retry - for attempt in Retrying( - stop=stop_after_delay(10), wait=wait_fixed(0.1), reraise=True - ): - with attempt: - trace = api_wrapper.get_trace(trace_id) - - # Find the score we created by name - created_score = next( - (s for s in trace["scores"] if s["name"] == "this-is-a-score"), None - ) - assert created_score is not None, "Score not found in trace" - assert created_score["id"] == score_id - assert created_score["dataType"] == "TEXT" + # Retrieve and verify + scores = wait_for_scores(trace_id=trace_id, name="this-is-a-score") - assert ( - created_score["stringValue"] - == "This is a detailed text evaluation of the output quality." - ) + assert len(scores) == 1, "Score not found in trace" + created_score = scores[0] + assert created_score.id == score_id + assert created_score.data_type == "TEXT" + assert ( + created_score.value + == "This is a detailed text evaluation of the output quality." + ) def test_create_score_with_custom_timestamp(): langfuse = Langfuse() - api_wrapper = LangfuseAPI() # Create a span and set trace properties with langfuse.start_as_current_observation(name="test-span") as span: @@ -409,7 +375,7 @@ def test_create_score_with_custom_timestamp(): # Ensure data is sent langfuse.flush() - api_wrapper.get_trace(trace_id) + wait_for_root_observation(trace_id) custom_timestamp = datetime.now(timezone.utc) - timedelta(hours=1) score_id = create_uuid() @@ -426,28 +392,16 @@ def test_create_score_with_custom_timestamp(): langfuse.flush() # Retrieve and verify - trace = api_wrapper.get_trace( - trace_id, - is_result_ready=lambda trace: any( - score["name"] == "custom-timestamp-score" - for score in trace.get("scores", []) - ), - ) + scores = wait_for_scores(trace_id=trace_id, name="custom-timestamp-score") - # Find the score we created by name - created_score = next( - (s for s in trace["scores"] if s["name"] == "custom-timestamp-score"), None - ) - assert created_score is not None, "Score not found in trace" - assert created_score["id"] == score_id - assert created_score["dataType"] == "NUMERIC" - assert created_score["value"] == 0.85 + assert len(scores) == 1, "Score not found in trace" + created_score = scores[0] + assert created_score.id == score_id + assert created_score.data_type == "NUMERIC" + assert created_score.value == 0.85 # Verify timestamp is close to our custom timestamp - # Parse the timestamp from the API response - response_timestamp = datetime.fromisoformat( - created_score["timestamp"].replace("Z", "+00:00") - ) + response_timestamp = created_score.timestamp # Check that the timestamps are within 1 second of each other # (allowing for some processing time and rounding) @@ -477,24 +431,14 @@ def test_create_trace(): langfuse.flush() # Retrieve the trace from the API - trace = LangfuseAPI().get_trace( - trace_id, - is_result_ready=lambda trace: ( - trace.get("name") == trace_name - and trace.get("userId") == "test" - and trace.get("metadata", {}).get("key") == "value" - and trace.get("tags") == ["tag1", "tag2"] - and trace.get("public") is True - ), - ) + root = wait_for_root_observation(trace_id) # Verify all trace properties - assert trace["name"] == trace_name - assert trace["userId"] == "test" - assert trace["metadata"]["key"] == "value" - assert trace["tags"] == ["tag1", "tag2"] - assert trace["public"] is True - assert True if not trace["externalId"] else False + assert root.trace_name == trace_name + assert root.user_id == "test" + assert user_metadata(root) == {"key": "value"} + assert root.tags == ["tag1", "tag2"] + assert root.public is True def test_create_update_trace(): @@ -525,23 +469,12 @@ def test_create_update_trace(): assert isinstance(trace_id, str) # Retrieve and verify trace - trace = wait_for_trace( - trace_id, - is_result_ready=lambda trace: ( - trace.name == trace_name - and trace.user_id == "test" - and trace.metadata is not None - and trace.metadata.get("key") == "value" - and trace.metadata.get("key2") == "value2" - and trace.public is True - ), - ) + root = wait_for_root_observation(trace_id) - assert trace.name == trace_name - assert trace.user_id == "test" - assert trace.metadata["key"] == "value" - assert trace.metadata["key2"] == "value2" - assert trace.public is True + assert root.trace_name == trace_name + assert root.user_id == "test" + assert user_metadata(root) == {"key": "value", "key2": "value2"} + assert root.public is True def test_create_update_current_trace(): @@ -549,14 +482,14 @@ def test_create_update_current_trace(): trace_name = create_uuid() - # Create initial span with trace properties using propagate_attributes and set_current_trace_io + # Create initial span with trace properties using propagate_attributes with langfuse.start_as_current_observation(name="test-span-current") as span: with propagate_attributes( trace_name=trace_name, user_id="test", metadata={"key": "value"}, ): - langfuse.set_current_trace_io(input="test_input") + langfuse.update_current_span(input="test_input") langfuse.set_current_trace_as_public() # Get trace ID for later reference trace_id = span.trace_id @@ -570,20 +503,18 @@ def test_create_update_current_trace(): # Ensure data is sent to the API langfuse.flush() - sleep(2) assert isinstance(trace_id, str) # Retrieve and verify trace - trace = get_api().trace.get(trace_id) + root = wait_for_root_observation(trace_id) # The 2nd update to the trace must not erase previously set attributes - assert trace.name == trace_name - assert trace.user_id == "test" - assert trace.metadata["key"] == "value" - assert trace.metadata["key2"] == "value2" - assert trace.public is True - assert trace.version == "1.0" - assert trace.input == "test_input" + assert root.trace_name == trace_name + assert root.user_id == "test" + assert user_metadata(root) == {"key": "value", "key2": "value2"} + assert root.public is True + assert root.version == "1.0" + assert root.input == "test_input" def test_create_generation(): @@ -620,19 +551,19 @@ def test_create_generation(): # Flush to ensure all data is sent langfuse.flush() - sleep(2) # Retrieve the trace from the API - trace = get_api().trace.get(trace_id) - - # Verify trace details - assert trace.name == "query-generation" - assert trace.user_id is None + observations = wait_for_observations(trace_id) - assert len(trace.observations) == 1 + assert len(observations) == 1 # Verify generation details - generation_api = trace.observations[0] + generation_api = observations[0] + + # Verify trace details + assert generation_api.is_root_observation is True + assert generation_api.trace_name == "query-generation" + assert generation_api.user_id is None assert generation_api.name == "query-generation" assert generation_api.start_time is not None @@ -707,15 +638,14 @@ def test_create_generation_complex( langfuse.flush() trace_id = generation.trace_id - trace = get_api().trace.get(trace_id) + observations = wait_for_observations(trace_id) - assert trace.name == "query-generation" - assert trace.user_id is None + assert len(observations) == 1 - assert len(trace.observations) == 1 - - generation_api = trace.observations[0] + generation_api = observations[0] + assert generation_api.trace_name == "query-generation" + assert generation_api.user_id is None assert generation_api.id == generation.id assert generation_api.name == "query-generation" assert generation_api.input == [ @@ -727,13 +657,7 @@ def test_create_generation_complex( ] assert generation_api.output == [{"foo": "bar"}] - # Check if metadata exists and has tags before asserting - if ( - hasattr(generation_api, "metadata") - and generation_api.metadata is not None - and "tags" in generation_api.metadata - ): - assert generation_api.metadata["tags"] == ["yo"] + assert user_metadata(generation_api) == {"tags": ["yo"]} assert generation_api.start_time is not None assert generation_api.usage_details == {"input": 51, "output": 0, "total": 100} @@ -759,19 +683,18 @@ def test_create_span(): # Ensure all data is sent langfuse.flush() - sleep(2) # Retrieve from API - trace = get_api().trace.get(trace_id) - - # Verify trace details - assert trace.name == "span" - assert trace.user_id is None + observations = wait_for_observations(trace_id) - assert len(trace.observations) == 1 + assert len(observations) == 1 # Verify span details - span_api = trace.observations[0] + span_api = observations[0] + + # Verify trace details + assert span_api.trace_name == "span" + assert span_api.user_id is None assert span_api.id == span_id assert span_api.name == "span" @@ -784,7 +707,6 @@ def test_create_span(): def test_score_trace(): langfuse = Langfuse() - api_wrapper = LangfuseAPI() trace_name = create_uuid() @@ -803,20 +725,17 @@ def test_score_trace(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = api_wrapper.get_trace(trace_id) - - assert trace["name"] == trace_name + assert wait_for_root_observation(trace_id).trace_name == trace_name - # Find the score we created by name (server may create additional auto-scores) - score = next((s for s in trace["scores"] if s["name"] == "valuation"), None) - assert score is not None - assert score["value"] == 0.5 - assert score["comment"] == "This is a comment" - assert score["observationId"] is None - assert score["dataType"] == "NUMERIC" + scores = wait_for_scores(trace_id=trace_id, name="valuation") + assert len(scores) == 1 + score = scores[0] + assert score.value == 0.5 + assert score.comment == "This is a comment" + assert score.subject.kind == "trace" + assert score.data_type == "NUMERIC" def test_score_trace_nested_trace(): @@ -839,19 +758,16 @@ def test_score_trace_nested_trace(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = get_api().trace.get(trace_id) + assert wait_for_root_observation(trace_id).trace_name == trace_name - assert trace.name == trace_name - - # Find the score we created by name (server may create additional auto-scores) - score = next((s for s in trace.scores if s.name == "valuation"), None) - assert score is not None + scores = wait_for_scores(trace_id=trace_id, name="valuation") + assert len(scores) == 1 + score = scores[0] assert score.value == 0.5 assert score.comment == "This is a comment" - assert score.observation_id is None # API returns this field name + assert score.subject.kind == "trace" assert score.data_type == "NUMERIC" @@ -882,25 +798,23 @@ def test_score_trace_nested_observation(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = get_api().trace.get(trace_id) - - assert trace.name == trace_name + assert wait_for_root_observation(trace_id).trace_name == trace_name - # Find the score we created by name (server may create additional auto-scores) - score = next((s for s in trace.scores if s.name == "valuation"), None) - assert score is not None + scores = wait_for_scores(trace_id=trace_id, name="valuation") + assert len(scores) == 1 + score = scores[0] assert score.value == 0.5 assert score.comment == "This is a comment" - assert score.observation_id == child_span_id # API returns this field name + assert score.subject.kind == "observation" + assert score.subject.id == child_span_id + assert score.subject.trace_id == trace_id assert score.data_type == "NUMERIC" def test_score_span(): langfuse = Langfuse() - api_wrapper = LangfuseAPI() # Create a span span = langfuse.start_observation( @@ -928,20 +842,19 @@ def test_score_span(): # Ensure data is sent langfuse.flush() - sleep(3) # Retrieve and verify - trace = api_wrapper.get_trace(trace_id) + assert len(wait_for_observations(trace_id)) == 1 - assert len(trace["observations"]) == 1 - - # Find the score we created by name (server may create additional auto-scores) - score = next((s for s in trace["scores"] if s["name"] == "valuation"), None) - assert score is not None - assert score["value"] == 1 - assert score["comment"] == "This is a comment" - assert score["observationId"] == span_id - assert score["dataType"] == "NUMERIC" + scores = wait_for_scores(trace_id=trace_id, observation_id=span_id) + assert len(scores) == 1 + score = scores[0] + assert score.name == "valuation" + assert score.value == 1 + assert score.comment == "This is a comment" + assert score.subject.kind == "observation" + assert score.subject.id == span_id + assert score.data_type == "NUMERIC" def test_create_trace_and_span(): @@ -963,16 +876,15 @@ def test_create_trace_and_span(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = get_api().trace.get(trace_id) + observations = wait_for_observations(trace_id, min_count=2) - assert trace.name == trace_name - assert len(trace.observations) == 2 # Parent span and child span + assert get_root_observation(observations).trace_name == trace_name + assert len(observations) == 2 # Parent span and child span # Find the child span - child_spans = [obs for obs in trace.observations if obs.name == "span"] + child_spans = [obs for obs in observations if obs.name == "span"] assert len(child_spans) == 1 span = child_spans[0] @@ -989,7 +901,7 @@ def test_create_trace_and_generation(): # Create parent span and set trace properties with langfuse.start_as_current_observation(name=trace_name) as parent_span: with propagate_attributes(trace_name=trace_name, session_id="test-session-id"): - parent_span.set_trace_io(input={"key": "value"}) + parent_span.update(input={"key": "value"}) # Create a generation as child generation = parent_span.start_observation( @@ -1004,35 +916,30 @@ def test_create_trace_and_generation(): # Ensure data is sent langfuse.flush() - sleep(2) - # Retrieve traces in two ways - dbTrace = get_api().trace.get(trace_id) - getTrace = get_api().trace.get( - trace_id - ) # Using API as direct getTrace not available + # Retrieve and verify + observations = wait_for_observations(trace_id, min_count=2) + root = get_root_observation(observations) # Verify trace details - assert dbTrace.name == trace_name - assert len(dbTrace.observations) == 2 # Parent span and generation - assert getTrace.name == trace_name - assert len(getTrace.observations) == 2 - assert getTrace.session_id == "test-session-id" + assert root.trace_name == trace_name + assert len(observations) == 2 # Parent span and generation + assert root.session_id == "test-session-id" # Find the generation - generations = [obs for obs in getTrace.observations if obs.name == "generation"] + generations = [obs for obs in observations if obs.name == "generation"] assert len(generations) == 1 generation = generations[0] assert generation.name == "generation" assert generation.trace_id == trace_id + assert generation.session_id == "test-session-id" assert generation.start_time is not None - assert getTrace.input == {"key": "value"} + assert root.input == {"key": "value"} def test_create_generation_and_trace(): langfuse = Langfuse() - api_wrapper = LangfuseAPI() trace_name = create_uuid() @@ -1063,23 +970,24 @@ def test_create_generation_and_trace(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = api_wrapper.get_trace(trace_id) - - assert trace["name"] == trace_name + observations = wait_for_observations(trace_id, min_count=2) # We should have 2 observations (the generation and the span for updating trace) - assert len(trace["observations"]) == 2 + assert len(observations) == 2 + + trace_update_spans = [obs for obs in observations if obs.name == "trace-update"] + assert len(trace_update_spans) == 1 + assert trace_update_spans[0].trace_name == trace_name # Find the generation - generations = [obs for obs in trace["observations"] if obs["name"] == "generation"] + generations = [obs for obs in observations if obs.name == "generation"] assert len(generations) == 1 generation_obs = generations[0] - assert generation_obs["name"] == "generation" - assert generation_obs["traceId"] == trace["id"] + assert generation_obs.name == "generation" + assert generation_obs.trace_id == trace_id def test_create_span_and_get_observation(): @@ -1096,12 +1004,12 @@ def test_create_span_and_get_observation(): # Flush and wait langfuse.flush() - sleep(2) - # Use API to fetch the observation by ID - observation = get_api().legacy.observations_v1.get(span_id) + observations = wait_for_observations(span.trace_id) # Verify observation properties + assert len(observations) == 1 + observation = observations[0] assert observation.name == "span" assert observation.id == span_id @@ -1123,20 +1031,19 @@ def test_update_generation(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = get_api().trace.get(trace_id) + observations = wait_for_observations(trace_id) # Verify trace properties - assert trace.name == "generation" - assert len(trace.observations) == 1 + assert len(observations) == 1 # Verify generation updates - retrieved_generation = trace.observations[0] + retrieved_generation = observations[0] + assert retrieved_generation.trace_name == "generation" assert retrieved_generation.name == "generation" assert retrieved_generation.trace_id == trace_id - assert retrieved_generation.metadata["dict"] == "value" + assert user_metadata(retrieved_generation) == {"dict": "value"} # Note: With OTEL, we can't verify exact start times from manually set timestamps, # as they are managed internally by the OTEL SDK @@ -1159,20 +1066,19 @@ def test_update_span(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = get_api().trace.get(trace_id) + observations = wait_for_observations(trace_id) # Verify trace properties - assert trace.name == "span" - assert len(trace.observations) == 1 + assert len(observations) == 1 # Verify span updates - retrieved_span = trace.observations[0] + retrieved_span = observations[0] + assert retrieved_span.trace_name == "span" assert retrieved_span.name == "span" assert retrieved_span.trace_id == trace_id - assert retrieved_span.metadata["dict"] == "value" + assert user_metadata(retrieved_span) == {"dict": "value"} def test_create_span_and_generation(): @@ -1197,17 +1103,16 @@ def test_create_span_and_generation(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = get_api().trace.get(trace_id) + observations = wait_for_observations(trace_id, min_count=2) # Verify trace details - assert len(trace.observations) == 2 + assert len(observations) == 2 # Find span and generation - spans = [obs for obs in trace.observations if obs.name == "span"] - generations = [obs for obs in trace.observations if obs.name == "generation"] + spans = [obs for obs in observations if obs.name == "span"] + generations = [obs for obs in observations if obs.name == "generation"] assert len(spans) == 1 assert len(generations) == 1 @@ -1222,7 +1127,6 @@ def test_create_span_and_generation(): def test_create_trace_with_id_and_generation(): langfuse = Langfuse() - api_wrapper = LangfuseAPI() trace_name = create_uuid() @@ -1244,28 +1148,27 @@ def test_create_trace_with_id_and_generation(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = api_wrapper.get_trace(trace_id) + observations = wait_for_observations(trace_id, min_count=2) # Verify trace properties - assert trace["name"] == trace_name - assert trace["id"] == trace_id - assert len(trace["observations"]) == 2 # Parent span and generation + root = get_root_observation(observations) + assert root.trace_name == trace_name + assert root.trace_id == trace_id + assert len(observations) == 2 # Parent span and generation # Find the generation - generations = [obs for obs in trace["observations"] if obs["name"] == "generation"] + generations = [obs for obs in observations if obs.name == "generation"] assert len(generations) == 1 gen = generations[0] - assert gen["name"] == "generation" - assert gen["traceId"] == trace["id"] + assert gen.name == "generation" + assert gen.trace_id == trace_id def test_end_generation(): langfuse = Langfuse() - api_wrapper = LangfuseAPI() # Create a generation generation = langfuse.start_observation( @@ -1294,21 +1197,14 @@ def test_end_generation(): langfuse.flush() # Retrieve and verify - trace = api_wrapper.get_trace( - trace_id, - is_result_ready=lambda trace: any( - obs["name"] == "query-generation" for obs in trace.get("observations", []) - ), - ) + observations = wait_for_observations(trace_id) # Find generation by name - generations = [ - obs for obs in trace["observations"] if obs["name"] == "query-generation" - ] + generations = [obs for obs in observations if obs.name == "query-generation"] assert len(generations) == 1 gen = generations[0] - assert gen["endTime"] is not None + assert gen.end_time is not None def test_end_generation_with_data(): @@ -1353,15 +1249,12 @@ def test_end_generation_with_data(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - fetched_trace = get_api().trace.get(trace_id) + observations = wait_for_observations(trace_id, min_count=2) # Find generation by name - generations = [ - obs for obs in fetched_trace.observations if obs.name == "query-generation" - ] + generations = [obs for obs in observations if obs.name == "query-generation"] assert len(generations) == 1 generation = generations[0] @@ -1371,7 +1264,7 @@ def test_end_generation_with_data(): 2023, 1, 1, 12, 3, tzinfo=timezone.utc ) assert generation.name == "query-generation" - assert generation.metadata["dict"] == "value" + assert user_metadata(generation) == {"dict": "value"} assert generation.level == "ERROR" assert generation.status_message == "Generation ended" assert generation.version == "1.0" @@ -1379,12 +1272,9 @@ def test_end_generation_with_data(): assert generation.model_parameters == {"param1": "value1", "param2": "value2"} assert generation.input == [{"test_input_key": "test_input_value"}] assert generation.output == {"test_output_key": "test_output_value"} - assert generation.usage.input == 100 - assert generation.usage.output == 200 - assert generation.usage.total == 500 - assert generation.calculated_input_cost == 111 - assert generation.calculated_output_cost == 222 - assert generation.calculated_total_cost == 444 + assert generation.usage_details == {"input": 100, "output": 200, "total": 500} + assert generation.cost_details == {"input": 111, "output": 222, "total": 444} + assert generation.total_cost == 444 def test_end_generation_with_openai_token_format(): @@ -1418,31 +1308,26 @@ def test_end_generation_with_openai_token_format(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = get_api().trace.get(trace_id) + observations = wait_for_observations(trace_id) # Find generation - generations = [obs for obs in trace.observations if obs.name == "query-generation"] + generations = [obs for obs in observations if obs.name == "query-generation"] assert len(generations) == 1 generation_api = generations[0] # Verify properties were converted correctly assert generation_api.end_time is not None - assert generation_api.usage.input == 100 # prompt_tokens mapped to input - assert generation_api.usage.output == 200 # completion_tokens mapped to output - assert generation_api.usage.total == 500 - assert generation_api.usage.unit == "TOKENS" # Default unit for OpenAI format - assert generation_api.calculated_input_cost == 111 - assert generation_api.calculated_output_cost == 222 - assert generation_api.calculated_total_cost == 444 + # OpenAI-style keys are mapped to input/output/total + assert generation_api.usage_details == {"input": 100, "output": 200, "total": 500} + assert generation_api.cost_details == {"input": 111, "output": 222, "total": 444} + assert generation_api.total_cost == 444 def test_end_span(): langfuse = Langfuse() - api_wrapper = LangfuseAPI() # Create a span span = langfuse.start_observation( @@ -1460,19 +1345,18 @@ def test_end_span(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = api_wrapper.get_trace(trace_id) + observations = wait_for_observations(trace_id) # Find span - spans = [obs for obs in trace["observations"] if obs["name"] == "span"] + spans = [obs for obs in observations if obs.name == "span"] assert len(spans) == 1 span_api = spans[0] # Verify end time was set - assert span_api["endTime"] is not None + assert span_api.end_time is not None def test_end_span_with_data(): @@ -1495,21 +1379,19 @@ def test_end_span_with_data(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = get_api().trace.get(trace_id) + observations = wait_for_observations(trace_id) # Find span - spans = [obs for obs in trace.observations if obs.name == "span"] + spans = [obs for obs in observations if obs.name == "span"] assert len(spans) == 1 span_api = spans[0] # Verify end time and metadata were updated assert span_api.end_time is not None - assert span_api.metadata["dict"] == "value" - assert span_api.metadata["interface"] == "whatsapp" + assert user_metadata(span_api) == {"dict": "value", "interface": "whatsapp"} def test_get_generations(): @@ -1535,16 +1417,15 @@ def test_get_generations(): # Ensure data is sent langfuse.flush() - sleep(3) # Fetch generations using API - generations = get_api().legacy.observations_v1.get_many(name=generation_name) + generations = wait_for_observations(name=generation_name) # Verify fetched generation matches what we created - assert len(generations.data) == 1 - assert generations.data[0].name == generation_name - assert generations.data[0].input == "great-prompt" - assert generations.data[0].output == "great-completion" + assert len(generations) == 1 + assert generations[0].name == generation_name + assert generations[0].input == "great-prompt" + assert generations[0].output == "great-completion" def test_get_generations_by_user(): @@ -1574,18 +1455,15 @@ def test_get_generations_by_user(): # Ensure data is sent langfuse.flush() - sleep(3) # Fetch generations by user ID using the API - generations = get_api().legacy.observations_v1.get_many( - user_id=user_id, type="GENERATION" - ) + generations = wait_for_observations(user_id=user_id, type="GENERATION") # Verify fetched generation matches what we created - assert len(generations.data) == 1 - assert generations.data[0].name == generation_name - assert generations.data[0].input == "great-prompt" - assert generations.data[0].output == "great-completion" + assert len(generations) == 1 + assert generations[0].name == generation_name + assert generations[0].input == "great-prompt" + assert generations[0].output == "great-completion" def test_kwargs(): @@ -1614,16 +1492,17 @@ def test_kwargs(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - observation = get_api().legacy.observations_v1.get(span_id) + observations = wait_for_observations(span.trace_id) + assert [observation.id for observation in observations] == [span_id] + observation = observations[0] # Verify kwargs were properly set as attributes assert observation.start_time is not None assert observation.input == {"key": "value"} assert observation.output == {"key": "value"} - assert observation.metadata["interface"] == "whatsapp" + assert user_metadata(observation) == {"interface": "whatsapp"} @pytest.mark.skip("Flaky") @@ -1660,16 +1539,13 @@ def test_timezone_awareness(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = get_api().trace.get(trace_id) + observations = wait_for_observations(trace_id, min_count=4) # Verify timestamps are in UTC regardless of local timezone - assert ( - len(trace.observations) == 4 - ) # Parent span, child span, generation, and event - for observation in trace.observations: + assert len(observations) == 4 # Parent span, child span, generation, and event + for observation in observations: # Check that start_time is within 5 seconds of current time delta = observation.start_time - utc_now assert delta.seconds < 5 @@ -1720,16 +1596,13 @@ def test_timezone_awareness_setting_timestamps(): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - trace = get_api().trace.get(trace_id) + observations = wait_for_observations(trace_id, min_count=4) # Verify timestamps are in UTC regardless of local timezone - assert ( - len(trace.observations) == 4 - ) # Parent span, child span, generation, and event - for observation in trace.observations: + assert len(observations) == 4 # Parent span, child span, generation, and event + for observation in observations: # Check that start_time is within 5 seconds of current time delta = abs((utc_now - observation.start_time).total_seconds()) assert delta < 5 @@ -1762,17 +1635,17 @@ def test_get_trace_by_session_id(): # Ensure data is sent langfuse.flush() - sleep(2) - # Retrieve the trace using the session_id - traces = get_api().trace.list(session_id=session_id) + # Retrieve the trace's observations using the session_id + observations = wait_for_observations(session_id=session_id) # Verify that the trace was retrieved correctly - assert len(traces.data) == 1 - retrieved_trace = traces.data[0] - assert retrieved_trace.name == trace_name - assert retrieved_trace.session_id == session_id - assert retrieved_trace.id == trace_id + assert len(observations) == 1 + retrieved_root = observations[0] + assert retrieved_root.is_root_observation is True + assert retrieved_root.trace_name == trace_name + assert retrieved_root.session_id == session_id + assert retrieved_root.trace_id == trace_id def test_fetch_trace(): @@ -1789,15 +1662,12 @@ def test_fetch_trace(): # Ensure data is sent langfuse.flush() - sleep(2) - # Fetch the trace using the get_api client - # Note: In the OTEL-based client, we use the API client directly - trace = get_api().trace.get(trace_id) + root = wait_for_root_observation(trace_id) # Verify trace properties - assert trace.id == trace_id - assert trace.name == name + assert root.trace_id == trace_id + assert root.trace_name == name def test_fetch_traces(): @@ -1812,7 +1682,7 @@ def test_fetch_traces(): # First trace with langfuse.start_as_current_observation(name="test1") as span: with propagate_attributes(trace_name=name, session_id="session-1"): - span.set_trace_io(input={"key": "value"}, output="output-value") + span.update(input={"key": "value"}, output="output-value") trace_ids.append(span.trace_id) sleep(1) # Ensure traces have different timestamps @@ -1820,7 +1690,7 @@ def test_fetch_traces(): # Second trace with langfuse.start_as_current_observation(name="test2") as span: with propagate_attributes(trace_name=name, session_id="session-1"): - span.set_trace_io(input={"key": "value"}, output="output-value") + span.update(input={"key": "value"}, output="output-value") trace_ids.append(span.trace_id) sleep(1) # Ensure traces have different timestamps @@ -1828,7 +1698,7 @@ def test_fetch_traces(): # Third trace with langfuse.start_as_current_observation(name="test3") as span: with propagate_attributes(trace_name=name, session_id="session-1"): - span.set_trace_io(input={"key": "value"}, output="output-value") + span.update(input={"key": "value"}, output="output-value") trace_ids.append(span.trace_id) # Ensure data is sent @@ -1836,38 +1706,50 @@ def test_fetch_traces(): expected_trace_ids = set(trace_ids) api = get_api(retry=False) + trace_name_filter = json.dumps( + [{"type": "string", "column": "traceName", "operator": "=", "value": name}] + ) - # Fetch all traces with the same name. - all_traces = wait_for_result( - lambda: api.trace.list(name=name, limit=10), - is_result_ready=lambda response: ( - {trace.id for trace in response.data} == expected_trace_ids + # Fetch the root observations of all traces with the same name. + roots = wait_for_observations( + filter=trace_name_filter, + is_result_ready=lambda observations: ( + {o.trace_id for o in observations} == expected_trace_ids ), ) # Verify we got all traces - assert len(all_traces.data) == 3 + assert len(roots) == 3 # Verify trace properties - for trace in all_traces.data: - assert trace.name == name - assert trace.session_id == "session-1" - assert trace.input == {"key": "value"} - assert trace.output == "output-value" - - # Test pagination by fetching the first three pages one at a time and - # confirming they collectively cover the created traces. - paginated_ids = set() - for page in range(1, 4): - paginated_response = wait_for_result( - lambda page=page: api.trace.list(name=name, limit=1, page=page), - is_result_ready=lambda response: ( - len(response.data) == 1 and response.data[0].id in expected_trace_ids - ), + for root in roots: + assert root.is_root_observation is True + assert root.trace_name == name + assert root.session_id == "session-1" + assert root.input == {"key": "value"} + assert root.output == "output-value" + + # Test cursor pagination by walking pages of one item and confirming they + # collectively cover the created traces. + paginated_ids = [] + cursor = None + for _ in range(3): + page = api.observations.get_many( + filter=trace_name_filter, limit=1, cursor=cursor + ) + assert len(page.data) == 1 + paginated_ids.append(page.data[0].trace_id) + cursor = page.meta.cursor + + assert set(paginated_ids) == expected_trace_ids + assert len(paginated_ids) == 3 + if cursor is not None: + assert ( + api.observations.get_many( + filter=trace_name_filter, limit=1, cursor=cursor + ).data + == [] ) - paginated_ids.add(paginated_response.data[0].id) - - assert paginated_ids == expected_trace_ids def test_get_observation(): @@ -1890,10 +1772,12 @@ def test_get_observation(): # Ensure data is sent langfuse.flush() - sleep(2) # Fetch the observation using the API - observation = get_api().legacy.observations_v1.get(generation_id) + observations = wait_for_observations(parent_span.trace_id, min_count=2) + matching = [o for o in observations if o.id == generation_id] + assert len(matching) == 1 + observation = matching[0] # Verify observation properties assert observation.id == generation_id @@ -1926,18 +1810,18 @@ def test_get_observations(): # Fetch observations using the API expected_generation_ids = {gen1_id, gen2_id} - observations = wait_for_result( - lambda: api.legacy.observations_v1.get_many(name=name, limit=10), - is_result_ready=lambda response: expected_generation_ids.issubset( - {obs.id for obs in response.data} + observations = wait_for_observations( + name=name, + is_result_ready=lambda observations: expected_generation_ids.issubset( + {obs.id for obs in observations} ), ) # Verify fetched observations - assert len(observations.data) == 2 + assert len(observations) == 2 # Filter for just the generations - generations = [obs for obs in observations.data if obs.type == "GENERATION"] + generations = [obs for obs in observations if obs.type == "GENERATION"] assert len(generations) == 2 # Verify the generation IDs match what we created @@ -1945,91 +1829,32 @@ def test_get_observations(): assert gen1_id in gen_ids assert gen2_id in gen_ids - # Test pagination by confirming both created generations can be reached - # across separate pages. - paginated_ids = set() - for page in range(1, 3): - paginated_response = wait_for_result( - lambda page=page: api.legacy.observations_v1.get_many( - name=name, limit=1, page=page - ), - is_result_ready=lambda response: ( - len(response.data) == 1 - and response.data[0].id in expected_generation_ids - ), - ) - paginated_ids.add(paginated_response.data[0].id) - - assert paginated_ids == expected_generation_ids - - -def test_get_trace_not_found(): - # Attempt to fetch a non-existent trace using the API - with pytest.raises(Exception): - get_api(retry=False).trace.get(create_uuid()) - + # Test cursor pagination by confirming both created generations can be + # reached across separate pages. + first_page = api.observations.get_many(name=name, limit=1) + assert len(first_page.data) == 1 + assert first_page.meta.cursor is not None + second_page = api.observations.get_many( + name=name, limit=1, cursor=first_page.meta.cursor + ) + assert len(second_page.data) == 1 -def test_get_observation_not_found(): - # Attempt to fetch a non-existent observation using the API - with pytest.raises(Exception): - get_api(retry=False).legacy.observations_v1.get(create_uuid()) + assert {first_page.data[0].id, second_page.data[0].id} == expected_generation_ids -def test_get_traces_empty(): - # Fetch traces with a filter that should return no results - response = get_api(retry=False).trace.list(name=create_uuid()) +def test_get_observations_for_unknown_trace_is_empty(): + response = get_api(retry=False).observations.get_many(trace_id=create_uuid()) - assert len(response.data) == 0 - assert response.meta.total_items == 0 + assert response.data == [] + assert response.meta.cursor is None def test_get_observations_empty(): # Fetch observations with a filter that should return no results - response = get_api(retry=False).legacy.observations_v1.get_many(name=create_uuid()) + response = get_api(retry=False).observations.get_many(name=create_uuid()) - assert len(response.data) == 0 - assert response.meta.total_items == 0 - - -def test_get_sessions(): - langfuse = Langfuse() - - # unique name - name = create_uuid() - session1 = create_uuid() - session2 = create_uuid() - session3 = create_uuid() - - # Create multiple traces with different session IDs - # Create first trace - with langfuse.start_as_current_observation(name=name): - with propagate_attributes(trace_name=name, session_id=session1): - pass - - # Create second trace - with langfuse.start_as_current_observation(name=name): - with propagate_attributes(trace_name=name, session_id=session2): - pass - - # Create third trace - with langfuse.start_as_current_observation(name=name): - with propagate_attributes(trace_name=name, session_id=session3): - pass - - langfuse.flush() - - # Fetch sessions - sleep(3) - response = get_api().sessions.list() - - # Assert the structure of the response, cannot check for the exact number of sessions as the table is not cleared between tests - assert hasattr(response, "data") - assert hasattr(response, "meta") - assert isinstance(response.data, list) - - # fetch only one, cannot check for the exact number of sessions as the table is not cleared between tests - response = get_api().sessions.list(limit=1, page=2) - assert len(response.data) == 1 + assert response.data == [] + assert response.meta.cursor is None @pytest.mark.skip( @@ -2037,7 +1862,6 @@ def test_get_sessions(): ) def test_create_trace_sampling_zero(): langfuse = Langfuse(sample_rate=0) - api_wrapper = LangfuseAPI() trace_name = create_uuid() # Create a span with trace properties - with sample_rate=0, this will not be sent to the API @@ -2061,12 +1885,9 @@ def test_create_trace_sampling_zero(): langfuse.flush() sleep(2) - # Try to fetch the trace - should fail as it wasn't sent to the API - fetched_trace = api_wrapper.get_trace(trace_id) - assert fetched_trace == { - "error": "LangfuseNotFoundError", - "message": f"Trace {trace_id} not found within authorized project", - } + # The trace's observations must not exist as they were never sent to the API + assert get_observations(trace_id=trace_id) == [] + assert get_scores(trace_id=trace_id) == [] def test_mask_function(request): @@ -2083,16 +1904,15 @@ def mask_func(data): return data langfuse = Langfuse(mask=mask_func) - api_wrapper = LangfuseAPI() # Create a root span with trace properties with langfuse.start_as_current_observation(name="test-span") as root_span: with propagate_attributes(trace_name="test_trace"): - root_span.set_trace_io(input={"sensitive": "data"}) + root_span.update(input={"sensitive": "data"}) # Get trace ID for later use trace_id = root_span.trace_id # Add output to the trace - root_span.set_trace_io(output={"more": "sensitive"}) + root_span.update(output={"more": "sensitive"}) # Create a generation as child gen = root_span.start_observation( @@ -2112,44 +1932,39 @@ def mask_func(data): # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - fetched_trace = api_wrapper.get_trace(trace_id) - assert fetched_trace["input"] == {"sensitive": "MASKED"} - assert fetched_trace["output"] == {"more": "MASKED"} + observations = wait_for_observations(trace_id, min_count=3) + fetched_root = get_root_observation(observations) + assert fetched_root.input == {"sensitive": "MASKED"} + assert fetched_root.output == {"more": "MASKED"} - fetched_gen = [ - o for o in fetched_trace["observations"] if o["type"] == "GENERATION" - ][0] - assert fetched_gen["input"] == {"prompt": "MASKED"} - assert fetched_gen["output"] == "MASKED" + fetched_gen = [o for o in observations if o.type == "GENERATION"][0] + assert fetched_gen.input == {"prompt": "MASKED"} + assert fetched_gen.output == "MASKED" fetched_span = [ - o - for o in fetched_trace["observations"] - if o["type"] == "SPAN" and o["name"] == "test_span" + o for o in observations if o.type == "SPAN" and o.name == "test_span" ][0] - assert fetched_span["input"] == {"data": "MASKED"} - assert fetched_span["output"] == "MASKED" + assert fetched_span.input == {"data": "MASKED"} + assert fetched_span.output == "MASKED" # Create a root span with trace properties with langfuse.start_as_current_observation(name="test-span") as root_span: with propagate_attributes(trace_name="test_trace"): - root_span.set_trace_io(input={"should_raise": "data"}) + root_span.update(input={"should_raise": "data"}) # Get trace ID for later use trace_id = root_span.trace_id # Add output to the trace - root_span.set_trace_io(output={"should_raise": "sensitive"}) + root_span.update(output={"should_raise": "sensitive"}) # Ensure data is sent langfuse.flush() - sleep(2) # Retrieve and verify - fetched_trace = api_wrapper.get_trace(trace_id) - assert fetched_trace["input"] == "" - assert fetched_trace["output"] == "" + fetched_root = wait_for_root_observation(trace_id) + assert fetched_root.input == "" + assert fetched_root.output == "" def test_get_project_id(): @@ -2218,13 +2033,11 @@ def test_start_as_current_observation_types(): pass langfuse.flush() - sleep(2) - api = get_api() - trace = api.trace.get(trace_id) + observations = wait_for_observations(trace_id, min_count=len(observation_types) + 1) # Check we have all expected observation types - found_types = {obs.type for obs in trace.observations} + found_types = {obs.type for obs in observations} expected_types = {obs_type.upper() for obs_type in observation_types} | { "SPAN" } # includes parent span @@ -2234,12 +2047,12 @@ def test_start_as_current_observation_types(): # Verify each specific observation exists for obs_type in observation_types: - observations = [ + matching = [ obs - for obs in trace.observations + for obs in observations if obs.name == f"test-{obs_type}" and obs.type == obs_type.upper() ] - assert len(observations) == 1, f"Expected one {obs_type.upper()} observation" + assert len(matching) == 1, f"Expected one {obs_type.upper()} observation" def test_that_generation_like_properties_are_actually_created(): @@ -2296,21 +2109,22 @@ def test_that_generation_like_properties_are_actually_created(): langfuse.flush() - api = get_api() - trace = api.trace.get(trace_id) + observations = wait_for_observations( + trace_id, min_count=len(generation_like_types) + 1 + ) # Verify that the properties are persisted in the API for generation-like types for obs_type in generation_like_types: - observations = [ + matching = [ obs - for obs in trace.observations + for obs in observations if obs.name == f"test-{obs_type}" and obs.type == obs_type.upper() ] - assert len(observations) == 1, ( - f"Expected one {obs_type.upper()} observation, but found {len(observations)}" + assert len(matching) == 1, ( + f"Expected one {obs_type.upper()} observation, but found {len(matching)}" ) - obs = observations[0] + obs = matching[0] assert obs.model == test_model, f"{obs_type} should have model property" assert obs.model_parameters == test_model_parameters, ( diff --git a/tests/e2e/test_decorators.py b/tests/e2e/test_decorators.py index 9fac4cf22..a33a4f3d5 100644 --- a/tests/e2e/test_decorators.py +++ b/tests/e2e/test_decorators.py @@ -15,7 +15,11 @@ from langfuse._client.resource_manager import LangfuseResourceManager from langfuse.langchain import CallbackHandler from langfuse.media import LangfuseMedia -from tests.support.utils import get_api, wait_for_trace +from tests.support.utils import ( + get_observations, + user_metadata, + wait_for_trace_snapshot, +) mock_metadata = {"key": "metadata"} mock_deep_metadata = {"key": "mock_deep_metadata"} @@ -100,7 +104,7 @@ def level_1_function(*args, **kwargs): assert result == "level_1" # Wrapped function returns correctly # ID setting for span or trace - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=3) assert len(trace_data.observations) == 3 # trace parameters if set anywhere in the call stack @@ -133,7 +137,7 @@ def level_1_function(*args, **kwargs): assert level_3_observation.name == "level_3" assert level_3_observation.metadata["key"] == mock_deep_metadata["key"] assert level_3_observation.type == "GENERATION" - assert level_3_observation.calculated_total_cost > 0 + assert level_3_observation.total_cost > 0 assert level_3_observation.output == "mock_output" assert level_3_observation.version == "version-1" @@ -182,7 +186,7 @@ def level_1_function(*args, **kwargs): assert result == "level_1" # Wrapped function returns correctly # ID setting for span or trace - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=3) assert len(trace_data.observations) == 3 # trace parameters if set anywhere in the call stack @@ -215,7 +219,7 @@ def level_1_function(*args, **kwargs): assert level_3_observation.name == "level_3" assert level_3_observation.metadata["key"] == mock_deep_metadata["key"] assert level_3_observation.type == "GENERATION" - assert level_3_observation.calculated_total_cost > 0 + assert level_3_observation.total_cost > 0 assert level_3_observation.output == "mock_output" assert level_3_observation.version == "version-1" @@ -260,7 +264,7 @@ def level_1_function(*args, **kwargs): langfuse.flush() - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=3) # trace parameters if set anywhere in the call stack assert trace_data.session_id == mock_session_id @@ -349,7 +353,7 @@ def level_1_function(*args, **kwargs): langfuse.flush() for mock_id in [mock_trace_id_1, mock_trace_id_2]: - trace_data = get_api().trace.get(mock_id) + trace_data = wait_for_trace_snapshot(mock_id, min_observations=3) assert len(trace_data.observations) == 3 # ID setting for span or trace @@ -382,7 +386,7 @@ def level_1_function(*args, **kwargs): assert level_3_observation.metadata["key"] == mock_deep_metadata["key"] assert level_3_observation.type == "GENERATION" - assert level_3_observation.calculated_total_cost > 0 + assert level_3_observation.total_cost > 0 def test_decorators_langchain(): @@ -427,7 +431,7 @@ def level_1_function(*args, **kwargs): langfuse.flush() - trace_data = wait_for_trace( + trace_data = wait_for_trace_snapshot( mock_trace_id, is_result_ready=lambda trace: ( trace.session_id == mock_session_id @@ -522,8 +526,10 @@ def level_1_function(*args, **kwargs): assert result == "level_3" # Wrapped function returns correctly # ID setting for span or trace - trace_data = wait_for_trace( + trace_data = wait_for_trace_snapshot( mock_trace_id, + min_observations=3, + min_scores=3, is_result_ready=lambda trace: { "test-observation-score", "test-trace-score", @@ -553,7 +559,7 @@ def level_1_function(*args, **kwargs): assert any( [ score.name == "another-test-trace-score" - and score.string_value == "my_value" + and score.value == "my_value" and score.data_type == "CATEGORICAL" for score in trace_scores ] @@ -599,7 +605,7 @@ def function_with_circular_arg(circular_obj, *args, **kwargs): # Validate that the function executed as expected assert result == "function response" - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=1) assert ( trace_data.observations[0].input["args"][0]["reference"] == "CircularRefObject" @@ -632,7 +638,7 @@ def main(*args, **kwargs): assert result == "function response" - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=2) # Check that disabled capture_io doesn't capture manually set input/output assert len(trace_data.observations) == 2 @@ -702,7 +708,7 @@ def level_1_function(self, *args, **kwargs): assert result == "level_1" # Wrapped function returns correctly # ID setting for span or trace - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=4) assert len(trace_data.observations) == 4 # trace parameters if set anywhere in the call stack @@ -743,7 +749,7 @@ def level_1_function(self, *args, **kwargs): assert level_3_observation.name == "level_3_function" assert level_3_observation.metadata["key"] == mock_deep_metadata["key"] assert level_3_observation.type == "GENERATION" - assert level_3_observation.calculated_total_cost > 0 + assert level_3_observation.total_cost > 0 assert level_3_observation.output == "mock_output" @@ -779,7 +785,7 @@ def main(**kwargs): assert result == mock_output - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=2) # Find the main and nested observations adjacencies = defaultdict(list) @@ -833,7 +839,7 @@ async def main_async(**kwargs): assert result == mock_output - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=2) # Check correct nesting adjacencies = defaultdict(list) @@ -902,7 +908,7 @@ async def level_1_function(*args, **kwargs): assert result == "level_1" # Wrapped function returns correctly # ID setting for span or trace - trace_data = wait_for_trace( + trace_data = wait_for_trace_snapshot( mock_trace_id, is_result_ready=lambda trace: ( trace.session_id == mock_session_id @@ -946,11 +952,11 @@ async def level_1_function(*args, **kwargs): "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.usage.input is not None - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["input"] is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None print(generation) - assert generation.output == 2 + assert generation.output == "2" def test_generator_as_function_input(): @@ -982,7 +988,7 @@ def main(**kwargs): assert result == mock_output - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=2) nested_obs = next(o for o in trace_data.observations if o.name == "nested") @@ -1019,7 +1025,7 @@ def main(**kwargs): main(langfuse_trace_id=mock_trace_id) langfuse.flush() - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=2) # Find the observation with name 'nested' nested_observation = next(o for o in trace_data.observations if o.name == "nested") @@ -1052,7 +1058,7 @@ def function(): assert result == mock_output - trace_data = wait_for_trace( + trace_data = wait_for_trace_snapshot( mock_trace_id, is_result_ready=lambda trace: any( observation.name == "function" and observation.output == mock_output @@ -1071,10 +1077,10 @@ def test_media(): media = LangfuseMedia(content_bytes=pdf_bytes, content_type="application/pdf") - @observe() + @observe(capture_input=False, capture_output=False) def main(): sleep(1) - langfuse.set_current_trace_io( + langfuse.update_current_span( input={ "context": { "nested": media, @@ -1099,7 +1105,7 @@ def main(): langfuse.flush() - trace_data = wait_for_trace( + trace_data = wait_for_trace_snapshot( mock_trace_id, is_result_ready=lambda trace: ( "@@@langfuseMedia:type=application/pdf|id=" @@ -1155,19 +1161,20 @@ def main(): langfuse.flush() - trace_data = wait_for_trace( + trace_data = wait_for_trace_snapshot( mock_trace_id, - is_result_ready=lambda trace: ( - trace.metadata is not None - and trace.metadata.get("key1") == "value1" - and trace.metadata.get("key2") == "value2" - and trace.tags == ["tag1", "tag2"] - ), + min_observations=2, + is_result_ready=lambda trace: trace.tags == ["tag1", "tag2"], ) - assert trace_data.metadata["key1"] == "value1" - assert trace_data.metadata["key2"] == "value2" + # Trace metadata comes from the root; the nested propagation only reaches + # the nested observation. + assert trace_data.metadata == {"key1": "value1"} + nested_observation = _get_observation_by_name(trace_data, "nested") + assert user_metadata(nested_observation) == {"key1": "value1", "key2": "value2"} + assert nested_observation.tags == ["tag1", "tag2"] + assert trace_data.root.tags == ["tag1"] assert trace_data.tags == ["tag1", "tag2"] @@ -1227,7 +1234,7 @@ def level_1_function(*args, **kwargs): assert result == "level_1" # Verify trace was created properly - trace_data = wait_for_trace( + trace_data = wait_for_trace_snapshot( mock_trace_id, is_result_ready=lambda trace: ( trace.name == mock_name and len(trace.observations) == 3 @@ -1287,7 +1294,7 @@ def level_1_function(*args, **kwargs): assert result == "level_4" - trace_data = wait_for_trace( + trace_data = wait_for_trace_snapshot( mock_trace_id, is_result_ready=lambda trace: ( trace.name == mock_name @@ -1362,7 +1369,7 @@ def level_1_function(*args, **kwargs): assert result == "level_1" - trace_data = wait_for_trace( + trace_data = wait_for_trace_snapshot( mock_trace_id, is_result_ready=lambda trace: ( trace.name == mock_name and len(trace.observations) == 2 @@ -1417,13 +1424,7 @@ def level_1_function(*args, **kwargs): # Should skip tracing entirely in multi-project setup without public key # This is expected behavior to prevent cross-project data leakage - try: - trace_data = get_api().trace.get(mock_trace_id) - # If trace is found, it should have no observations (tracing was skipped) - assert len(trace_data.observations) == 0 - except Exception: - # Trace not found is also expected - tracing was completely disabled - pass + assert get_observations(trace_id=mock_trace_id) == [] # Reset instances to not leak to other test suites removeMockResourceManagerInstances() @@ -1486,7 +1487,7 @@ async def async_level_1_function(*args, **kwargs): assert result == "async_level_3" # Verify trace was created properly - trace_data = wait_for_trace( + trace_data = wait_for_trace_snapshot( mock_trace_id, is_result_ready=lambda trace: ( trace.name == mock_name @@ -1568,7 +1569,7 @@ async def async_level_1_function(*args, **kwargs): assert result == "sync_level_4" - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=4) assert len(trace_data.observations) == 4 assert trace_data.name == mock_name @@ -1647,8 +1648,8 @@ async def async_level_1_function(task_id, *args, **kwargs): assert result2 == "async_level_3_task_2" # Verify both traces were created correctly and didn't interfere - trace_data_1 = get_api().trace.get(trace_id_1) - trace_data_2 = get_api().trace.get(trace_id_2) + trace_data_1 = wait_for_trace_snapshot(trace_id_1, min_observations=3) + trace_data_2 = wait_for_trace_snapshot(trace_id_2, min_observations=3) assert trace_data_1.name == f"{mock_name}_task_1" assert trace_data_2.name == f"{mock_name}_task_2" @@ -1721,7 +1722,7 @@ async def async_consumer_function(): assert result == "Hello, Async World!" - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=2) assert len(trace_data.observations) == 2 assert trace_data.name == mock_name @@ -1786,7 +1787,7 @@ async def async_root_function(*args, **kwargs): assert result == "exception_handled" - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=3) assert len(trace_data.observations) == 3 assert trace_data.name == mock_name @@ -1851,11 +1852,11 @@ def root_function(): ) # Verify trace structure - trace_data = wait_for_trace( + trace_data = wait_for_trace_snapshot( mock_trace_id, is_result_ready=lambda trace: ( len(trace.observations) >= 2 - and {"parent_root", "child_stream"}.issubset( + and {"root", "sync_generator"}.issubset( { observation.name for observation in trace.observations @@ -1928,7 +1929,7 @@ async def root_function(): ) # Verify trace structure - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=2) assert len(trace_data.observations) == 2 # Verify both observations are present @@ -1996,7 +1997,7 @@ async def parent_function(): ) # Verify trace structure - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=2) assert len(trace_data.observations) == 2 # Check both observations exist @@ -2043,7 +2044,7 @@ async def root_function(): assert items == ["first_item"] # Verify trace structure - should have both observations despite exception - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=2) assert len(trace_data.observations) == 2 # Check that the failing generator observation has ERROR level @@ -2083,7 +2084,7 @@ def root_function(): assert items == [] # Verify trace structure - trace_data = get_api().trace.get(mock_trace_id) + trace_data = wait_for_trace_snapshot(mock_trace_id, min_observations=2) assert len(trace_data.observations) == 2 # Verify empty generator observation diff --git a/tests/e2e/test_media.py b/tests/e2e/test_media.py index d322e1788..ad8a8cb37 100644 --- a/tests/e2e/test_media.py +++ b/tests/e2e/test_media.py @@ -4,7 +4,7 @@ from langfuse._client.client import Langfuse from langfuse.media import LangfuseMedia -from tests.support.utils import wait_for_trace +from tests.support.utils import wait_for_observations def test_replace_media_reference_string_in_object(): @@ -30,25 +30,26 @@ def test_replace_media_reference_string_in_object(): langfuse.flush() - fetched_trace = wait_for_trace( + fetched_observations = wait_for_observations( span.trace_id, - is_result_ready=lambda trace: ( - bool(trace.observations) - and re.match( + is_result_ready=lambda observations: ( + re.match( r"^@@@langfuseMedia:type=audio/wav\|id=.+\|source=base64_data_uri@@@$", - trace.observations[0].metadata.get("context", {}).get("nested", ""), + observations[0].metadata.get("context", {}).get("nested", ""), ) is not None ), ) - media_ref = fetched_trace.observations[0].metadata["context"]["nested"] + assert len(fetched_observations) == 1 + fetched_observation = fetched_observations[0] + media_ref = fetched_observation.metadata["context"]["nested"] assert re.match( r"^@@@langfuseMedia:type=audio/wav\|id=.+\|source=base64_data_uri@@@$", media_ref, ) resolved_obs = langfuse.resolve_media_references( - obj=fetched_trace.observations[0], resolve_with="base64_data_uri" + obj=fetched_observation, resolve_with="base64_data_uri" ) expected_base64 = f"data:audio/wav;base64,{base64_audio}" @@ -61,15 +62,11 @@ def test_replace_media_reference_string_in_object(): langfuse.flush() - fetched_trace2 = wait_for_trace( + fetched_observations2 = wait_for_observations( span2.trace_id, - is_result_ready=lambda trace: ( - bool(trace.observations) - and trace.observations[0].metadata.get("context", {}).get("nested") - == fetched_trace.observations[0].metadata["context"]["nested"] + is_result_ready=lambda observations: ( + observations[0].metadata.get("context", {}).get("nested") == media_ref ), ) - assert ( - fetched_trace2.observations[0].metadata["context"]["nested"] - == fetched_trace.observations[0].metadata["context"]["nested"] - ) + assert len(fetched_observations2) == 1 + assert fetched_observations2[0].metadata["context"]["nested"] == media_ref diff --git a/tests/e2e/test_prompt.py b/tests/e2e/test_prompt.py index 6e113cb41..5b611d1fb 100644 --- a/tests/e2e/test_prompt.py +++ b/tests/e2e/test_prompt.py @@ -1,7 +1,7 @@ import pytest from langfuse._client.client import Langfuse -from tests.support.utils import create_uuid, get_api +from tests.support.utils import create_uuid, wait_for_observations def test_create_prompt(): @@ -429,18 +429,17 @@ def test_prompt_end_to_end(): langfuse.flush() - api = get_api() - - trace = api.trace.get(generation.trace_id) - - assert len(trace.observations) == 1 - - generation = trace.observations[0] - assert generation.prompt_id is not None + observations = wait_for_observations( + generation.trace_id, + is_result_ready=lambda observations: observations[0].prompt_id is not None, + ) - observation = api.legacy.observations_v1.get(generation.id) + assert len(observations) == 1 + observation = observations[0] assert observation.prompt_id is not None + assert observation.prompt_name == "test" + assert observation.prompt_version == prompt.version def test_do_not_return_fallback_if_fetch_success(): @@ -525,11 +524,10 @@ def test_do_not_link_observation_if_fallback(): ).end() langfuse.flush() - api = get_api() - trace = api.trace.get(generation.trace_id) + observations = wait_for_observations(generation.trace_id) - assert len(trace.observations) == 1 - assert trace.observations[0].prompt_id is None + assert len(observations) == 1 + assert observations[0].prompt_id is None def test_variable_names_on_content_with_variable_names(): diff --git a/tests/live_provider/test_langchain.py b/tests/live_provider/test_langchain.py index d04ddb81c..c1ebe8566 100644 --- a/tests/live_provider/test_langchain.py +++ b/tests/live_provider/test_langchain.py @@ -1,3 +1,4 @@ +import json import random import string import time @@ -18,7 +19,12 @@ from langfuse._client.client import Langfuse from langfuse.langchain import CallbackHandler -from tests.support.utils import create_uuid, encode_file_to_base64, get_api +from tests.support.utils import ( + create_uuid, + encode_file_to_base64, + wait_for_observations, + wait_for_trace_snapshot, +) def test_callback_generated_from_trace_chat(): @@ -44,7 +50,7 @@ def test_callback_generated_from_trace_chat(): langfuse.flush() - trace = get_api().trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) assert trace.input is None assert trace.output is None @@ -92,7 +98,7 @@ def test_callback_generated_from_lcel_chain(): langfuse.flush() - trace = get_api().trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) assert trace.input is None assert trace.output is None @@ -151,10 +157,10 @@ def test_basic_chat_openai(): # Ensure data is flushed to API sleep(2) - # Retrieve trace by name - traces = get_api().trace.list(name=test_name) - assert len(traces.data) > 0 - trace = get_api().trace.get(traces.data[0].id) + # Retrieve trace by its root observation, which carries the run name + roots = wait_for_observations(name=test_name) + assert len(roots) > 0 + trace = wait_for_trace_snapshot(roots[0].trace_id, min_observations=2) # Assertions assert trace.name == test_name @@ -193,7 +199,7 @@ def test_callback_simple_openai(): sleep(2) # Retrieve trace - trace = get_api().trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) # Assertions - add 1 for the wrapping span assert len(trace.observations) > 1 @@ -241,7 +247,7 @@ def test_callback_multiple_invocations_on_different_traces(): sleep(2) # Retrieve trace - trace = get_api().trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=3) # Add 1 to account for the wrapping span assert len(trace.observations) > 2 @@ -300,7 +306,7 @@ def test_openai_instruct_usage(): lf_handler._langfuse_client.flush() - observations = get_api().trace.get(trace_id).observations + observations = wait_for_observations(trace_id, min_count=3) assert len(observations) >= 3 assert any( @@ -318,7 +324,7 @@ def test_openai_instruct_usage(): assert observation.output != "" assert observation.input is not None assert observation.input != "" - assert observation.usage is not None + assert observation.usage_details is not None assert observation.usage_details["input"] is not None assert observation.usage_details["output"] is not None assert observation.usage_details["total"] is not None @@ -479,7 +485,7 @@ def test_link_langfuse_prompts_invoke(): langfuse_handler._langfuse_client.flush() sleep(2) - trace = get_api().trace.get(trace_id=trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=4) observations = trace.observations @@ -567,7 +573,12 @@ def test_link_langfuse_prompts_stream(): langfuse_handler._langfuse_client.flush() sleep(2) - trace = get_api().trace.get(trace_id=trace_id) + trace = wait_for_trace_snapshot( + trace_id, + is_result_ready=lambda trace: ( + len([o for o in trace.observations if o.type == "GENERATION"]) >= 4 + ), + ) observations = trace.observations @@ -653,11 +664,26 @@ def test_link_langfuse_prompts_batch(): langfuse_handler._langfuse_client.flush() - traces = get_api().trace.list(name=trace_name).data + trace_name_filter = json.dumps( + [ + { + "type": "string", + "column": "traceName", + "operator": "=", + "value": trace_name, + } + ] + ) + traced_observations = wait_for_observations(filter=trace_name_filter) - assert len(traces) == 1 + assert {o.trace_id for o in traced_observations} == {trace_id} - trace = get_api().trace.get(trace_id=trace_id) + trace = wait_for_trace_snapshot( + trace_id, + is_result_ready=lambda trace: ( + len([o for o in trace.observations if o.type == "GENERATION"]) >= 10 + ), + ) observations = trace.observations @@ -783,7 +809,7 @@ class GetWeather(BaseModel): handler._langfuse_client.flush() - trace = get_api().trace.get(trace_id=trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) generations = list(filter(lambda x: x.type == "GENERATION", trace.observations)) assert len(generations) > 0 @@ -882,7 +908,7 @@ def test_multimodal(): handler._langfuse_client.flush() - trace = get_api().trace.get(trace_id=trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) assert len(trace.observations) >= 2 assert any( @@ -979,7 +1005,7 @@ def call_model(state: MessagesState): print(final_state["messages"][-1].content) handler._langfuse_client.flush() - trace = get_api().trace.get(trace_id=trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) assert len(trace.observations) > 0 @@ -1012,7 +1038,7 @@ def test_cached_token_usage(): handler._langfuse_client.flush() - trace = get_api().trace.get(handler.get_trace_id()) + trace = wait_for_trace_snapshot(handler.get_trace_id()) generation = next((o for o in trace.observations if o.type == "GENERATION")) diff --git a/tests/live_provider/test_langchain_integration.py b/tests/live_provider/test_langchain_integration.py index edb5455c4..dd6b166b2 100644 --- a/tests/live_provider/test_langchain_integration.py +++ b/tests/live_provider/test_langchain_integration.py @@ -7,7 +7,7 @@ from langfuse import Langfuse from langfuse.langchain import CallbackHandler -from tests.support.utils import create_uuid, get_api +from tests.support.utils import create_uuid, wait_for_trace_snapshot def _is_streaming_response(response): @@ -45,8 +45,7 @@ def test_stream_chat_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -61,15 +60,15 @@ def test_stream_chat_models(model_name): assert generation.model_parameters.get("max_completion_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert generation.metadata["tags"] == tags - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.output["content"] is not None assert generation.output["role"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -100,8 +99,7 @@ def test_stream_completions_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -116,14 +114,14 @@ def test_stream_completions_models(model_name): assert generation.model_parameters.get("max_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert generation.metadata["tags"] == tags - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.output is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -150,8 +148,7 @@ def test_invoke_chat_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -165,15 +162,15 @@ def test_invoke_chat_models(model_name): assert generation.model_parameters.get("max_completion_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert generation.metadata["tags"] == tags - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.output["content"] is not None assert generation.output["role"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -201,8 +198,7 @@ def test_invoke_in_completions_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -216,14 +212,14 @@ def test_invoke_in_completions_models(model_name): assert generation.model_parameters.get("max_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert generation.metadata["tags"] == tags - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert test_phrase in generation.output assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -251,8 +247,7 @@ def test_batch_in_completions_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=3) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -266,13 +261,13 @@ def test_batch_in_completions_models(model_name): assert generation.model_parameters.get("max_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert generation.metadata["tags"] == tags - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -300,8 +295,7 @@ def test_batch_in_chat_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=3) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -314,13 +308,13 @@ def test_batch_in_chat_models(model_name): assert generation.model_parameters.get("max_completion_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert generation.metadata["tags"] == tags - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -354,8 +348,7 @@ async def test_astream_chat_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -369,15 +362,15 @@ async def test_astream_chat_models(model_name): assert generation.model_parameters.get("max_completion_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert generation.metadata["tags"] == tags - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.output["content"] is not None assert generation.output["role"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -411,8 +404,7 @@ async def test_astream_completions_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -427,14 +419,14 @@ async def test_astream_completions_models(model_name): assert generation.model_parameters.get("max_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert generation.metadata["tags"] == tags - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert test_phrase in generation.output assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -463,8 +455,7 @@ async def test_ainvoke_chat_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -478,15 +469,15 @@ async def test_ainvoke_chat_models(model_name): assert generation.model_parameters.get("max_completion_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert generation.metadata["tags"] == tags - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.output["content"] is not None assert generation.output["role"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -514,8 +505,7 @@ async def test_ainvoke_in_completions_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=2) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -529,14 +519,14 @@ async def test_ainvoke_in_completions_models(model_name): assert generation.model_parameters.get("max_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert generation.metadata["tags"] == tags - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert test_phrase in generation.output assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -571,8 +561,7 @@ def test_chains_batch_in_chat_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=9) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -585,13 +574,13 @@ def test_chains_batch_in_chat_models(model_name): assert generation.model_parameters.get("max_completion_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert all(x in generation.metadata["tags"] for x in tags) - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -622,8 +611,7 @@ def test_chains_batch_in_completions_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=9) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -636,13 +624,13 @@ def test_chains_batch_in_completions_models(model_name): assert generation.model_parameters.get("max_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert all(x in generation.metadata["tags"] for x in tags) - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -675,8 +663,7 @@ async def test_chains_abatch_in_chat_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=9) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -689,13 +676,13 @@ async def test_chains_abatch_in_chat_models(model_name): assert generation.model_parameters.get("max_completion_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert all(x in generation.metadata["tags"] for x in tags) - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -725,8 +712,7 @@ async def test_chains_abatch_in_completions_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=9) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 assert len(trace.observations) == 9 @@ -738,13 +724,13 @@ async def test_chains_abatch_in_completions_models(model_name): assert generation.model_parameters.get("max_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert all(x in generation.metadata["tags"] for x in tags) - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -778,8 +764,7 @@ async def test_chains_ainvoke_chat_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=5) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -792,15 +777,15 @@ async def test_chains_ainvoke_chat_models(model_name): assert generation.model_parameters.get("max_completion_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert all(x in generation.metadata["tags"] for x in tags) - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.output["content"] is not None assert generation.output["role"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -834,8 +819,7 @@ async def test_chains_ainvoke_completions_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=5) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -848,13 +832,13 @@ async def test_chains_ainvoke_completions_models(model_name): assert generation.model_parameters.get("max_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert all(x in generation.metadata["tags"] for x in tags) - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -894,8 +878,7 @@ async def test_chains_astream_chat_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=5) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -910,15 +893,15 @@ async def test_chains_astream_chat_models(model_name): assert generation.model_parameters.get("max_completion_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert all(x in generation.metadata["tags"] for x in tags) - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.output["content"] is not None assert generation.output["role"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None @@ -956,8 +939,7 @@ async def test_chains_astream_completions_models(model_name): langfuse_client.flush() assert handler.runs == {} - api = get_api() - trace = api.trace.get(trace_id) + trace = wait_for_trace_snapshot(trace_id, min_observations=5) generationList = list(filter(lambda o: o.type == "GENERATION", trace.observations)) assert len(generationList) != 0 @@ -972,11 +954,11 @@ async def test_chains_astream_completions_models(model_name): assert generation.model_parameters.get("max_tokens") is not None assert generation.model_parameters.get("temperature") is not None assert all(x in generation.metadata["tags"] for x in tags) - assert generation.usage.output is not None - assert generation.usage.total is not None + assert generation.usage_details["output"] is not None + assert generation.usage_details["total"] is not None assert generation.input_price is not None assert generation.output_price is not None - assert generation.calculated_input_cost is not None - assert generation.calculated_output_cost is not None - assert generation.calculated_total_cost is not None + assert generation.cost_details["input"] is not None + assert generation.cost_details["output"] is not None + assert generation.total_cost is not None assert generation.latency is not None diff --git a/tests/live_provider/test_openai.py b/tests/live_provider/test_openai.py index 43a07be23..18dc61b81 100644 --- a/tests/live_provider/test_openai.py +++ b/tests/live_provider/test_openai.py @@ -6,7 +6,11 @@ from pydantic import BaseModel from langfuse._client.client import Langfuse -from tests.support.utils import create_uuid, encode_file_to_base64, get_api +from tests.support.utils import ( + create_uuid, + encode_file_to_base64, + wait_for_observations, +) langfuse: Langfuse | None = None @@ -58,27 +62,25 @@ def test_openai_chat_completion(openai): sleep(1) - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].metadata["someKey"] == "someResponse" + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].metadata["someKey"] == "someResponse" assert len(completion.choices) != 0 - assert generation.data[0].input == [ + assert generation[0].input == [ { "content": "You are an expert mathematician", "role": "assistant", }, {"content": "1 + 1 = ", "role": "user"}, ] - assert generation.data[0].type == "GENERATION" - assert "gpt-3.5-turbo-0125" in generation.data[0].model - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + assert generation[0].type == "GENERATION" + assert "gpt-3.5-turbo-0125" in generation[0].model + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "service_tier": "default", "temperature": 0, "top_p": 1, @@ -86,11 +88,11 @@ def test_openai_chat_completion(openai): "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - assert "2" in generation.data[0].output["content"] - assert generation.data[0].output["role"] == "assistant" + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None + assert "2" in generation[0].output["content"] + assert generation[0].output["role"] == "assistant" def test_openai_chat_completion_stream(openai): @@ -116,21 +118,19 @@ def test_openai_chat_completion_stream(openai): langfuse.flush() sleep(3) - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") + + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].metadata["someKey"] == "someResponse" - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].metadata["someKey"] == "someResponse" - - assert generation.data[0].input == [{"content": "1 + 1 = ", "role": "user"}] - assert generation.data[0].type == "GENERATION" - assert "gpt-3.5-turbo-0125" in generation.data[0].model - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + assert generation[0].input == [{"content": "1 + 1 = ", "role": "user"}] + assert generation[0].type == "GENERATION" + assert "gpt-3.5-turbo-0125" in generation[0].model + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "service_tier": "default", "temperature": 0, "top_p": 1, @@ -138,16 +138,16 @@ def test_openai_chat_completion_stream(openai): "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - assert generation.data[0].output == 2 - assert generation.data[0].completion_start_time is not None + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None + assert generation[0].output == "2" + assert generation[0].completion_start_time is not None # Completion start time for time-to-first-token - assert generation.data[0].completion_start_time is not None - assert generation.data[0].completion_start_time >= generation.data[0].start_time - assert generation.data[0].completion_start_time <= generation.data[0].end_time + assert generation[0].completion_start_time is not None + assert generation[0].completion_start_time >= generation[0].start_time + assert generation[0].completion_start_time <= generation[0].end_time def test_openai_chat_completion_stream_with_next_iteration(openai): @@ -177,21 +177,19 @@ def test_openai_chat_completion_stream_with_next_iteration(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") + + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].metadata["someKey"] == "someResponse" - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].metadata["someKey"] == "someResponse" - - assert generation.data[0].input == [{"content": "1 + 1 = ", "role": "user"}] - assert generation.data[0].type == "GENERATION" - assert generation.data[0].model == "gpt-3.5-turbo-0125" - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + assert generation[0].input == [{"content": "1 + 1 = ", "role": "user"}] + assert generation[0].type == "GENERATION" + assert generation[0].model == "gpt-3.5-turbo-0125" + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "service_tier": "default", "temperature": 0, "top_p": 1, @@ -199,16 +197,16 @@ def test_openai_chat_completion_stream_with_next_iteration(openai): "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - assert generation.data[0].output == 2 - assert generation.data[0].completion_start_time is not None + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None + assert generation[0].output == "2" + assert generation[0].completion_start_time is not None # Completion start time for time-to-first-token - assert generation.data[0].completion_start_time is not None - assert generation.data[0].completion_start_time >= generation.data[0].start_time - assert generation.data[0].completion_start_time <= generation.data[0].end_time + assert generation[0].completion_start_time is not None + assert generation[0].completion_start_time >= generation[0].start_time + assert generation[0].completion_start_time <= generation[0].end_time def test_openai_chat_completion_stream_fail(openai): @@ -227,33 +225,28 @@ def test_openai_chat_completion_stream_fail(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].metadata["someKey"] == "someResponse" - - assert generation.data[0].input == [{"content": "1 + 1 = ", "role": "user"}] - assert generation.data[0].type == "GENERATION" - assert generation.data[0].model == "fake" - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].metadata["someKey"] == "someResponse" + + assert generation[0].input == [{"content": "1 + 1 = ", "role": "user"}] + assert generation[0].type == "GENERATION" + assert generation[0].model == "fake" + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "temperature": 0, "top_p": 1, "frequency_penalty": 0, "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - assert generation.data[0].level == "ERROR" - assert generation.data[0].status_message is not None - assert generation.data[0].output is None + assert generation[0].level == "ERROR" + assert generation[0].status_message is not None + assert generation[0].output is None openai.api_key = os.environ["OPENAI_API_KEY"] @@ -277,13 +270,11 @@ def test_openai_chat_completion_with_langfuse_prompt(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert isinstance(generation.data[0].prompt_id, str) + assert len(generation) != 0 + assert generation[0].name == generation_name + assert isinstance(generation[0].prompt_id, str) def test_openai_chat_completion_fail(openai): @@ -300,29 +291,27 @@ def test_openai_chat_completion_fail(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) - - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].metadata["someKey"] == "someResponse" - assert generation.data[0].input == [{"content": "1 + 1 = ", "role": "user"}] - assert generation.data[0].type == "GENERATION" - assert generation.data[0].model == "fake" - assert generation.data[0].level == "ERROR" - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].status_message is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + generation = wait_for_observations(name=generation_name, type="GENERATION") + + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].metadata["someKey"] == "someResponse" + assert generation[0].input == [{"content": "1 + 1 = ", "role": "user"}] + assert generation[0].type == "GENERATION" + assert generation[0].model == "fake" + assert generation[0].level == "ERROR" + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].status_message is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "temperature": 0, "top_p": 1, "frequency_penalty": 0, "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].output is None + assert generation[0].output is None openai.api_key = os.environ["OPENAI_API_KEY"] @@ -360,25 +349,21 @@ def test_openai_chat_completion_two_calls(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name + assert len(generation) != 0 + assert generation[0].name == generation_name assert len(completion.choices) != 0 - assert generation.data[0].input == [{"content": "1 + 1 = ", "role": "user"}] + assert generation[0].input == [{"content": "1 + 1 = ", "role": "user"}] - generation_2 = get_api().legacy.observations_v1.get_many( - name=generation_name_2, type="GENERATION" - ) + generation_2 = wait_for_observations(name=generation_name_2, type="GENERATION") - assert len(generation_2.data) != 0 - assert generation_2.data[0].name == generation_name_2 + assert len(generation_2) != 0 + assert generation_2[0].name == generation_name_2 assert len(completion_2.choices) != 0 - assert generation_2.data[0].input == [{"content": "2 + 2 = ", "role": "user"}] + assert generation_2[0].input == [{"content": "2 + 2 = ", "role": "user"}] def test_openai_chat_completion_with_seed(openai): @@ -394,11 +379,9 @@ def test_openai_chat_completion_with_seed(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert generation.data[0].model_parameters == { + assert generation[0].model_parameters == { "service_tier": "default", "temperature": 0, "top_p": 1, @@ -423,32 +406,30 @@ def test_openai_completion(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].metadata["someKey"] == "someResponse" + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].metadata["someKey"] == "someResponse" assert len(completion.choices) != 0 - assert completion.choices[0].text == generation.data[0].output - assert generation.data[0].input == "1 + 1 = " - assert generation.data[0].type == "GENERATION" - assert "gpt-3.5-turbo-instruct" in generation.data[0].model - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + assert completion.choices[0].text == generation[0].output + assert generation[0].input == "1 + 1 = " + assert generation[0].type == "GENERATION" + assert "gpt-3.5-turbo-instruct" in generation[0].model + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "temperature": 0, "top_p": 1, "frequency_penalty": 0, "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - assert generation.data[0].output == "2\n\n1 + 2 = 3\n\n2 + 3 = " + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None + assert generation[0].output == "2\n\n1 + 2 = 3\n\n2 + 3 = " @requires_legacy_completion_model @@ -472,37 +453,35 @@ def test_openai_completion_stream(openai): assert len(content) > 0 - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].metadata["someKey"] == "someResponse" - - assert generation.data[0].input == "1 + 1 = " - assert generation.data[0].type == "GENERATION" - assert "gpt-3.5-turbo-instruct" in generation.data[0].model - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].metadata["someKey"] == "someResponse" + + assert generation[0].input == "1 + 1 = " + assert generation[0].type == "GENERATION" + assert "gpt-3.5-turbo-instruct" in generation[0].model + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "temperature": 0, "top_p": 1, "frequency_penalty": 0, "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - assert generation.data[0].output == "2\n\n1 + 2 = 3\n\n2 + 3 = " - assert generation.data[0].completion_start_time is not None + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None + assert generation[0].output == "2\n\n1 + 2 = 3\n\n2 + 3 = " + assert generation[0].completion_start_time is not None # Completion start time for time-to-first-token - assert generation.data[0].completion_start_time is not None - assert generation.data[0].completion_start_time >= generation.data[0].start_time - assert generation.data[0].completion_start_time <= generation.data[0].end_time + assert generation[0].completion_start_time is not None + assert generation[0].completion_start_time >= generation[0].start_time + assert generation[0].completion_start_time <= generation[0].end_time def test_openai_completion_fail(openai): @@ -521,29 +500,27 @@ def test_openai_completion_fail(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) - - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].metadata["someKey"] == "someResponse" - assert generation.data[0].input == "1 + 1 = " - assert generation.data[0].type == "GENERATION" - assert generation.data[0].model == "fake" - assert generation.data[0].level == "ERROR" - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].status_message is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + generation = wait_for_observations(name=generation_name, type="GENERATION") + + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].metadata["someKey"] == "someResponse" + assert generation[0].input == "1 + 1 = " + assert generation[0].type == "GENERATION" + assert generation[0].model == "fake" + assert generation[0].level == "ERROR" + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].status_message is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "temperature": 0, "top_p": 1, "frequency_penalty": 0, "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].output is None + assert generation[0].output is None openai.api_key = os.environ["OPENAI_API_KEY"] @@ -564,33 +541,28 @@ def test_openai_completion_stream_fail(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") + + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].metadata["someKey"] == "someResponse" - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].metadata["someKey"] == "someResponse" - - assert generation.data[0].input == "1 + 1 = " - assert generation.data[0].type == "GENERATION" - assert generation.data[0].model == "gpt-3.5-turbo" - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + assert generation[0].input == "1 + 1 = " + assert generation[0].type == "GENERATION" + assert generation[0].model == "gpt-3.5-turbo" + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "temperature": 0, "top_p": 1, "frequency_penalty": 0, "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - assert generation.data[0].level == "ERROR" - assert generation.data[0].status_message is not None - assert generation.data[0].output is None + assert generation[0].level == "ERROR" + assert generation[0].status_message is not None + assert generation[0].output is None openai.api_key = os.environ["OPENAI_API_KEY"] @@ -614,13 +586,11 @@ def test_openai_completion_with_langfuse_prompt(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert isinstance(generation.data[0].prompt_id, str) + assert len(generation) != 0 + assert generation[0].name == generation_name + assert isinstance(generation[0].prompt_id, str) def test_fails_wrong_name(openai): @@ -656,21 +626,19 @@ async def test_async_chat(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name + assert len(generation) != 0 + assert generation[0].name == generation_name assert len(completion.choices) != 0 - assert generation.data[0].input == [{"content": "1 + 1 = ", "role": "user"}] - assert generation.data[0].type == "GENERATION" - assert generation.data[0].model == "gpt-3.5-turbo-0125" - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + assert generation[0].input == [{"content": "1 + 1 = ", "role": "user"}] + assert generation[0].type == "GENERATION" + assert generation[0].model == "gpt-3.5-turbo-0125" + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "service_tier": "default", "temperature": 1, "top_p": 1, @@ -678,11 +646,11 @@ async def test_async_chat(openai): "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - assert "2" in generation.data[0].output["content"] - assert generation.data[0].output["role"] == "assistant" + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None + assert "2" in generation[0].output["content"] + assert generation[0].output["role"] == "assistant" @pytest.mark.asyncio @@ -703,19 +671,17 @@ async def test_async_chat_stream(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) - - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].input == [{"content": "1 + 1 = ", "role": "user"}] - assert generation.data[0].type == "GENERATION" - assert generation.data[0].model == "gpt-3.5-turbo-0125" - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + generation = wait_for_observations(name=generation_name, type="GENERATION") + + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].input == [{"content": "1 + 1 = ", "role": "user"}] + assert generation[0].type == "GENERATION" + assert generation[0].model == "gpt-3.5-turbo-0125" + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "service_tier": "default", "temperature": 1, "top_p": 1, @@ -723,15 +689,15 @@ async def test_async_chat_stream(openai): "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - assert "2" in str(generation.data[0].output) + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None + assert "2" in str(generation[0].output) # Completion start time for time-to-first-token - assert generation.data[0].completion_start_time is not None - assert generation.data[0].completion_start_time >= generation.data[0].start_time - assert generation.data[0].completion_start_time <= generation.data[0].end_time + assert generation[0].completion_start_time is not None + assert generation[0].completion_start_time >= generation[0].start_time + assert generation[0].completion_start_time <= generation[0].end_time @pytest.mark.asyncio @@ -762,21 +728,19 @@ async def test_async_chat_stream_with_anext(openai): print(result) - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].input == [ + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].input == [ {"content": "Give me a one-liner joke", "role": "user"} ] - assert generation.data[0].type == "GENERATION" - assert generation.data[0].model == "gpt-3.5-turbo-0125" - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + assert generation[0].type == "GENERATION" + assert generation[0].model == "gpt-3.5-turbo-0125" + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "service_tier": "default", "temperature": 1, "top_p": 1, @@ -784,14 +748,14 @@ async def test_async_chat_stream_with_anext(openai): "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None # Completion start time for time-to-first-token - assert generation.data[0].completion_start_time is not None - assert generation.data[0].completion_start_time >= generation.data[0].start_time - assert generation.data[0].completion_start_time <= generation.data[0].end_time + assert generation[0].completion_start_time is not None + assert generation[0].completion_start_time >= generation[0].start_time + assert generation[0].completion_start_time <= generation[0].end_time def test_openai_function_call(openai): @@ -825,14 +789,12 @@ class StepByStepAIResponse(BaseModel): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].output is not None - assert "function_call" in generation.data[0].output + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].output is not None + assert "function_call" in generation[0].output assert output["title"] is not None @@ -869,14 +831,12 @@ class StepByStepAIResponse(BaseModel): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].output is not None - assert "function_call" in generation.data[0].output + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].output is not None + assert "function_call" in generation[0].output def test_openai_tool_call(openai): @@ -913,21 +873,17 @@ def test_openai_tool_call(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name + assert len(generation) != 0 + assert generation[0].name == generation_name assert ( - generation.data[0].output["tool_calls"][0]["function"]["name"] + generation[0].output["tool_calls"][0]["function"]["name"] == "get_current_weather" ) - assert ( - generation.data[0].output["tool_calls"][0]["function"]["arguments"] is not None - ) - assert generation.data[0].input["tools"] == tools - assert generation.data[0].input["messages"] == messages + assert generation[0].output["tool_calls"][0]["function"]["arguments"] is not None + assert generation[0].input["tools"] == tools + assert generation[0].input["messages"] == messages def test_openai_tool_call_streamed(openai): @@ -969,22 +925,18 @@ def test_openai_tool_call_streamed(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name + assert len(generation) != 0 + assert generation[0].name == generation_name assert ( - generation.data[0].output["tool_calls"][0]["function"]["name"] + generation[0].output["tool_calls"][0]["function"]["name"] == "get_current_weather" ) - assert ( - generation.data[0].output["tool_calls"][0]["function"]["arguments"] is not None - ) - assert generation.data[0].input["tools"] == tools - assert generation.data[0].input["messages"] == messages + assert generation[0].output["tool_calls"][0]["function"]["arguments"] is not None + assert generation[0].input["tools"] == tools + assert generation[0].input["messages"] == messages def test_langchain_integration(openai): @@ -1047,28 +999,28 @@ def test_structured_output_response_format_kwarg(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" + generation = wait_for_observations( + name=generation_name, type="GENERATION", expand_metadata="response_format" ) - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].metadata["someKey"] == "someResponse" - assert generation.data[0].metadata["response_format"] == { + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].metadata["someKey"] == "someResponse" + assert generation[0].metadata["response_format"] == { "type": "json_schema", "json_schema": json_schema, } - assert generation.data[0].input == [ + assert generation[0].input == [ {"role": "system", "content": "You are a helpful math tutor."}, {"content": "solve 8x + 31 = 2", "role": "user"}, ] - assert generation.data[0].type == "GENERATION" - assert generation.data[0].model == "gpt-4o-2024-08-06" - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + assert generation[0].type == "GENERATION" + assert generation[0].model == "gpt-4o-2024-08-06" + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "service_tier": "default", "temperature": 1, "top_p": 1, @@ -1076,10 +1028,10 @@ def test_structured_output_response_format_kwarg(openai): "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - assert generation.data[0].output["role"] == "assistant" + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None + assert generation[0].output["role"] == "assistant" def test_structured_output_beta_completions_parse(openai): @@ -1117,31 +1069,29 @@ class CalendarEvent(BaseModel): if Version(openai.__version__) >= Version("1.50.0"): # Check the trace and observation properties - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) == 1 - assert generation.data[0].name == generation_name - assert generation.data[0].type == "GENERATION" - assert "gpt-4o" in generation.data[0].model - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time + assert len(generation) == 1 + assert generation[0].name == generation_name + assert generation[0].type == "GENERATION" + assert "gpt-4o" in generation[0].model + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time # Check input and output - assert len(generation.data[0].input) == 2 - assert generation.data[0].input[0]["role"] == "system" - assert generation.data[0].input[1]["role"] == "user" - assert isinstance(generation.data[0].output, dict) - assert "name" in generation.data[0].output["content"] - assert "date" in generation.data[0].output["content"] - assert "participants" in generation.data[0].output["content"] + assert len(generation[0].input) == 2 + assert generation[0].input[0]["role"] == "system" + assert generation[0].input[1]["role"] == "user" + assert isinstance(generation[0].output, dict) + assert "name" in generation[0].output["content"] + assert "date" in generation[0].output["content"] + assert "participants" in generation[0].output["content"] # Check usage - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None @pytest.mark.asyncio @@ -1163,19 +1113,17 @@ async def test_close_async_stream(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) - - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].input == [{"content": "1 + 1 = ", "role": "user"}] - assert generation.data[0].type == "GENERATION" - assert generation.data[0].model == "gpt-3.5-turbo-0125" - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].model_parameters == { + generation = wait_for_observations(name=generation_name, type="GENERATION") + + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].input == [{"content": "1 + 1 = ", "role": "user"}] + assert generation[0].type == "GENERATION" + assert generation[0].model == "gpt-3.5-turbo-0125" + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].model_parameters == { "service_tier": "default", "temperature": 1, "top_p": 1, @@ -1183,15 +1131,15 @@ async def test_close_async_stream(openai): "max_tokens": "Infinity", "presence_penalty": 0, } - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - assert "2" in str(generation.data[0].output) + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None + assert "2" in str(generation[0].output) # Completion start time for time-to-first-token - assert generation.data[0].completion_start_time is not None - assert generation.data[0].completion_start_time >= generation.data[0].start_time - assert generation.data[0].completion_start_time <= generation.data[0].end_time + assert generation[0].completion_start_time is not None + assert generation[0].completion_start_time >= generation[0].start_time + assert generation[0].completion_start_time <= generation[0].end_time def test_base_64_image_input(openai): @@ -1225,26 +1173,24 @@ def test_base_64_image_input(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name - assert generation.data[0].input[0]["content"][0]["text"] == "What’s in this image?" + assert len(generation) != 0 + assert generation[0].name == generation_name + assert generation[0].input[0]["content"][0]["text"] == "What’s in this image?" assert ( f"@@@langfuseMedia:type={content_type}|id=" - in generation.data[0].input[0]["content"][1]["image_url"]["url"] + in generation[0].input[0]["content"][1]["image_url"]["url"] ) - assert generation.data[0].type == "GENERATION" - assert "gpt-4o-mini" in generation.data[0].model - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - assert "dog" in generation.data[0].output["content"] + assert generation[0].type == "GENERATION" + assert "gpt-4o-mini" in generation[0].model + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None + assert "dog" in generation[0].output["content"] def test_audio_input_and_output(openai): @@ -1277,32 +1223,28 @@ def test_audio_input_and_output(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - assert generation.data[0].name == generation_name + assert len(generation) != 0 + assert generation[0].name == generation_name assert ( - generation.data[0].input[0]["content"][0]["text"] - == "Do what this recording says." + generation[0].input[0]["content"][0]["text"] == "Do what this recording says." ) assert ( "@@@langfuseMedia:type=audio/wav|id=" - in generation.data[0].input[0]["content"][1]["input_audio"]["data"] - ) - assert generation.data[0].type == "GENERATION" - assert generation.data[0].model == model - assert generation.data[0].start_time is not None - assert generation.data[0].end_time is not None - assert generation.data[0].start_time < generation.data[0].end_time - assert generation.data[0].usage.input is not None - assert generation.data[0].usage.output is not None - assert generation.data[0].usage.total is not None - print(generation.data[0].output) + in generation[0].input[0]["content"][1]["input_audio"]["data"] + ) + assert generation[0].type == "GENERATION" + assert generation[0].model == model + assert generation[0].start_time is not None + assert generation[0].end_time is not None + assert generation[0].start_time < generation[0].end_time + assert generation[0].usage_details["input"] is not None + assert generation[0].usage_details["output"] is not None + assert generation[0].usage_details["total"] is not None + print(generation[0].output) assert ( - "@@@langfuseMedia:type=audio/wav|id=" - in generation.data[0].output["audio"]["data"] + "@@@langfuseMedia:type=audio/wav|id=" in generation[0].output["audio"]["data"] ) @@ -1317,25 +1259,22 @@ def test_response_api_text_input(openai): ) langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - generationData = generation.data[0] + assert len(generation) != 0 + generationData = generation[0] assert generationData.name == generation_name assert ( - generation.data[0].input - == "Tell me a three sentence bedtime story about a unicorn." + generation[0].input == "Tell me a three sentence bedtime story about a unicorn." ) assert generationData.type == "GENERATION" assert "gpt-4o" in generationData.model assert generationData.start_time is not None assert generationData.end_time is not None assert generationData.start_time < generationData.end_time - assert generationData.usage.input is not None - assert generationData.usage.output is not None - assert generationData.usage.total is not None + assert generationData.usage_details["input"] is not None + assert generationData.usage_details["output"] is not None + assert generationData.usage_details["total"] is not None assert generationData.output is not None @@ -1363,22 +1302,20 @@ def test_response_api_image_input(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - generationData = generation.data[0] + assert len(generation) != 0 + generationData = generation[0] assert generationData.name == generation_name - assert generation.data[0].input[0]["content"][0]["text"] == "what is in this image?" + assert generation[0].input[0]["content"][0]["text"] == "what is in this image?" assert generationData.type == "GENERATION" assert "gpt-4o" in generationData.model assert generationData.start_time is not None assert generationData.end_time is not None assert generationData.start_time < generationData.end_time - assert generationData.usage.input is not None - assert generationData.usage.output is not None - assert generationData.usage.total is not None + assert generationData.usage_details["input"] is not None + assert generationData.usage_details["output"] is not None + assert generationData.usage_details["total"] is not None assert generationData.output is not None @@ -1395,12 +1332,10 @@ def test_response_api_web_search(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - generationData = generation.data[0] + assert len(generation) != 0 + generationData = generation[0] assert generationData.name == generation_name assert generationData.input == { "input": "What was a positive news story from today?", @@ -1411,9 +1346,9 @@ def test_response_api_web_search(openai): assert generationData.start_time is not None assert generationData.end_time is not None assert generationData.start_time < generationData.end_time - assert generationData.usage.input is not None - assert generationData.usage.output is not None - assert generationData.usage.total is not None + assert generationData.usage_details["input"] is not None + assert generationData.usage_details["output"] is not None + assert generationData.usage_details["total"] is not None assert generationData.output is not None assert generationData.metadata is not None @@ -1435,14 +1370,12 @@ def test_response_api_streaming(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - generationData = generation.data[0] + assert len(generation) != 0 + generationData = generation[0] assert generationData.name == generation_name - assert generation.data[0].input == [ + assert generation[0].input == [ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Hello!"}, ] @@ -1451,9 +1384,9 @@ def test_response_api_streaming(openai): assert generationData.start_time is not None assert generationData.end_time is not None assert generationData.start_time < generationData.end_time - assert generationData.usage.input is not None - assert generationData.usage.output is not None - assert generationData.usage.total is not None + assert generationData.usage_details["input"] is not None + assert generationData.usage_details["output"] is not None + assert generationData.usage_details["total"] is not None assert generationData.output is not None assert generationData.metadata is not None assert generationData.metadata["instructions"] == "You are a helpful assistant." @@ -1492,14 +1425,12 @@ def test_response_api_functions(openai): langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - generationData = generation.data[0] + assert len(generation) != 0 + generationData = generation[0] assert generationData.name == generation_name - assert generation.data[0].input == { + assert generation[0].input == { "input": "What is the weather like in Boston today?", "tools": tools, "tool_choice": "auto", @@ -1509,9 +1440,9 @@ def test_response_api_functions(openai): assert generationData.start_time is not None assert generationData.end_time is not None assert generationData.start_time < generationData.end_time - assert generationData.usage.input is not None - assert generationData.usage.output is not None - assert generationData.usage.total is not None + assert generationData.usage_details["input"] is not None + assert generationData.usage_details["output"] is not None + assert generationData.usage_details["total"] is not None assert generationData.output is not None assert generationData.metadata is not None @@ -1528,22 +1459,20 @@ def test_response_api_reasoning(openai): ) langfuse.flush() - generation = get_api().legacy.observations_v1.get_many( - name=generation_name, type="GENERATION" - ) + generation = wait_for_observations(name=generation_name, type="GENERATION") - assert len(generation.data) != 0 - generationData = generation.data[0] + assert len(generation) != 0 + generationData = generation[0] assert generationData.name == generation_name - assert generation.data[0].input == "How much wood would a woodchuck chuck?" + assert generation[0].input == "How much wood would a woodchuck chuck?" assert generationData.type == "GENERATION" assert "o3-mini" in generationData.model assert generationData.start_time is not None assert generationData.end_time is not None assert generationData.start_time < generationData.end_time - assert generationData.usage.input is not None - assert generationData.usage.output is not None - assert generationData.usage.total is not None + assert generationData.usage_details["input"] is not None + assert generationData.usage_details["output"] is not None + assert generationData.usage_details["total"] is not None assert generationData.output is not None assert generationData.metadata is not None @@ -1560,12 +1489,10 @@ def test_openai_embeddings(openai): langfuse.flush() sleep(1) - embedding = get_api().legacy.observations_v1.get_many( - name=embedding_name, type="EMBEDDING" - ) + embedding = wait_for_observations(name=embedding_name, type="EMBEDDING") - assert len(embedding.data) != 0 - embedding_data = embedding.data[0] + assert len(embedding) != 0 + embedding_data = embedding[0] assert embedding_data.name == embedding_name assert embedding_data.metadata["test_key"] == "test_value" assert embedding_data.input == "The quick brown fox jumps over the lazy dog" @@ -1574,8 +1501,8 @@ def test_openai_embeddings(openai): assert embedding_data.start_time is not None assert embedding_data.end_time is not None assert embedding_data.start_time < embedding_data.end_time - assert embedding_data.usage.input is not None - assert embedding_data.usage.total is not None + assert embedding_data.usage_details["input"] is not None + assert embedding_data.usage_details["total"] is not None assert embedding_data.output is not None assert "dimensions" in embedding_data.output assert "count" in embedding_data.output @@ -1596,18 +1523,16 @@ def test_openai_embeddings_multiple_inputs(openai): langfuse.flush() sleep(1) - embedding = get_api().legacy.observations_v1.get_many( - name=embedding_name, type="EMBEDDING" - ) + embedding = wait_for_observations(name=embedding_name, type="EMBEDDING") - assert len(embedding.data) != 0 - embedding_data = embedding.data[0] + assert len(embedding) != 0 + embedding_data = embedding[0] assert embedding_data.name == embedding_name assert embedding_data.input == inputs assert embedding_data.type == "EMBEDDING" assert "text-embedding-ada-002" in embedding_data.model - assert embedding_data.usage.input is not None - assert embedding_data.usage.total is not None + assert embedding_data.usage_details["input"] is not None + assert embedding_data.usage_details["total"] is not None assert embedding_data.output["count"] == len(inputs) @@ -1629,16 +1554,14 @@ async def test_async_openai_embeddings(openai): langfuse.flush() sleep(1) - embedding = get_api().legacy.observations_v1.get_many( - name=embedding_name, type="EMBEDDING" - ) + embedding = wait_for_observations(name=embedding_name, type="EMBEDDING") - assert len(embedding.data) != 0 - embedding_data = embedding.data[0] + assert len(embedding) != 0 + embedding_data = embedding[0] assert embedding_data.name == embedding_name assert embedding_data.input == "Async embedding test" assert embedding_data.type == "EMBEDDING" assert "text-embedding-ada-002" in embedding_data.model assert embedding_data.metadata["async"] is True - assert embedding_data.usage.input is not None - assert embedding_data.usage.total is not None + assert embedding_data.usage_details["input"] is not None + assert embedding_data.usage_details["total"] is not None diff --git a/tests/support/api_wrapper.py b/tests/support/api_wrapper.py deleted file mode 100644 index c4519252f..000000000 --- a/tests/support/api_wrapper.py +++ /dev/null @@ -1,130 +0,0 @@ -import os - -import httpx - -from langfuse.api.commons.errors.not_found_error import NotFoundError -from tests.support.retry import ( - DEFAULT_RETRY_INTERVAL_SECONDS, - DEFAULT_RETRY_TIMEOUT_SECONDS, - is_not_found_payload, - retry_until_ready, -) - - -class LangfuseAPI: - def __init__(self, username=None, password=None, base_url=None): - username = username if username else os.environ["LANGFUSE_PUBLIC_KEY"] - password = password if password else os.environ["LANGFUSE_SECRET_KEY"] - self.auth = (username, password) - self.BASE_URL = base_url if base_url else os.environ["LANGFUSE_BASE_URL"] - - def _get_json( - self, - url, - params=None, - *, - retry=True, - is_result_ready=None, - timeout_seconds=DEFAULT_RETRY_TIMEOUT_SECONDS, - interval_seconds=DEFAULT_RETRY_INTERVAL_SECONDS, - ): - def _request(): - response = httpx.get(url, params=params, auth=self.auth) - payload = response.json() - - if response.status_code == 404 and is_not_found_payload(payload): - raise NotFoundError(body=payload, headers=dict(response.headers)) - - return payload - - if not retry: - return _request() - - return retry_until_ready( - _request, - is_result_ready=is_result_ready, - timeout_seconds=timeout_seconds, - interval_seconds=interval_seconds, - ) - - def get_observation( - self, - observation_id, - *, - retry=True, - is_result_ready=None, - timeout_seconds=DEFAULT_RETRY_TIMEOUT_SECONDS, - interval_seconds=DEFAULT_RETRY_INTERVAL_SECONDS, - ): - url = f"{self.BASE_URL}/api/public/observations/{observation_id}" - return self._get_json( - url, - retry=retry, - is_result_ready=is_result_ready, - timeout_seconds=timeout_seconds, - interval_seconds=interval_seconds, - ) - - def get_scores( - self, - page=None, - limit=None, - user_id=None, - name=None, - *, - retry=True, - is_result_ready=None, - timeout_seconds=DEFAULT_RETRY_TIMEOUT_SECONDS, - interval_seconds=DEFAULT_RETRY_INTERVAL_SECONDS, - ): - params = {"page": page, "limit": limit, "userId": user_id, "name": name} - url = f"{self.BASE_URL}/api/public/scores" - return self._get_json( - url, - params=params, - retry=retry, - is_result_ready=is_result_ready, - timeout_seconds=timeout_seconds, - interval_seconds=interval_seconds, - ) - - def get_traces( - self, - page=None, - limit=None, - user_id=None, - name=None, - *, - retry=True, - is_result_ready=None, - timeout_seconds=DEFAULT_RETRY_TIMEOUT_SECONDS, - interval_seconds=DEFAULT_RETRY_INTERVAL_SECONDS, - ): - params = {"page": page, "limit": limit, "userId": user_id, "name": name} - url = f"{self.BASE_URL}/api/public/traces" - return self._get_json( - url, - params=params, - retry=retry, - is_result_ready=is_result_ready, - timeout_seconds=timeout_seconds, - interval_seconds=interval_seconds, - ) - - def get_trace( - self, - trace_id, - *, - retry=True, - is_result_ready=None, - timeout_seconds=DEFAULT_RETRY_TIMEOUT_SECONDS, - interval_seconds=DEFAULT_RETRY_INTERVAL_SECONDS, - ): - url = f"{self.BASE_URL}/api/public/traces/{trace_id}" - return self._get_json( - url, - retry=retry, - is_result_ready=is_result_ready, - timeout_seconds=timeout_seconds, - interval_seconds=interval_seconds, - ) diff --git a/tests/support/utils.py b/tests/support/utils.py index a29274d3a..191eeae05 100644 --- a/tests/support/utils.py +++ b/tests/support/utils.py @@ -1,19 +1,42 @@ import base64 +import json import os -from typing import Any, Callable, TypeVar +from dataclasses import dataclass +from typing import Any, Callable, Sequence, TypeVar from uuid import uuid4 -from langfuse.api import LangfuseAPI +from langfuse.api import LangfuseAPI, ObservationV2, ScoreV3 from tests.support.retry import ( DEFAULT_RETRY_INTERVAL_SECONDS, DEFAULT_RETRY_TIMEOUT_SECONDS, retry_until_ready, ) -READ_METHOD_NAMES = {"get", "get_by_id", "get_many", "get_run", "list"} -PAGINATION_ARGUMENTS = {"limit", "page"} +READ_METHOD_NAMES = {"get", "get_by_id", "get_many", "get_many_v3", "get_run", "list"} +PAGINATION_ARGUMENTS = {"limit", "page", "cursor", "fields", "expand_metadata"} T = TypeVar("T") +ALL_OBSERVATION_FIELDS = ( + "core,basic,time,io,metadata,model,usage,prompt,metrics,trace_context" +) +SCORE_FIELDS = "details,subject" + +# The v2 observations API returns "" instead of null for unset string fields. +_EMPTY_AS_NONE_FIELDS = ( + "name", + "status_message", + "version", + "user_id", + "session_id", + "model", + "internal_model_id", + "prompt_id", + "prompt_name", + "trace_name", + "release", +) +_SDK_METADATA_KEY_PREFIXES = ("scope.", "resourceAttributes.") + def _has_filters(kwargs: dict[str, Any]) -> bool: return any( @@ -51,7 +74,9 @@ def _call(*args: Any, **kwargs: Any) -> Any: def _result_ready(method_name: str, kwargs: dict[str, Any]): - if method_name not in {"get_many", "list"} or not _has_filters(kwargs): + if method_name not in {"get_many", "get_many_v3", "list"} or not _has_filters( + kwargs + ): return None def _has_data(result: Any) -> bool: @@ -89,17 +114,315 @@ def wait_for_result( ) -def wait_for_trace( +def _parse_json_string(value: Any) -> Any: + # Only objects and arrays: the SDK sends string IO unquoted, so text such as + # "2" or "true" must stay a string. + if not isinstance(value, str) or not value.lstrip().startswith(("{", "[")): + return value + + try: + return json.loads(value) + except ValueError: + return value + + +def normalize_observation(observation: ObservationV2) -> ObservationV2: + """Map a v2 observation to the shape the SDK sent. + + The v2 API returns input/output as raw strings and unset string fields as + "". This parses JSON object/array input/output and maps "" to None. + """ + update: dict[str, Any] = { + "input": _parse_json_string(observation.input), + "output": _parse_json_string(observation.output), + } + for field in _EMPTY_AS_NONE_FIELDS: + if getattr(observation, field, None) == "": + update[field] = None + + return observation.model_copy(update=update) + + +def user_metadata(observation: ObservationV2) -> dict[str, Any]: + """Observation metadata without the scope/resource keys the server adds.""" + metadata = observation.metadata or {} + assert isinstance(metadata, dict), metadata + + return { + key: value + for key, value in metadata.items() + if not key.startswith(_SDK_METADATA_KEY_PREFIXES) + } + + +def get_observations( + *, + fields: str = ALL_OBSERVATION_FIELDS, + api: Any = None, + **filters: Any, +) -> list[ObservationV2]: + """Fetch all observations matching `filters` (all pages), oldest first.""" + api = api or get_api(retry=False) + observations: list[ObservationV2] = [] + cursor = None + + while True: + response = api.observations.get_many( + fields=fields, limit=1000, cursor=cursor, **filters + ) + observations.extend(response.data) + cursor = response.meta.cursor + if not cursor or not response.data: + break + + # The events table can briefly return several rows for one span until + # ClickHouse merges them; keep the most recently updated row per id. + latest_by_id: dict[str, ObservationV2] = {} + for observation in observations: + current = latest_by_id.get(observation.id) + if current is None or (observation.updated_at or observation.start_time) >= ( + current.updated_at or current.start_time + ): + latest_by_id[observation.id] = observation + + return sorted( + (normalize_observation(observation) for observation in latest_by_id.values()), + key=lambda observation: observation.start_time, + ) + + +def wait_for_observations( + trace_id: str | None = None, + *, + min_count: int = 1, + is_result_ready: Callable[[list[ObservationV2]], bool] | None = None, + fields: str = ALL_OBSERVATION_FIELDS, + timeout_seconds: float = DEFAULT_RETRY_TIMEOUT_SECONDS, + interval_seconds: float = DEFAULT_RETRY_INTERVAL_SECONDS, + **filters: Any, +) -> list[ObservationV2]: + """Poll the v2 observations API until at least `min_count` observations + match (and `is_result_ready` holds), then return them oldest first.""" + if trace_id is not None: + filters["trace_id"] = trace_id + assert filters, "wait_for_observations needs at least one filter" + + def _ready(observations: list[ObservationV2]) -> bool: + return len(observations) >= min_count and ( + is_result_ready is None or is_result_ready(observations) + ) + + return wait_for_result( + lambda: get_observations(fields=fields, **filters), + is_result_ready=_ready, + timeout_seconds=timeout_seconds, + interval_seconds=interval_seconds, + ) + + +def get_root_observation(observations: Sequence[ObservationV2]) -> ObservationV2: + """Return the single root observation, which carries the trace-level + attributes (trace name, user, session, tags, public) and trace IO.""" + roots = [ + observation for observation in observations if observation.is_root_observation + ] + assert len(roots) == 1, ( + f"expected exactly one root observation, got {[r.name for r in roots]}" + ) + + return roots[0] + + +def wait_for_root_observation( + trace_id: str, + *, + min_count: int = 1, + is_result_ready: Callable[[ObservationV2], bool] | None = None, + timeout_seconds: float = DEFAULT_RETRY_TIMEOUT_SECONDS, + interval_seconds: float = DEFAULT_RETRY_INTERVAL_SECONDS, +) -> ObservationV2: + def _ready(observations: list[ObservationV2]) -> bool: + roots = [o for o in observations if o.is_root_observation] + return len(roots) == 1 and ( + is_result_ready is None or is_result_ready(roots[0]) + ) + + return get_root_observation( + wait_for_observations( + trace_id, + min_count=min_count, + is_result_ready=_ready, + timeout_seconds=timeout_seconds, + interval_seconds=interval_seconds, + ) + ) + + +@dataclass(frozen=True) +class TraceSnapshot: + """A trace as v4 exposes it: its observations plus (optionally) scores. + + Mirrors how the platform aggregates events into a trace: input, output + and metadata come from the root observation; name, user, session, + version, release and environment are the latest non-empty value across + all observations; tags are the union; public is true if any observation + is public. + """ + + id: str + observations: list[ObservationV2] + scores: list[ScoreV3] | None = None + + @property + def root(self) -> ObservationV2: + return get_root_observation(self.observations) + + def _latest(self, field: str) -> Any: + values = [ + getattr(observation, field) + for observation in self.observations + if getattr(observation, field) + ] + return values[-1] if values else None + + @property + def name(self) -> str | None: + # The API falls back to the root's own name when no trace name was + # set, so a root whose trace_name equals its name carries no signal. + explicit_names = [ + observation.trace_name + for observation in self.observations + if observation.trace_name + and not ( + observation.is_root_observation + and observation.trace_name == observation.name + ) + ] + return explicit_names[-1] if explicit_names else self.root.trace_name + + @property + def user_id(self) -> str | None: + return self._latest("user_id") + + @property + def session_id(self) -> str | None: + return self._latest("session_id") + + @property + def tags(self) -> list[str]: + return sorted( + {tag for observation in self.observations for tag in observation.tags or []} + ) + + @property + def public(self) -> bool: + return any(observation.public for observation in self.observations) + + @property + def version(self) -> str | None: + return self._latest("version") + + @property + def release(self) -> str | None: + return self._latest("release") + + @property + def environment(self) -> str | None: + return self._latest("environment") + + @property + def input(self) -> Any: + return self.root.input + + @property + def output(self) -> Any: + return self.root.output + + @property + def metadata(self) -> dict[str, Any]: + return user_metadata(self.root) + + +def wait_for_trace_snapshot( trace_id: str, *, - is_result_ready: Callable[[Any], bool] | None = None, + min_observations: int = 1, + min_scores: int | None = None, + is_result_ready: Callable[[TraceSnapshot], bool] | None = None, timeout_seconds: float = DEFAULT_RETRY_TIMEOUT_SECONDS, interval_seconds: float = DEFAULT_RETRY_INTERVAL_SECONDS, -): - api = get_api(retry=False) +) -> TraceSnapshot: + """Poll until the trace has `min_observations` observations (and + `min_scores` scores, which are only fetched when given).""" + + def _fetch() -> TraceSnapshot: + return TraceSnapshot( + id=trace_id, + observations=get_observations(trace_id=trace_id), + scores=None if min_scores is None else get_scores(trace_id=trace_id), + ) + + def _ready(snapshot: TraceSnapshot) -> bool: + if len(snapshot.observations) < min_observations: + return False + if min_scores is not None and len(snapshot.scores or []) < min_scores: + return False + if is_result_ready is None: + return True + try: + return is_result_ready(snapshot) + except AssertionError: + # Root-derived attributes are unavailable until the root arrives. + return False + return wait_for_result( - lambda: api.trace.get(trace_id), - is_result_ready=is_result_ready, + _fetch, + is_result_ready=_ready, + timeout_seconds=timeout_seconds, + interval_seconds=interval_seconds, + ) + + +def get_scores( + *, fields: str = SCORE_FIELDS, api: Any = None, **filters: Any +) -> list[ScoreV3]: + api = api or get_api(retry=False) + scores: list[ScoreV3] = [] + cursor = None + + while True: + response = api.scores_v3.get_many_v3( + fields=fields, limit=100, cursor=cursor, **filters + ) + scores.extend(response.data) + cursor = response.meta.cursor + if not cursor or not response.data: + break + + return scores + + +def wait_for_scores( + *, + min_count: int = 1, + is_result_ready: Callable[[list[ScoreV3]], bool] | None = None, + fields: str = SCORE_FIELDS, + timeout_seconds: float = DEFAULT_RETRY_TIMEOUT_SECONDS, + interval_seconds: float = DEFAULT_RETRY_INTERVAL_SECONDS, + **filters: Any, +) -> list[ScoreV3]: + """Poll the v3 scores API (filters: trace_id, session_id, observation_id, + name, ...) until at least `min_count` scores match.""" + assert filters, "wait_for_scores needs at least one filter" + + def _ready(scores: list[ScoreV3]) -> bool: + return len(scores) >= min_count and ( + is_result_ready is None or is_result_ready(scores) + ) + + return wait_for_result( + lambda: get_scores(fields=fields, **filters), + is_result_ready=_ready, timeout_seconds=timeout_seconds, interval_seconds=interval_seconds, ) diff --git a/tests/unit/test_e2e_support.py b/tests/unit/test_e2e_support.py index 8320bd2fe..98d25127c 100644 --- a/tests/unit/test_e2e_support.py +++ b/tests/unit/test_e2e_support.py @@ -1,144 +1,260 @@ +from datetime import datetime, timedelta, timezone from types import SimpleNamespace +from langfuse.api import ObservationV2 from langfuse.api.commons.errors.not_found_error import NotFoundError -from tests.support.api_wrapper import LangfuseAPI as SupportLangfuseAPI from tests.support.retry import retry_until_ready -from tests.support.utils import get_api, wait_for_trace +from tests.support.utils import ( + TraceSnapshot, + get_api, + get_observations, + normalize_observation, + user_metadata, + wait_for_observations, + wait_for_scores, + wait_for_trace_snapshot, +) +START = datetime(2024, 1, 1, tzinfo=timezone.utc) -def test_get_api_retries_not_found(monkeypatch): - monkeypatch.setattr("tests.support.retry.sleep", lambda _: None) - attempts = {"count": 0} +def _observation(index: int = 0, **fields) -> ObservationV2: + defaults = { + "id": f"obs-{index}", + "trace_id": "trace-123", + "start_time": START + timedelta(seconds=index), + "project_id": "project", + "type": "SPAN", + "is_root_observation": False, + } + return ObservationV2(**{**defaults, **fields}) - class FakeTraceService: - def get(self, trace_id): - attempts["count"] += 1 - if attempts["count"] < 3: - raise NotFoundError( - body={ - "error": "LangfuseNotFoundError", - "message": f"Trace {trace_id} not found within authorized project", - } - ) +def _page(data, cursor=None): + return SimpleNamespace(data=data, meta=SimpleNamespace(cursor=cursor)) - return {"id": trace_id} - class FakeClient: - trace = FakeTraceService() +def _install_client(monkeypatch, **services): + monkeypatch.setattr("tests.support.retry.sleep", lambda _: None) + client = SimpleNamespace(**services) + monkeypatch.setattr("tests.support.utils.LangfuseAPI", lambda **_: client) - monkeypatch.setattr("tests.support.utils.LangfuseAPI", lambda **_: FakeClient()) - trace = get_api().trace.get("trace-123") +def test_get_api_retries_not_found(monkeypatch): + attempts = {"count": 0} - assert trace == {"id": "trace-123"} - assert attempts["count"] == 3 + def get_many(**kwargs): + attempts["count"] += 1 + if attempts["count"] < 3: + raise NotFoundError( + body={ + "error": "LangfuseNotFoundError", + "message": "Observations not found within authorized project", + } + ) -def test_get_api_retries_filtered_lists(monkeypatch): - monkeypatch.setattr("tests.support.retry.sleep", lambda _: None) + return _page([kwargs["trace_id"]]) - attempts = {"count": 0} + _install_client(monkeypatch, observations=SimpleNamespace(get_many=get_many)) - class FakeTraceService: - def list(self, **kwargs): - attempts["count"] += 1 + response = get_api().observations.get_many(trace_id="trace-123") - if attempts["count"] < 3: - return SimpleNamespace(data=[]) + assert response.data == ["trace-123"] + assert attempts["count"] == 3 - return SimpleNamespace(data=[kwargs["name"]]) - class FakeClient: - trace = FakeTraceService() +def test_get_api_retries_filtered_lists(monkeypatch): + attempts = {"count": 0} - monkeypatch.setattr("tests.support.utils.LangfuseAPI", lambda **_: FakeClient()) + def get_many(**kwargs): + attempts["count"] += 1 + return _page([] if attempts["count"] < 3 else [kwargs["name"]]) - response = get_api().trace.list(name="ready-trace") + _install_client(monkeypatch, observations=SimpleNamespace(get_many=get_many)) - assert response.data == ["ready-trace"] + response = get_api().observations.get_many(name="ready-observation") + + assert response.data == ["ready-observation"] assert attempts["count"] == 3 def test_get_api_retry_can_be_disabled(monkeypatch): attempts = {"count": 0} - class FakeTraceService: - def list(self, **kwargs): - attempts["count"] += 1 - return SimpleNamespace(data=[]) - - class FakeClient: - trace = FakeTraceService() + def get_many(**kwargs): + attempts["count"] += 1 + return _page([]) - monkeypatch.setattr("tests.support.utils.LangfuseAPI", lambda **_: FakeClient()) + _install_client(monkeypatch, observations=SimpleNamespace(get_many=get_many)) - response = get_api(retry=False).trace.list(name="missing-trace") + response = get_api(retry=False).observations.get_many(name="missing") assert response.data == [] assert attempts["count"] == 1 -def test_raw_api_wrapper_retries_not_found_payload(monkeypatch): - monkeypatch.setattr("tests.support.retry.sleep", lambda _: None) +def test_normalize_observation_parses_io_and_maps_empty_strings_to_none(): + observation = normalize_observation( + _observation( + input='{"question": "hi"}', + output="plain text", + name="", + session_id="", + user_id="user-1", + ) + ) - attempts = {"count": 0} + assert observation.input == {"question": "hi"} + assert observation.output == "plain text" + assert observation.name is None + assert observation.session_id is None + assert observation.user_id == "user-1" - class FakeResponse: - def __init__(self, status_code, payload): - self.status_code = status_code - self._payload = payload - self.headers = {} - def json(self): - return self._payload +def test_user_metadata_drops_server_added_keys(): + observation = _observation( + metadata={ + "key": "value", + "scope.name": "langfuse-sdk", + "resourceAttributes.service.name": "test", + } + ) - def fake_get(*args, **kwargs): - attempts["count"] += 1 + assert user_metadata(observation) == {"key": "value"} - if attempts["count"] < 3: - return FakeResponse( - 404, - { - "error": "LangfuseNotFoundError", - "message": "Trace trace-123 not found within authorized project", - }, - ) - return FakeResponse(200, {"id": "trace-123", "observations": []}) +def test_get_observations_follows_cursor_and_sorts_by_start_time(monkeypatch): + calls = [] + pages = { + None: _page([_observation(2), _observation(0)], cursor="next"), + "next": _page([_observation(1)]), + } - monkeypatch.setattr("tests.support.api_wrapper.httpx.get", fake_get) + def get_many(**kwargs): + calls.append(kwargs) + return pages[kwargs["cursor"]] - api = SupportLangfuseAPI(username="user", password="pass", base_url="http://test") - trace = api.get_trace("trace-123") + _install_client(monkeypatch, observations=SimpleNamespace(get_many=get_many)) - assert trace["id"] == "trace-123" - assert attempts["count"] == 3 + observations = get_observations(trace_id="trace-123") + assert [o.id for o in observations] == ["obs-0", "obs-1", "obs-2"] + assert [call["cursor"] for call in calls] == [None, "next"] + assert all(call["trace_id"] == "trace-123" for call in calls) -def test_wait_for_trace_retries_until_predicate_matches(monkeypatch): - monkeypatch.setattr("tests.support.retry.sleep", lambda _: None) +def test_wait_for_observations_polls_until_min_count(monkeypatch): attempts = {"count": 0} - class FakeTraceService: - def get(self, trace_id): - attempts["count"] += 1 - return {"id": trace_id, "observations": [1] * attempts["count"]} + def get_many(**kwargs): + attempts["count"] += 1 + return _page([_observation(i) for i in range(attempts["count"])]) + + _install_client(monkeypatch, observations=SimpleNamespace(get_many=get_many)) - class FakeClient: - trace = FakeTraceService() + observations = wait_for_observations("trace-123", min_count=3) + + assert len(observations) == 3 + assert attempts["count"] == 3 - monkeypatch.setattr("tests.support.utils.LangfuseAPI", lambda **_: FakeClient()) - trace = wait_for_trace( - "trace-123", is_result_ready=lambda trace: len(trace["observations"]) == 3 +def test_wait_for_trace_snapshot_waits_for_root_and_scores(monkeypatch): + attempts = {"observations": 0, "scores": 0} + + def get_many(**kwargs): + attempts["observations"] += 1 + observations = [_observation(1, name="child")] + if attempts["observations"] >= 2: + observations.append(_observation(0, name="root", is_root_observation=True)) + return _page(observations) + + def get_many_v3(**kwargs): + attempts["scores"] += 1 + return _page(["score"] if attempts["scores"] >= 3 else []) + + _install_client( + monkeypatch, + observations=SimpleNamespace(get_many=get_many), + scores_v3=SimpleNamespace(get_many_v3=get_many_v3), ) - assert trace["id"] == "trace-123" - assert len(trace["observations"]) == 3 - assert attempts["count"] == 3 + snapshot = wait_for_trace_snapshot( + "trace-123", + min_scores=1, + is_result_ready=lambda trace: trace.root.name == "root", + ) + + assert snapshot.root.name == "root" + assert snapshot.scores == ["score"] + assert attempts["scores"] == 3 + + +def test_trace_snapshot_aggregates_trace_attributes_like_the_platform(): + snapshot = TraceSnapshot( + id="trace-123", + observations=[ + _observation( + 0, + name="root", + trace_name="root", + is_root_observation=True, + input={"q": 1}, + metadata={"root_key": "root", "scope.name": "sdk"}, + tags=["b"], + ), + _observation( + 1, + name="child", + trace_name="explicit-name", + session_id="session-1", + user_id="user-1", + tags=["a"], + public=True, + ), + _observation(2, name="grandchild", session_id="session-2"), + ], + ) + + assert snapshot.name == "explicit-name" + assert snapshot.session_id == "session-2" + assert snapshot.user_id == "user-1" + assert snapshot.tags == ["a", "b"] + assert snapshot.public is True + assert snapshot.input == {"q": 1} + assert snapshot.metadata == {"root_key": "root"} + + +def test_trace_snapshot_name_falls_back_to_root_trace_name(): + snapshot = TraceSnapshot( + id="trace-123", + observations=[ + _observation(0, name="root", trace_name="root", is_root_observation=True), + _observation(1, name="child", trace_name="root"), + ], + ) + + assert snapshot.name == "root" + assert snapshot.session_id is None + assert snapshot.public is False + + +def test_wait_for_scores_follows_cursor_and_polls(monkeypatch): + attempts = {"count": 0} + + def get_many_v3(**kwargs): + attempts["count"] += 1 + if attempts["count"] < 2: + return _page([]) + if kwargs["cursor"] is None: + return _page(["score-1"], cursor="next") + return _page(["score-2"]) + + _install_client(monkeypatch, scores_v3=SimpleNamespace(get_many_v3=get_many_v3)) + + scores = wait_for_scores(min_count=2, trace_id="trace-123") + + assert scores == ["score-1", "score-2"] def test_retry_until_ready_clears_stale_error_after_success(monkeypatch): @@ -171,3 +287,37 @@ def operation(): assert trace["id"] == "trace-123" assert trace["attempt"] == 3 + + +def test_normalize_observation_keeps_scalar_text_io_as_strings(): + observation = normalize_observation( + ObservationV2( + id="obs", + trace_id="trace", + start_time=datetime(2026, 1, 1, tzinfo=timezone.utc), + project_id="project", + type="SPAN", + input="2", + output="true", + ) + ) + + assert observation.input == "2" + assert observation.output == "true" + + +def test_get_observations_keeps_the_latest_row_per_observation_id(monkeypatch): + stale = _observation(0, name="stale", updated_at=START) + fresh = _observation(0, name="fresh", updated_at=START + timedelta(seconds=1)) + + def get_many(**kwargs): + return _page([fresh, stale, _observation(1)]) + + _install_client(monkeypatch, observations=SimpleNamespace(get_many=get_many)) + + observations = get_observations(trace_id="trace-123") + + assert [(o.id, o.name) for o in observations] == [ + ("obs-0", "fresh"), + ("obs-1", None), + ] diff --git a/tests/unit/test_mask_otel_spans.py b/tests/unit/test_mask_otel_spans.py index 5d1d52ee4..d9a23befb 100644 --- a/tests/unit/test_mask_otel_spans.py +++ b/tests/unit/test_mask_otel_spans.py @@ -293,7 +293,6 @@ def test_export_stage_media_processes_string_sequence_attributes(): @pytest.mark.parametrize( ("attribute_key", "expected_field"), [ - ("langfuse.trace.input", "input"), ("langfuse.observation.input", "input"), ("ai.prompt.messages", "input"), ("gcp.vertex.agent.tool_call_args", "input"), @@ -302,7 +301,6 @@ def test_export_stage_media_processes_string_sequence_attributes(): ("gen_ai.input.messages", "input"), ("gen_ai.prompt.0.content", "input"), ("llm.input_messages.0.message.content", "input"), - ("langfuse.trace.output", "output"), ("langfuse.observation.output", "output"), ("ai.response.toolCalls", "output"), ("gcp.vertex.agent.tool_response", "output"), diff --git a/tests/unit/test_otel.py b/tests/unit/test_otel.py index ccdec39fd..6c4b23d3e 100644 --- a/tests/unit/test_otel.py +++ b/tests/unit/test_otel.py @@ -498,8 +498,8 @@ def test_generation_name_update(self, langfuse_client, memory_exporter): def test_trace_update(self, langfuse_client, memory_exporter): """Test updating trace level attributes.""" - # Create a span and set trace attributes using propagate_attributes and set_trace_io - with langfuse_client.start_as_current_observation(name="trace-span") as span: + # Create a span and set trace attributes using propagate_attributes + with langfuse_client.start_as_current_observation(name="trace-span"): with propagate_attributes( trace_name="updated-trace-name", user_id="test-user", @@ -507,7 +507,7 @@ def test_trace_update(self, langfuse_client, memory_exporter): tags=["tag1", "tag2"], metadata={"trace-meta": "data"}, ): - span.set_trace_io(input={"trace-input": "value"}) + pass # Get the span data spans = self.get_spans_by_name(memory_exporter, "trace-span") @@ -526,12 +526,10 @@ def test_trace_update(self, langfuse_client, memory_exporter): else: tags = list(attributes[LangfuseOtelSpanAttributes.TRACE_TAGS]) - input_data = json.loads(attributes[LangfuseOtelSpanAttributes.TRACE_INPUT]) metadata = attributes[f"{LangfuseOtelSpanAttributes.TRACE_METADATA}.trace-meta"] # Check attribute values assert sorted(tags) == sorted(["tag1", "tag2"]) - assert input_data == {"trace-input": "value"} assert metadata == "data" def test_complex_scenario(self, langfuse_client, memory_exporter): diff --git a/tests/unit/test_resource_manager.py b/tests/unit/test_resource_manager.py index f66a1e052..f48e61555 100644 --- a/tests/unit/test_resource_manager.py +++ b/tests/unit/test_resource_manager.py @@ -1,5 +1,6 @@ """Test the LangfuseResourceManager and get_client() function.""" +import logging from queue import Queue from types import SimpleNamespace from typing import Sequence @@ -410,6 +411,36 @@ def test_at_fork_reinit_new_httpx_client_uses_configured_timeout_and_headers( client.shutdown() +def test_create_score_with_invalid_input_logs_error_without_enqueueing( + monkeypatch, caplog +): + monkeypatch.setenv("LANGFUSE_MEDIA_UPLOAD_ENABLED", "false") + + with LangfuseResourceManager._lock: + LangfuseResourceManager._instances.clear() + + client = Langfuse( + public_key="pk-invalid-score", + secret_key="sk-invalid-score", + span_exporter=NoOpSpanExporter(), + ) + rm = client._resources + assert rm is not None + enqueued = [] + monkeypatch.setattr(rm, "add_score_task", lambda event, **_: enqueued.append(event)) + + with caplog.at_level(logging.ERROR, logger="langfuse"): + client.create_score(name="invalid", value=object(), trace_id="a" * 32) + + assert enqueued == [] + assert rm._score_ingestion_queue.empty() + error_records = [r for r in caplog.records if r.levelno == logging.ERROR] + assert len(error_records) == 1 + assert "Error creating score" in error_records[0].getMessage() + + client.shutdown() + + def test_stop_and_join_consumer_threads_broadcasts_media_shutdown_after_pausing_all(): events = []