diff --git a/langfuse/_client/constants.py b/langfuse/_client/constants.py index c2d0aa7aa..95968507a 100644 --- a/langfuse/_client/constants.py +++ b/langfuse/_client/constants.py @@ -11,6 +11,8 @@ LANGFUSE_SDK_EXPERIMENT_ENVIRONMENT = "sdk-experiment" +MASK_FALLBACK_VALUE = "" + """Note: this type is used with .__args__ / get_args in some cases and therefore must remain flat""" ObservationTypeGenerationLike: TypeAlias = Literal[ "generation", diff --git a/langfuse/_client/span.py b/langfuse/_client/span.py index 0c1d6db90..21bb9fabb 100644 --- a/langfuse/_client/span.py +++ b/langfuse/_client/span.py @@ -46,6 +46,7 @@ create_trace_attributes, ) from langfuse._client.constants import ( + MASK_FALLBACK_VALUE, ObservationTypeGenerationLike, ObservationTypeLiteral, ObservationTypeLiteralNoEvent, @@ -706,10 +707,17 @@ def _process_media_and_apply_mask( The processed and masked data """ return self._mask_attribute( - data=self._process_media_in_attribute(data=data, field=field) + data=self._process_media_in_attribute(data=data, field=field), field=field ) - def _mask_attribute(self, *, data: Any) -> Any: + def _mask_attribute( + self, + *, + data: Any, + field: Optional[ + Union[Literal["input"], Literal["output"], Literal["metadata"]] + ] = None, + ) -> Any: """Apply the configured mask function to data. Internal method that applies the client's configured masking function to @@ -717,6 +725,7 @@ def _mask_attribute(self, *, data: Any) -> Any: Args: data: The data to mask + field: The attribute the data belongs to Returns: The masked data, or the original data if no mask is configured @@ -733,7 +742,13 @@ def _mask_attribute(self, *, data: Any) -> Any: e, ) - return "" + # Dict metadata is written per key, so mask each key instead of + # writing a plain string to the bare metadata attribute. An empty dict + # has no keys to mask, so it keeps the plain string to stay visible + if field == "metadata" and isinstance(data, dict) and data: + return {key: MASK_FALLBACK_VALUE for key in data} + + return MASK_FALLBACK_VALUE def _process_media_in_attribute( self, diff --git a/tests/unit/test_otel.py b/tests/unit/test_otel.py index 6c4b23d3e..286a5e4d0 100644 --- a/tests/unit/test_otel.py +++ b/tests/unit/test_otel.py @@ -1980,6 +1980,108 @@ def update_random_metadata(thread_id): assert "version" in system_data assert "features" in system_data + MASK_FALLBACK = "" + + def get_metadata_attributes(self, span_data: dict) -> dict: + prefix = LangfuseOtelSpanAttributes.OBSERVATION_METADATA + return { + key: value + for key, value in span_data["attributes"].items() + if key == prefix or key.startswith(f"{prefix}.") + } + + def test_failed_mask_on_update_masks_each_metadata_key( + self, configurable_langfuse_client, memory_exporter + ): + def mask(*, data, **kwargs): + if isinstance(data, dict) and "secret" in data: + raise ValueError("mask failed") + return data + + langfuse_client = configurable_langfuse_client(mask=mask) + span = langfuse_client.start_observation(name="mask-fail", metadata={"a": 1}) + span.update(metadata={"secret": "pw", "b": 2}) + span.end() + + span_data = self.get_spans_by_name(memory_exporter, "mask-fail")[0] + prefix = LangfuseOtelSpanAttributes.OBSERVATION_METADATA + assert self.get_metadata_attributes(span_data) == { + f"{prefix}.a": 1, + f"{prefix}.secret": self.MASK_FALLBACK, + f"{prefix}.b": self.MASK_FALLBACK, + } + + def test_failed_mask_on_start_masks_each_metadata_key( + self, configurable_langfuse_client, memory_exporter + ): + def mask(*, data, **kwargs): + if isinstance(data, dict): + raise ValueError("mask failed") + return data + + langfuse_client = configurable_langfuse_client(mask=mask) + span = langfuse_client.start_observation( + name="mask-fail-start", metadata={"secret": "pw", "b": 2} + ) + span.end() + + span_data = self.get_spans_by_name(memory_exporter, "mask-fail-start")[0] + prefix = LangfuseOtelSpanAttributes.OBSERVATION_METADATA + assert self.get_metadata_attributes(span_data) == { + f"{prefix}.secret": self.MASK_FALLBACK, + f"{prefix}.b": self.MASK_FALLBACK, + } + + def test_failed_mask_keeps_string_fallback_for_empty_dict_metadata( + self, configurable_langfuse_client, memory_exporter + ): + def mask(*, data, **kwargs): + if isinstance(data, dict): + raise ValueError("mask failed") + return data + + langfuse_client = configurable_langfuse_client(mask=mask) + span = langfuse_client.start_observation( + name="mask-fail-empty-dict", metadata={} + ) + span.end() + + span_data = self.get_spans_by_name(memory_exporter, "mask-fail-empty-dict")[0] + assert self.get_metadata_attributes(span_data) == { + LangfuseOtelSpanAttributes.OBSERVATION_METADATA: self.MASK_FALLBACK, + } + + def test_failed_mask_keeps_string_fallback_for_non_dict_values( + self, configurable_langfuse_client, memory_exporter + ): + def mask(*, data, **kwargs): + if data is not None: + raise ValueError("mask failed") + return data + + langfuse_client = configurable_langfuse_client(mask=mask) + span = langfuse_client.start_observation( + name="mask-fail-non-dict", + input={"secret": "pw"}, + output={"secret": "pw"}, + metadata="plain-string", + ) + span.end() + + span_data = self.get_spans_by_name(memory_exporter, "mask-fail-non-dict")[0] + attributes = span_data["attributes"] + assert self.get_metadata_attributes(span_data) == { + LangfuseOtelSpanAttributes.OBSERVATION_METADATA: self.MASK_FALLBACK, + } + assert ( + attributes[LangfuseOtelSpanAttributes.OBSERVATION_INPUT] + == self.MASK_FALLBACK + ) + assert ( + attributes[LangfuseOtelSpanAttributes.OBSERVATION_OUTPUT] + == self.MASK_FALLBACK + ) + class TestMultiProjectSetup(TestOTelBase): """Tests for multi-project setup within the same process.