diff --git a/README.md b/README.md index efc6c3f..fd235d8 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,35 @@ Issues and feature requests are tracked in a variety of places. To report this f | Portal | User Interface or Experience Issue | [File an Issue](https://github.com/azure/azure-functions-ux/issues) | | Templates | Code Issues with Creation Template | [File an Issue](https://github.com/Azure/azure-functions-templates/issues) | +## Durable Functions JSON serialization + +`azure.functions._durable_functions.df_dumps(value, *, ensure_ascii=True)` +preserves legacy ASCII-escaped JSON by default. Pass `ensure_ascii=False` to +opt in to emitting ordinary Unicode literally in either loose or strict typing +mode. Surrogate code units, including lone surrogates and explicit surrogate-pair +strings, remain JSON-escaped so the output is safe to encode as UTF-8 and pass +through protobuf string fields. ASCII DEL also retains its legacy `\u007f` +escape. Unicode is not normalized. ASCII output, whitespace, custom-object +envelopes, and strict/loose type rules are unchanged; `df_loads` reads both +escaped and literal-Unicode payloads. + +For the synthetic Python value `{"text": "\u0416" * 100000}` (100,000 Cyrillic +characters), `df_dumps(value, ensure_ascii=False)` produces **200,012 UTF-8 +bytes**, compared with **600,012 bytes** using the default or explicit +`ensure_ascii=True`. This isolates escaping overhead; it does not measure a +complete production request or remove aggregate transport limits. + +**Opt-in compatibility:** Serialized bytes change for ordinary non-ASCII text +only when selecting `ensure_ascii=False`, even though escaped and literal JSON +decode to the same values. Applications deriving hashes, signatures, IDs, or +replay decisions from serialized JSON must review their byte-dependent logic +before opting in or switching back. Existing callers, including the activity +trigger converter, continue to use ASCII-escaped output. Literal-Unicode output +has not been validated against live hosted backends; provider integration/replay +coverage and real-host staging validation are tracked in +[microsoft/durabletask-python#291](https://github.com/microsoft/durabletask-python/issues/291), +linked from [#371](https://github.com/Azure/azure-functions-python-library/issues/371). + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/azure/functions/_durable_functions.py b/azure/functions/_durable_functions.py index 393c7b5..42c9c90 100644 --- a/azure/functions/_durable_functions.py +++ b/azure/functions/_durable_functions.py @@ -4,6 +4,7 @@ import json import logging import os +import re import sys import warnings from typing import Any, Callable, Optional, Union @@ -163,12 +164,21 @@ def _deserialize_custom_object(obj: dict) -> object: # --------------------------------------------------------------------------- -def df_dumps(value: Any) -> str: +def _escape_json_code_units(json_string: str) -> str: + """Preserve DEL's ASCII representation and UTF-8-safe surrogate escapes.""" + return re.sub( + r"[\x7f\ud800-\udfff]", + lambda match: f"\\u{ord(match.group()):04x}", + json_string, + ) + + +def df_dumps(value: Any, *, ensure_ascii: bool = True) -> str: """Serialize *value* to a JSON string. In **loose mode** (default) this is equivalent to - ``json.dumps(value, default=_serialize_custom_object)``: nested - custom objects are wrapped recursively in the + ``json.dumps(value, default=_serialize_custom_object)``. Nested custom + objects are wrapped recursively in the ``{"__class__", "__module__", "__data__"}`` envelope. In **strict mode** (``AZURE_FUNCTIONS_DURABLE_STRICT_TYPING`` set @@ -177,14 +187,26 @@ def df_dumps(value: Any) -> str: without a ``default=`` hook. ``to_json()`` must therefore return a value that is natively JSON-serializable, and ``TypeError`` is raised if any nested value is not. + + The keyword-only ``ensure_ascii`` option defaults to ``True``, preserving + legacy ASCII-escaped output. Set it to ``False`` to emit ordinary Unicode + literally, without normalization. DEL and surrogate code units (including + explicit surrogate pairs) retain their JSON escapes for ASCII compatibility + and UTF-8/protobuf transport safety. JSON whitespace is unchanged. + Opting in changes non-ASCII serialized bytes; byte-dependent hashes, + signatures, IDs and replay logic must account for this change. """ if _is_strict_mode(): if hasattr(value, "to_json"): envelope = _serialize_custom_object(value) - return json.dumps(envelope) - # Primitive / plain-JSON value -- serialize without default=. - return json.dumps(value) - return json.dumps(value, default=_serialize_custom_object) + result = json.dumps(envelope, ensure_ascii=ensure_ascii) + else: + # Primitive / plain-JSON value -- serialize without default=. + result = json.dumps(value, ensure_ascii=ensure_ascii) + else: + result = json.dumps( + value, default=_serialize_custom_object, ensure_ascii=ensure_ascii) + return result if ensure_ascii else _escape_json_code_units(result) def df_loads(s: str, expected_type: Optional[type] = None) -> Any: diff --git a/spec-functions-sdk-df-serialization.md b/spec-functions-sdk-df-serialization.md index 6c059db..d908d02 100644 --- a/spec-functions-sdk-df-serialization.md +++ b/spec-functions-sdk-df-serialization.md @@ -40,11 +40,13 @@ Problems: Add two public functions to `azure/functions/_durable_functions.py`: ```python -def df_dumps(value: Any) -> str: +def df_dumps(value: Any, *, ensure_ascii: bool = True) -> str: """Serialize *value* to a JSON string using the Durable Functions convention. In loose mode (default), equivalent to - ``json.dumps(value, default=_serialize_custom_object)``. + ``json.dumps(value, default=_serialize_custom_object)`` by default. + The keyword-only ensure_ascii=False option emits ordinary Unicode literally, + but DEL and surrogate code units retain their legacy JSON escapes. In strict mode, the top-level custom object is wrapped in the legacy envelope but its ``to_json()`` output is serialized as @@ -78,12 +80,24 @@ def df_loads( ### 2.1 Wire format — Loose mode -**No change.** `df_dumps` produces the same JSON that -`json.dumps(value, default=_serialize_custom_object)` produces today: +`df_dumps` preserves the JSON structure produced by +`json.dumps(value, default=_serialize_custom_object)`, including its exact +ASCII-escaped output by default and with explicit `ensure_ascii=True`: - Builtins → plain JSON - Custom objects with `to_json` → `{"__class__": ..., "__module__": ..., "__data__": ...}` - Nested custom objects are recursively wrapped via the `default=` hook +The keyword-only `ensure_ascii=False` option emits ordinary Unicode literally +while retaining DEL and surrogate escapes. ASCII output and whitespace/separators +are unchanged. Both options apply to strict-mode primitives and custom-object +envelopes. `df_loads` reads escaped and literal-Unicode payloads, and Unicode +is not normalized. Selecting literal output changes non-ASCII serialized bytes; +applications using JSON bytes for hashes, signatures, IDs, or replay decisions +must review compatibility before opting in or switching back. Existing activity +trigger converters continue to use the default ASCII-escaped output. +See [Durable Functions JSON serialization](README.md#durable-functions-json-serialization) +for the synthetic UTF-8 size example and rollout-validation caveats. + ### 2.1.1 Wire format — Strict mode `df_dumps` in strict mode: diff --git a/tests/test_durable_functions_codec.py b/tests/test_durable_functions_codec.py index dbff96a..18b2df5 100644 --- a/tests/test_durable_functions_codec.py +++ b/tests/test_durable_functions_codec.py @@ -17,6 +17,7 @@ from azure.functions._durable_functions import ( _STRICT_ENV_VAR, _get_serialize_default, + _serialize_custom_object, df_dumps, df_loads, ) @@ -173,6 +174,14 @@ def loose(monkeypatch): monkeypatch.delenv(_STRICT_ENV_VAR, raising=False) +@pytest.fixture(params=["loose", "strict"]) +def codec_mode(request, monkeypatch): + """Run wire-format checks in both typing modes.""" + monkeypatch.setenv( + _STRICT_ENV_VAR, "1" if request.param == "strict" else "0") + return request.param + + # =================================================================== # 1. PRIMITIVES (str, int, float, bool, None, list, dict) # =================================================================== @@ -528,8 +537,11 @@ def test_loose_mode_custom_object_still_works(self, loose): class TestWireFormat: + def test_ensure_ascii_is_keyword_only(self, codec_mode): + with pytest.raises(TypeError, match="positional"): + df_dumps("\u0416", False) + def test_df_dumps_matches_legacy_json_dumps(self): - from azure.functions._durable_functions import _serialize_custom_object value = {"key": "value", "list": [1, 2, 3]} assert df_dumps(value) == json.dumps(value, default=_serialize_custom_object) @@ -557,6 +569,173 @@ def test_nested_object_produces_plain_json_data(self): assert raw["__data__"] == {"item": "gadget", "hat": {"color": "blue"}} +@pytest.mark.parametrize("ensure_ascii", [True, False]) +class TestUnicodeWireFormat: + """Opt in to compact Unicode without changing legacy default behavior.""" + + @pytest.mark.parametrize("value", [ + None, + True, + False, + 42, + -3.14, + "", + pytest.param("".join(chr(code) for code in range(128)), id="all-ascii"), + [1, None, {"text": "quotes \" and backslashes \\"}], + {1: "one", "tuple": (True, False)}, + PlainPerson("andy", 99), + ScalarPerson("andy"), + SmartOrder("gadget", Hat("blue")), + pytest.param(PlainPerson("".join(chr(code) for code in range(128)), 99), + id="custom-all-ascii"), + pytest.param(ScalarPerson("".join(chr(code) for code in range(128))), + id="scalar-all-ascii"), + ]) + def test_ascii_output_is_unchanged(self, codec_mode, ensure_ascii, value): + legacy = json.dumps(value, default=_serialize_custom_object) + assert df_dumps(value).encode("utf-8") == legacy.encode("utf-8") + assert df_dumps(value, ensure_ascii=ensure_ascii).encode( + "utf-8") == legacy.encode("utf-8") + + @pytest.mark.parametrize("text", [ + pytest.param("\u041f\u0440\u0438\u0432\u0435\u0442", id="cyrillic"), + pytest.param("\u4f60\u597d", id="chinese"), + pytest.param("\u65e5\u672c\u8a9e", id="japanese"), + pytest.param("\u0645\u0631\u062d\u0628\u0627", id="arabic"), + pytest.param("\u0939\u093f\u0928\u094d\u0926\u0940", id="devanagari"), + pytest.param("e\u0301", id="combining"), + pytest.param("\u00e9", id="precomposed"), + pytest.param("\U0001f600\U00020000", id="non-bmp"), + pytest.param("\u2028\u2029", id="line-separators"), + ]) + def test_plain_unicode_respects_option(self, codec_mode, ensure_ascii, text): + for value in [text, {text: ["nested", {"text": text}]}]: + assert df_dumps(value) == json.dumps(value) + encoded = df_dumps(value, ensure_ascii=ensure_ascii) + assert encoded == json.dumps(value, ensure_ascii=ensure_ascii) + if not ensure_ascii: + assert text in encoded + assert df_loads(encoded, expected_type=type(value)) == value + assert encoded.encode("utf-8").decode("utf-8") == encoded + + @pytest.mark.parametrize("value", [ + PlainPerson("\u0416\u4f60\U0001f600", 99), + ScalarPerson("e\u0301\u0645\u0939"), + SmartOrder("\u65e5", Hat("\u0416")), + ]) + def test_custom_envelope_respects_option( + self, codec_mode, ensure_ascii, value): + envelope = _serialize_custom_object(value) + assert df_dumps(value) == json.dumps(envelope) + encoded = df_dumps(value, ensure_ascii=ensure_ascii) + assert encoded == json.dumps(envelope, ensure_ascii=ensure_ascii) + assert json.loads(encoded) == envelope + assert df_loads(encoded, expected_type=type(value)) == value + + @pytest.mark.parametrize("value", [ + NaiveOrder("\u0416", Hat("\u4f60")), + {"\u0416": PlainPerson("\u4f60", 99)}, + [ScalarPerson("\U0001f600")], + ]) + def test_nested_custom_object_rules_are_unchanged( + self, codec_mode, ensure_ascii, value): + if codec_mode == "strict": + with pytest.raises(TypeError): + df_dumps(value, ensure_ascii=ensure_ascii) + else: + assert df_dumps(value) == json.dumps( + value, default=_serialize_custom_object) + encoded = df_dumps(value, ensure_ascii=ensure_ascii) + assert encoded == json.dumps( + value, default=_serialize_custom_object, + ensure_ascii=ensure_ascii) + assert df_loads(encoded, expected_type=type(value)) == value + assert df_loads(encoded) == value + + def test_controls_quotes_and_backslashes_remain_escaped( + self, codec_mode, ensure_ascii): + text = "\u0416" + "".join(chr(code) for code in range(32)) + text += '"\\ and literal escapes \\u0416 \\ud800 \\n' + assert df_dumps(text) == json.dumps(text) + encoded = df_dumps(text, ensure_ascii=ensure_ascii) + assert encoded == json.dumps(text, ensure_ascii=ensure_ascii) + assert "\\u0000" in encoded + assert '\\"' in encoded + assert "\\\\" in encoded + assert not any(chr(code) in encoded for code in range(32)) + assert df_loads(encoded, expected_type=str) == text + + @pytest.mark.parametrize("value", [ + {"\u0416": ["\u4f60", "e\u0301", "\U0001f600"]}, + PlainPerson("\u0416", 99), + ScalarPerson("\U0001f600"), + SmartOrder("\u0416", Hat("\u4f60")), + ]) + def test_legacy_escaped_payloads_remain_readable( + self, codec_mode, ensure_ascii, value): + legacy = json.dumps(value, default=_serialize_custom_object) + assert "\\u" in legacy + assert df_dumps(value) == legacy + assert df_loads(legacy, expected_type=type(value)) == value + assert df_loads( + df_dumps(value, ensure_ascii=ensure_ascii), + expected_type=type(value)) == value + if codec_mode == "loose" or isinstance(value, dict): + assert df_loads(legacy) == value + + @pytest.mark.parametrize("text", [ + pytest.param("\ud800", id="first-high-surrogate"), + pytest.param("\udbff", id="last-high-surrogate"), + pytest.param("\udc00", id="first-low-surrogate"), + pytest.param("\udfff", id="last-low-surrogate"), + pytest.param("\ud83d\ude00", id="explicit-surrogate-pair"), + pytest.param("\udc00\ud800", id="reversed-surrogates"), + pytest.param("\ud800x\udfff", id="separated-surrogates"), + ]) + @pytest.mark.parametrize("custom", [False, True], ids=["plain", "custom"]) + def test_surrogates_remain_transport_safe( + self, codec_mode, ensure_ascii, text, custom): + value = ScalarPerson(text) if custom else {text: [text]} + legacy = json.dumps(value, default=_serialize_custom_object) + assert df_dumps(value) == legacy + encoded = df_dumps(value, ensure_ascii=ensure_ascii) + assert encoded == legacy + assert encoded.encode("utf-8").decode("utf-8") == encoded + # json.loads combines escaped surrogate pairs, as it did before. + assert df_loads(encoded, expected_type=type(value)) == df_loads( + legacy, expected_type=type(value)) + + @pytest.mark.parametrize("custom", [False, True], ids=["plain", "custom"]) + def test_unicode_and_surrogates_can_coexist( + self, codec_mode, ensure_ascii, custom): + text = "\u0416\ud800\U0001f600\udfff\\ud800\x7f" + value = ScalarPerson(text) if custom else {text: [text]} + legacy = json.dumps(value, default=_serialize_custom_object) + assert df_dumps(value) == legacy + encoded = df_dumps(value, ensure_ascii=ensure_ascii) + if ensure_ascii: + assert encoded == legacy + else: + assert "\u0416" in encoded + assert "\U0001f600" in encoded + assert "\\ud800" in encoded + assert "\\udfff" in encoded + assert "\\\\ud800" in encoded + assert "\\u007f" in encoded + assert encoded.encode("utf-8").decode("utf-8") == encoded + assert df_loads(encoded, expected_type=type(value)) == value + + def test_cyrillic_payload_utf8_size(self, codec_mode, ensure_ascii): + value = {"text": "\u0416" * 100000} + legacy = json.dumps(value) + assert df_dumps(value) == legacy + encoded = df_dumps(value, ensure_ascii=ensure_ascii) + assert len(legacy.encode("utf-8")) == 600012 + assert len(encoded.encode("utf-8")) == ( + 600012 if ensure_ascii else 200012) + assert df_loads(encoded, expected_type=dict) == value + + # =================================================================== # 12. _get_serialize_default # =================================================================== diff --git a/tests/test_durable_functions_serialization.py b/tests/test_durable_functions_serialization.py index 62f8f18..3257548 100644 --- a/tests/test_durable_functions_serialization.py +++ b/tests/test_durable_functions_serialization.py @@ -358,6 +358,29 @@ def test_encode_uses_df_dumps(self): ActivityTriggerConverter.encode({"x": 1}, expected_type=None) spy.assert_called_once_with({"x": 1}) + def test_unicode_output_stays_ascii_escaped_in_both_modes(self): + value = { + "\u4f60": [ + "\u0416", "e\u0301", "\U0001f600", + "\ud800", "\ud83d\ude00", "\\ud800", + ], + } + for mode in ["0", "1"]: + with self.subTest(mode=mode), _strict_env(mode): + datum = ActivityTriggerConverter.encode( + value, expected_type=dict) + self.assertEqual(datum.type, "json") + self.assertEqual(datum.value, json.dumps(value)) + self.assertIn("\\u0416", datum.value) + self.assertIn("\\ud83d\\ude00", datum.value) + self.assertIn("\\ud800", datum.value) + self.assertEqual( + datum.value.encode("utf-8").decode("utf-8"), datum.value) + self.assertEqual( + ActivityTriggerConverter.decode( + datum, trigger_metadata=None), + json.loads(json.dumps(value))) + def test_decode_custom_object_loaded_module(self): s = df_dumps(Hat("green")) datum = Datum(type="json", value=s)