From a7afdec28097ad804f476e5934756c5a816955eb Mon Sep 17 00:00:00 2001 From: Bernd Verst Date: Mon, 12 Oct 2026 14:49:00 -0700 Subject: [PATCH 1/2] fix: reduce Unicode inflation in Durable JSON payloads Emit ordinary Unicode literally in all df_dumps branches while retaining legacy DEL and surrogate escapes. Preserve existing envelopes, typing rules, and decoding compatibility, with focused wire-format and UTF-8 size regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d995f8bb-1172-43c6-a345-6d3b07d8092f --- README.md | 24 +++ azure/functions/_durable_functions.py | 28 +++- spec-functions-sdk-df-serialization.md | 16 +- tests/test_durable_functions_codec.py | 149 +++++++++++++++++- tests/test_durable_functions_serialization.py | 22 +++ 5 files changed, 231 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index efc6c3f4..1d15c7e2 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,30 @@ Issues and feature requests are tracked in a variety of places. To report this f | Portal | User Interface or Experience Issue | [File an Issue](https://github.com/azure/azure-functions-ux/issues) | | Templates | Code Issues with Creation Template | [File an Issue](https://github.com/Azure/azure-functions-templates/issues) | +## Durable Functions JSON serialization + +`azure.functions._durable_functions.df_dumps` emits ordinary Unicode literally +in both loose and strict typing modes instead of expanding it into `\uXXXX` +escapes. Surrogate code units, including lone surrogates and explicit +surrogate-pair strings, remain JSON-escaped so the output is safe to encode as +UTF-8 and pass through protobuf string fields. Unicode is not normalized. +ASCII output, whitespace, custom-object envelopes, and strict/loose type rules +are unchanged; `df_loads` continues to read legacy escaped payloads. + +For the synthetic Python value `{"text": "\u0416" * 100000}` (100,000 Cyrillic +characters), the serialized UTF-8 size is **200,012 bytes**, compared with +**600,012 bytes** when ASCII-escaped. This isolates escaping overhead; it does +not measure a complete production request or remove aggregate transport limits. + +**Upgrade compatibility:** Serialized bytes change for ordinary non-ASCII text, +even though escaped and literal JSON decode to the same values. Applications +deriving hashes, signatures, IDs, or replay decisions from serialized JSON must +review their byte-dependent logic for upgrades and rollbacks. This change has +not been validated against live hosted backends; provider integration/replay +coverage and real-host staging validation are tracked in +[microsoft/durabletask-python#291](https://github.com/microsoft/durabletask-python/issues/291), +linked from [#371](https://github.com/Azure/azure-functions-python-library/issues/371). + ## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a diff --git a/azure/functions/_durable_functions.py b/azure/functions/_durable_functions.py index 393c7b58..4434fcf8 100644 --- a/azure/functions/_durable_functions.py +++ b/azure/functions/_durable_functions.py @@ -4,6 +4,7 @@ import json import logging import os +import re import sys import warnings from typing import Any, Callable, Optional, Union @@ -163,11 +164,21 @@ def _deserialize_custom_object(obj: dict) -> object: # --------------------------------------------------------------------------- +def _escape_json_code_units(json_string: str) -> str: + """Preserve DEL's ASCII representation and UTF-8-safe surrogate escapes.""" + return re.sub( + r"[\x7f\ud800-\udfff]", + lambda match: f"\\u{ord(match.group()):04x}", + json_string, + ) + + def df_dumps(value: Any) -> str: """Serialize *value* to a JSON string. In **loose mode** (default) this is equivalent to - ``json.dumps(value, default=_serialize_custom_object)``: nested + ``json.dumps(value, default=_serialize_custom_object, ensure_ascii=False)`` + except that DEL and surrogate code units remain escaped. Nested custom objects are wrapped recursively in the ``{"__class__", "__module__", "__data__"}`` envelope. @@ -177,14 +188,23 @@ def df_dumps(value: Any) -> str: without a ``default=`` hook. ``to_json()`` must therefore return a value that is natively JSON-serializable, and ``TypeError`` is raised if any nested value is not. + + Ordinary Unicode is emitted literally, without normalization. Surrogate + code units (including explicit surrogate pairs) retain their JSON escapes + for UTF-8/protobuf transport safety. ASCII output and JSON whitespace are + unchanged. Non-ASCII serialized bytes differ from the legacy escaped + representation; byte-dependent hashes, signatures, IDs and replay logic + must account for this change. """ if _is_strict_mode(): if hasattr(value, "to_json"): envelope = _serialize_custom_object(value) - return json.dumps(envelope) + return _escape_json_code_units( + json.dumps(envelope, ensure_ascii=False)) # Primitive / plain-JSON value -- serialize without default=. - return json.dumps(value) - return json.dumps(value, default=_serialize_custom_object) + return _escape_json_code_units(json.dumps(value, ensure_ascii=False)) + return _escape_json_code_units(json.dumps( + value, default=_serialize_custom_object, ensure_ascii=False)) def df_loads(s: str, expected_type: Optional[type] = None) -> Any: diff --git a/spec-functions-sdk-df-serialization.md b/spec-functions-sdk-df-serialization.md index 6c059db2..cff0b453 100644 --- a/spec-functions-sdk-df-serialization.md +++ b/spec-functions-sdk-df-serialization.md @@ -44,7 +44,8 @@ def df_dumps(value: Any) -> str: """Serialize *value* to a JSON string using the Durable Functions convention. In loose mode (default), equivalent to - ``json.dumps(value, default=_serialize_custom_object)``. + ``json.dumps(value, default=_serialize_custom_object, ensure_ascii=False)`` + except that DEL and surrogate code units retain their legacy JSON escapes. In strict mode, the top-level custom object is wrapped in the legacy envelope but its ``to_json()`` output is serialized as @@ -78,12 +79,21 @@ def df_loads( ### 2.1 Wire format — Loose mode -**No change.** `df_dumps` produces the same JSON that -`json.dumps(value, default=_serialize_custom_object)` produces today: +`df_dumps` preserves the JSON structure produced by +`json.dumps(value, default=_serialize_custom_object)`, but emits ordinary +Unicode literally while retaining escapes for surrogate code units: - Builtins → plain JSON - Custom objects with `to_json` → `{"__class__": ..., "__module__": ..., "__data__": ...}` - Nested custom objects are recursively wrapped via the `default=` hook +ASCII output and whitespace/separators are unchanged. This Unicode policy also +applies to strict-mode primitives and custom-object envelopes. `df_loads` +continues to read legacy escaped payloads, and Unicode is not normalized. +Serialized bytes for non-ASCII text change; applications using serialized JSON +for hashes, signatures, IDs, or replay decisions must review upgrade and +rollback compatibility. See [Durable Functions JSON serialization](README.md#durable-functions-json-serialization) +for the synthetic UTF-8 size example and rollout-validation caveats. + ### 2.1.1 Wire format — Strict mode `df_dumps` in strict mode: diff --git a/tests/test_durable_functions_codec.py b/tests/test_durable_functions_codec.py index dbff96a1..6bf03a62 100644 --- a/tests/test_durable_functions_codec.py +++ b/tests/test_durable_functions_codec.py @@ -17,6 +17,7 @@ from azure.functions._durable_functions import ( _STRICT_ENV_VAR, _get_serialize_default, + _serialize_custom_object, df_dumps, df_loads, ) @@ -173,6 +174,14 @@ def loose(monkeypatch): monkeypatch.delenv(_STRICT_ENV_VAR, raising=False) +@pytest.fixture(params=["loose", "strict"]) +def codec_mode(request, monkeypatch): + """Run wire-format checks in both typing modes.""" + monkeypatch.setenv( + _STRICT_ENV_VAR, "1" if request.param == "strict" else "0") + return request.param + + # =================================================================== # 1. PRIMITIVES (str, int, float, bool, None, list, dict) # =================================================================== @@ -529,7 +538,6 @@ def test_loose_mode_custom_object_still_works(self, loose): class TestWireFormat: def test_df_dumps_matches_legacy_json_dumps(self): - from azure.functions._durable_functions import _serialize_custom_object value = {"key": "value", "list": [1, 2, 3]} assert df_dumps(value) == json.dumps(value, default=_serialize_custom_object) @@ -557,6 +565,145 @@ def test_nested_object_produces_plain_json_data(self): assert raw["__data__"] == {"item": "gadget", "hat": {"color": "blue"}} +class TestUnicodeWireFormat: + """Keep Unicode compact without changing JSON or custom-object semantics.""" + + @pytest.mark.parametrize("value", [ + None, + True, + False, + 42, + -3.14, + "", + pytest.param("".join(chr(code) for code in range(128)), id="all-ascii"), + [1, None, {"text": "quotes \" and backslashes \\"}], + {1: "one", "tuple": (True, False)}, + PlainPerson("andy", 99), + ScalarPerson("andy"), + SmartOrder("gadget", Hat("blue")), + pytest.param(PlainPerson("".join(chr(code) for code in range(128)), 99), + id="custom-all-ascii"), + pytest.param(ScalarPerson("".join(chr(code) for code in range(128))), + id="scalar-all-ascii"), + ]) + def test_ascii_output_is_unchanged(self, codec_mode, value): + legacy = json.dumps(value, default=_serialize_custom_object) + assert df_dumps(value).encode("utf-8") == legacy.encode("utf-8") + + @pytest.mark.parametrize("text", [ + pytest.param("\u041f\u0440\u0438\u0432\u0435\u0442", id="cyrillic"), + pytest.param("\u4f60\u597d", id="chinese"), + pytest.param("\u65e5\u672c\u8a9e", id="japanese"), + pytest.param("\u0645\u0631\u062d\u0628\u0627", id="arabic"), + pytest.param("\u0939\u093f\u0928\u094d\u0926\u0940", id="devanagari"), + pytest.param("e\u0301", id="combining"), + pytest.param("\u00e9", id="precomposed"), + pytest.param("\U0001f600\U00020000", id="non-bmp"), + pytest.param("\u2028\u2029", id="line-separators"), + ]) + def test_plain_unicode_is_literal(self, codec_mode, text): + for value in [text, {text: ["nested", {"text": text}]}]: + encoded = df_dumps(value) + assert encoded == json.dumps(value, ensure_ascii=False) + assert text in encoded + assert df_loads(encoded, expected_type=type(value)) == value + assert encoded.encode("utf-8").decode("utf-8") == encoded + + @pytest.mark.parametrize("value", [ + PlainPerson("\u0416\u4f60\U0001f600", 99), + ScalarPerson("e\u0301\u0645\u0939"), + SmartOrder("\u65e5", Hat("\u0416")), + ]) + def test_custom_envelope_preserves_literal_unicode(self, codec_mode, value): + envelope = _serialize_custom_object(value) + encoded = df_dumps(value) + assert encoded == json.dumps(envelope, ensure_ascii=False) + assert json.loads(encoded) == envelope + assert df_loads(encoded, expected_type=type(value)) == value + + @pytest.mark.parametrize("value", [ + NaiveOrder("\u0416", Hat("\u4f60")), + {"\u0416": PlainPerson("\u4f60", 99)}, + [ScalarPerson("\U0001f600")], + ]) + def test_nested_custom_object_rules_are_unchanged(self, codec_mode, value): + if codec_mode == "strict": + with pytest.raises(TypeError): + df_dumps(value) + else: + encoded = df_dumps(value) + assert encoded == json.dumps( + value, default=_serialize_custom_object, ensure_ascii=False) + assert df_loads(encoded, expected_type=type(value)) == value + assert df_loads(encoded) == value + + def test_controls_quotes_and_backslashes_remain_escaped(self, codec_mode): + text = "\u0416" + "".join(chr(code) for code in range(32)) + text += '"\\ and literal escapes \\u0416 \\ud800 \\n' + encoded = df_dumps(text) + assert encoded == json.dumps(text, ensure_ascii=False) + assert "\\u0000" in encoded + assert '\\"' in encoded + assert "\\\\" in encoded + assert not any(chr(code) in encoded for code in range(32)) + assert df_loads(encoded, expected_type=str) == text + + @pytest.mark.parametrize("value", [ + {"\u0416": ["\u4f60", "e\u0301", "\U0001f600"]}, + PlainPerson("\u0416", 99), + ScalarPerson("\U0001f600"), + SmartOrder("\u0416", Hat("\u4f60")), + ]) + def test_legacy_escaped_payloads_remain_readable(self, codec_mode, value): + legacy = json.dumps(value, default=_serialize_custom_object) + assert "\\u" in legacy + assert df_loads(legacy, expected_type=type(value)) == value + if codec_mode == "loose" or isinstance(value, dict): + assert df_loads(legacy) == value + + @pytest.mark.parametrize("text", [ + pytest.param("\ud800", id="first-high-surrogate"), + pytest.param("\udbff", id="last-high-surrogate"), + pytest.param("\udc00", id="first-low-surrogate"), + pytest.param("\udfff", id="last-low-surrogate"), + pytest.param("\ud83d\ude00", id="explicit-surrogate-pair"), + pytest.param("\udc00\ud800", id="reversed-surrogates"), + pytest.param("\ud800x\udfff", id="separated-surrogates"), + ]) + @pytest.mark.parametrize("custom", [False, True], ids=["plain", "custom"]) + def test_surrogates_remain_transport_safe(self, codec_mode, text, custom): + value = ScalarPerson(text) if custom else {text: [text]} + legacy = json.dumps(value, default=_serialize_custom_object) + encoded = df_dumps(value) + assert encoded == legacy + assert encoded.encode("utf-8").decode("utf-8") == encoded + # json.loads combines escaped surrogate pairs, as it did before. + assert df_loads(encoded, expected_type=type(value)) == df_loads( + legacy, expected_type=type(value)) + + @pytest.mark.parametrize("custom", [False, True], ids=["plain", "custom"]) + def test_unicode_and_surrogates_can_coexist(self, codec_mode, custom): + text = "\u0416\ud800\U0001f600\udfff\\ud800\x7f" + value = ScalarPerson(text) if custom else {text: [text]} + encoded = df_dumps(value) + assert "\u0416" in encoded + assert "\U0001f600" in encoded + assert "\\ud800" in encoded + assert "\\udfff" in encoded + assert "\\\\ud800" in encoded + assert "\\u007f" in encoded + assert encoded.encode("utf-8").decode("utf-8") == encoded + assert df_loads(encoded, expected_type=type(value)) == value + + def test_cyrillic_payload_utf8_size(self, codec_mode): + value = {"text": "\u0416" * 100000} + legacy = json.dumps(value) + encoded = df_dumps(value) + assert len(legacy.encode("utf-8")) == 600012 + assert len(encoded.encode("utf-8")) == 200012 + assert df_loads(encoded, expected_type=dict) == value + + # =================================================================== # 12. _get_serialize_default # =================================================================== diff --git a/tests/test_durable_functions_serialization.py b/tests/test_durable_functions_serialization.py index 62f8f189..b9e2f39e 100644 --- a/tests/test_durable_functions_serialization.py +++ b/tests/test_durable_functions_serialization.py @@ -358,6 +358,28 @@ def test_encode_uses_df_dumps(self): ActivityTriggerConverter.encode({"x": 1}, expected_type=None) spy.assert_called_once_with({"x": 1}) + def test_unicode_output_is_transport_safe_in_both_modes(self): + value = { + "\u4f60": [ + "\u0416", "e\u0301", "\U0001f600", + "\ud800", "\ud83d\ude00", "\\ud800", + ], + } + for mode in ["0", "1"]: + with self.subTest(mode=mode), _strict_env(mode): + datum = ActivityTriggerConverter.encode( + value, expected_type=dict) + self.assertEqual(datum.type, "json") + self.assertIn("\u0416", datum.value) + self.assertIn("\U0001f600", datum.value) + self.assertIn("\\ud800", datum.value) + self.assertEqual( + datum.value.encode("utf-8").decode("utf-8"), datum.value) + self.assertEqual( + ActivityTriggerConverter.decode( + datum, trigger_metadata=None), + json.loads(json.dumps(value))) + def test_decode_custom_object_loaded_module(self): s = df_dumps(Hat("green")) datum = Datum(type="json", value=s) From 69e9472af79b73418471647ae45197ee7af6fc89 Mon Sep 17 00:00:00 2001 From: Bernd Verst Date: Mon, 12 Oct 2026 15:02:22 -0700 Subject: [PATCH 2/2] fix: make Unicode Durable JSON serialization opt-in Add keyword-only ensure_ascii=True to retain exact legacy output by default. Keep ensure_ascii=False surrogate-safe and DEL-compatible in every typing branch, and document and test the explicit opt-in behavior. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: d995f8bb-1172-43c6-a345-6d3b07d8092f --- README.md | 37 ++++---- azure/functions/_durable_functions.py | 34 ++++---- spec-functions-sdk-df-serialization.md | 26 +++--- tests/test_durable_functions_codec.py | 84 +++++++++++++------ tests/test_durable_functions_serialization.py | 7 +- 5 files changed, 116 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 1d15c7e2..fd235d85 100644 --- a/README.md +++ b/README.md @@ -47,24 +47,29 @@ Issues and feature requests are tracked in a variety of places. To report this f ## Durable Functions JSON serialization -`azure.functions._durable_functions.df_dumps` emits ordinary Unicode literally -in both loose and strict typing modes instead of expanding it into `\uXXXX` -escapes. Surrogate code units, including lone surrogates and explicit -surrogate-pair strings, remain JSON-escaped so the output is safe to encode as -UTF-8 and pass through protobuf string fields. Unicode is not normalized. -ASCII output, whitespace, custom-object envelopes, and strict/loose type rules -are unchanged; `df_loads` continues to read legacy escaped payloads. +`azure.functions._durable_functions.df_dumps(value, *, ensure_ascii=True)` +preserves legacy ASCII-escaped JSON by default. Pass `ensure_ascii=False` to +opt in to emitting ordinary Unicode literally in either loose or strict typing +mode. Surrogate code units, including lone surrogates and explicit surrogate-pair +strings, remain JSON-escaped so the output is safe to encode as UTF-8 and pass +through protobuf string fields. ASCII DEL also retains its legacy `\u007f` +escape. Unicode is not normalized. ASCII output, whitespace, custom-object +envelopes, and strict/loose type rules are unchanged; `df_loads` reads both +escaped and literal-Unicode payloads. For the synthetic Python value `{"text": "\u0416" * 100000}` (100,000 Cyrillic -characters), the serialized UTF-8 size is **200,012 bytes**, compared with -**600,012 bytes** when ASCII-escaped. This isolates escaping overhead; it does -not measure a complete production request or remove aggregate transport limits. - -**Upgrade compatibility:** Serialized bytes change for ordinary non-ASCII text, -even though escaped and literal JSON decode to the same values. Applications -deriving hashes, signatures, IDs, or replay decisions from serialized JSON must -review their byte-dependent logic for upgrades and rollbacks. This change has -not been validated against live hosted backends; provider integration/replay +characters), `df_dumps(value, ensure_ascii=False)` produces **200,012 UTF-8 +bytes**, compared with **600,012 bytes** using the default or explicit +`ensure_ascii=True`. This isolates escaping overhead; it does not measure a +complete production request or remove aggregate transport limits. + +**Opt-in compatibility:** Serialized bytes change for ordinary non-ASCII text +only when selecting `ensure_ascii=False`, even though escaped and literal JSON +decode to the same values. Applications deriving hashes, signatures, IDs, or +replay decisions from serialized JSON must review their byte-dependent logic +before opting in or switching back. Existing callers, including the activity +trigger converter, continue to use ASCII-escaped output. Literal-Unicode output +has not been validated against live hosted backends; provider integration/replay coverage and real-host staging validation are tracked in [microsoft/durabletask-python#291](https://github.com/microsoft/durabletask-python/issues/291), linked from [#371](https://github.com/Azure/azure-functions-python-library/issues/371). diff --git a/azure/functions/_durable_functions.py b/azure/functions/_durable_functions.py index 4434fcf8..42c9c90d 100644 --- a/azure/functions/_durable_functions.py +++ b/azure/functions/_durable_functions.py @@ -173,13 +173,12 @@ def _escape_json_code_units(json_string: str) -> str: ) -def df_dumps(value: Any) -> str: +def df_dumps(value: Any, *, ensure_ascii: bool = True) -> str: """Serialize *value* to a JSON string. In **loose mode** (default) this is equivalent to - ``json.dumps(value, default=_serialize_custom_object, ensure_ascii=False)`` - except that DEL and surrogate code units remain escaped. Nested - custom objects are wrapped recursively in the + ``json.dumps(value, default=_serialize_custom_object)``. Nested custom + objects are wrapped recursively in the ``{"__class__", "__module__", "__data__"}`` envelope. In **strict mode** (``AZURE_FUNCTIONS_DURABLE_STRICT_TYPING`` set @@ -189,22 +188,25 @@ def df_dumps(value: Any) -> str: a value that is natively JSON-serializable, and ``TypeError`` is raised if any nested value is not. - Ordinary Unicode is emitted literally, without normalization. Surrogate - code units (including explicit surrogate pairs) retain their JSON escapes - for UTF-8/protobuf transport safety. ASCII output and JSON whitespace are - unchanged. Non-ASCII serialized bytes differ from the legacy escaped - representation; byte-dependent hashes, signatures, IDs and replay logic - must account for this change. + The keyword-only ``ensure_ascii`` option defaults to ``True``, preserving + legacy ASCII-escaped output. Set it to ``False`` to emit ordinary Unicode + literally, without normalization. DEL and surrogate code units (including + explicit surrogate pairs) retain their JSON escapes for ASCII compatibility + and UTF-8/protobuf transport safety. JSON whitespace is unchanged. + Opting in changes non-ASCII serialized bytes; byte-dependent hashes, + signatures, IDs and replay logic must account for this change. """ if _is_strict_mode(): if hasattr(value, "to_json"): envelope = _serialize_custom_object(value) - return _escape_json_code_units( - json.dumps(envelope, ensure_ascii=False)) - # Primitive / plain-JSON value -- serialize without default=. - return _escape_json_code_units(json.dumps(value, ensure_ascii=False)) - return _escape_json_code_units(json.dumps( - value, default=_serialize_custom_object, ensure_ascii=False)) + result = json.dumps(envelope, ensure_ascii=ensure_ascii) + else: + # Primitive / plain-JSON value -- serialize without default=. + result = json.dumps(value, ensure_ascii=ensure_ascii) + else: + result = json.dumps( + value, default=_serialize_custom_object, ensure_ascii=ensure_ascii) + return result if ensure_ascii else _escape_json_code_units(result) def df_loads(s: str, expected_type: Optional[type] = None) -> Any: diff --git a/spec-functions-sdk-df-serialization.md b/spec-functions-sdk-df-serialization.md index cff0b453..d908d027 100644 --- a/spec-functions-sdk-df-serialization.md +++ b/spec-functions-sdk-df-serialization.md @@ -40,12 +40,13 @@ Problems: Add two public functions to `azure/functions/_durable_functions.py`: ```python -def df_dumps(value: Any) -> str: +def df_dumps(value: Any, *, ensure_ascii: bool = True) -> str: """Serialize *value* to a JSON string using the Durable Functions convention. In loose mode (default), equivalent to - ``json.dumps(value, default=_serialize_custom_object, ensure_ascii=False)`` - except that DEL and surrogate code units retain their legacy JSON escapes. + ``json.dumps(value, default=_serialize_custom_object)`` by default. + The keyword-only ensure_ascii=False option emits ordinary Unicode literally, + but DEL and surrogate code units retain their legacy JSON escapes. In strict mode, the top-level custom object is wrapped in the legacy envelope but its ``to_json()`` output is serialized as @@ -80,18 +81,21 @@ def df_loads( ### 2.1 Wire format — Loose mode `df_dumps` preserves the JSON structure produced by -`json.dumps(value, default=_serialize_custom_object)`, but emits ordinary -Unicode literally while retaining escapes for surrogate code units: +`json.dumps(value, default=_serialize_custom_object)`, including its exact +ASCII-escaped output by default and with explicit `ensure_ascii=True`: - Builtins → plain JSON - Custom objects with `to_json` → `{"__class__": ..., "__module__": ..., "__data__": ...}` - Nested custom objects are recursively wrapped via the `default=` hook -ASCII output and whitespace/separators are unchanged. This Unicode policy also -applies to strict-mode primitives and custom-object envelopes. `df_loads` -continues to read legacy escaped payloads, and Unicode is not normalized. -Serialized bytes for non-ASCII text change; applications using serialized JSON -for hashes, signatures, IDs, or replay decisions must review upgrade and -rollback compatibility. See [Durable Functions JSON serialization](README.md#durable-functions-json-serialization) +The keyword-only `ensure_ascii=False` option emits ordinary Unicode literally +while retaining DEL and surrogate escapes. ASCII output and whitespace/separators +are unchanged. Both options apply to strict-mode primitives and custom-object +envelopes. `df_loads` reads escaped and literal-Unicode payloads, and Unicode +is not normalized. Selecting literal output changes non-ASCII serialized bytes; +applications using JSON bytes for hashes, signatures, IDs, or replay decisions +must review compatibility before opting in or switching back. Existing activity +trigger converters continue to use the default ASCII-escaped output. +See [Durable Functions JSON serialization](README.md#durable-functions-json-serialization) for the synthetic UTF-8 size example and rollout-validation caveats. ### 2.1.1 Wire format — Strict mode diff --git a/tests/test_durable_functions_codec.py b/tests/test_durable_functions_codec.py index 6bf03a62..18b2df5a 100644 --- a/tests/test_durable_functions_codec.py +++ b/tests/test_durable_functions_codec.py @@ -537,6 +537,10 @@ def test_loose_mode_custom_object_still_works(self, loose): class TestWireFormat: + def test_ensure_ascii_is_keyword_only(self, codec_mode): + with pytest.raises(TypeError, match="positional"): + df_dumps("\u0416", False) + def test_df_dumps_matches_legacy_json_dumps(self): value = {"key": "value", "list": [1, 2, 3]} assert df_dumps(value) == json.dumps(value, default=_serialize_custom_object) @@ -565,8 +569,9 @@ def test_nested_object_produces_plain_json_data(self): assert raw["__data__"] == {"item": "gadget", "hat": {"color": "blue"}} +@pytest.mark.parametrize("ensure_ascii", [True, False]) class TestUnicodeWireFormat: - """Keep Unicode compact without changing JSON or custom-object semantics.""" + """Opt in to compact Unicode without changing legacy default behavior.""" @pytest.mark.parametrize("value", [ None, @@ -586,9 +591,11 @@ class TestUnicodeWireFormat: pytest.param(ScalarPerson("".join(chr(code) for code in range(128))), id="scalar-all-ascii"), ]) - def test_ascii_output_is_unchanged(self, codec_mode, value): + def test_ascii_output_is_unchanged(self, codec_mode, ensure_ascii, value): legacy = json.dumps(value, default=_serialize_custom_object) assert df_dumps(value).encode("utf-8") == legacy.encode("utf-8") + assert df_dumps(value, ensure_ascii=ensure_ascii).encode( + "utf-8") == legacy.encode("utf-8") @pytest.mark.parametrize("text", [ pytest.param("\u041f\u0440\u0438\u0432\u0435\u0442", id="cyrillic"), @@ -601,11 +608,13 @@ def test_ascii_output_is_unchanged(self, codec_mode, value): pytest.param("\U0001f600\U00020000", id="non-bmp"), pytest.param("\u2028\u2029", id="line-separators"), ]) - def test_plain_unicode_is_literal(self, codec_mode, text): + def test_plain_unicode_respects_option(self, codec_mode, ensure_ascii, text): for value in [text, {text: ["nested", {"text": text}]}]: - encoded = df_dumps(value) - assert encoded == json.dumps(value, ensure_ascii=False) - assert text in encoded + assert df_dumps(value) == json.dumps(value) + encoded = df_dumps(value, ensure_ascii=ensure_ascii) + assert encoded == json.dumps(value, ensure_ascii=ensure_ascii) + if not ensure_ascii: + assert text in encoded assert df_loads(encoded, expected_type=type(value)) == value assert encoded.encode("utf-8").decode("utf-8") == encoded @@ -614,10 +623,12 @@ def test_plain_unicode_is_literal(self, codec_mode, text): ScalarPerson("e\u0301\u0645\u0939"), SmartOrder("\u65e5", Hat("\u0416")), ]) - def test_custom_envelope_preserves_literal_unicode(self, codec_mode, value): + def test_custom_envelope_respects_option( + self, codec_mode, ensure_ascii, value): envelope = _serialize_custom_object(value) - encoded = df_dumps(value) - assert encoded == json.dumps(envelope, ensure_ascii=False) + assert df_dumps(value) == json.dumps(envelope) + encoded = df_dumps(value, ensure_ascii=ensure_ascii) + assert encoded == json.dumps(envelope, ensure_ascii=ensure_ascii) assert json.loads(encoded) == envelope assert df_loads(encoded, expected_type=type(value)) == value @@ -626,22 +637,28 @@ def test_custom_envelope_preserves_literal_unicode(self, codec_mode, value): {"\u0416": PlainPerson("\u4f60", 99)}, [ScalarPerson("\U0001f600")], ]) - def test_nested_custom_object_rules_are_unchanged(self, codec_mode, value): + def test_nested_custom_object_rules_are_unchanged( + self, codec_mode, ensure_ascii, value): if codec_mode == "strict": with pytest.raises(TypeError): - df_dumps(value) + df_dumps(value, ensure_ascii=ensure_ascii) else: - encoded = df_dumps(value) + assert df_dumps(value) == json.dumps( + value, default=_serialize_custom_object) + encoded = df_dumps(value, ensure_ascii=ensure_ascii) assert encoded == json.dumps( - value, default=_serialize_custom_object, ensure_ascii=False) + value, default=_serialize_custom_object, + ensure_ascii=ensure_ascii) assert df_loads(encoded, expected_type=type(value)) == value assert df_loads(encoded) == value - def test_controls_quotes_and_backslashes_remain_escaped(self, codec_mode): + def test_controls_quotes_and_backslashes_remain_escaped( + self, codec_mode, ensure_ascii): text = "\u0416" + "".join(chr(code) for code in range(32)) text += '"\\ and literal escapes \\u0416 \\ud800 \\n' - encoded = df_dumps(text) - assert encoded == json.dumps(text, ensure_ascii=False) + assert df_dumps(text) == json.dumps(text) + encoded = df_dumps(text, ensure_ascii=ensure_ascii) + assert encoded == json.dumps(text, ensure_ascii=ensure_ascii) assert "\\u0000" in encoded assert '\\"' in encoded assert "\\\\" in encoded @@ -654,10 +671,15 @@ def test_controls_quotes_and_backslashes_remain_escaped(self, codec_mode): ScalarPerson("\U0001f600"), SmartOrder("\u0416", Hat("\u4f60")), ]) - def test_legacy_escaped_payloads_remain_readable(self, codec_mode, value): + def test_legacy_escaped_payloads_remain_readable( + self, codec_mode, ensure_ascii, value): legacy = json.dumps(value, default=_serialize_custom_object) assert "\\u" in legacy + assert df_dumps(value) == legacy assert df_loads(legacy, expected_type=type(value)) == value + assert df_loads( + df_dumps(value, ensure_ascii=ensure_ascii), + expected_type=type(value)) == value if codec_mode == "loose" or isinstance(value, dict): assert df_loads(legacy) == value @@ -671,10 +693,12 @@ def test_legacy_escaped_payloads_remain_readable(self, codec_mode, value): pytest.param("\ud800x\udfff", id="separated-surrogates"), ]) @pytest.mark.parametrize("custom", [False, True], ids=["plain", "custom"]) - def test_surrogates_remain_transport_safe(self, codec_mode, text, custom): + def test_surrogates_remain_transport_safe( + self, codec_mode, ensure_ascii, text, custom): value = ScalarPerson(text) if custom else {text: [text]} legacy = json.dumps(value, default=_serialize_custom_object) - encoded = df_dumps(value) + assert df_dumps(value) == legacy + encoded = df_dumps(value, ensure_ascii=ensure_ascii) assert encoded == legacy assert encoded.encode("utf-8").decode("utf-8") == encoded # json.loads combines escaped surrogate pairs, as it did before. @@ -682,12 +706,18 @@ def test_surrogates_remain_transport_safe(self, codec_mode, text, custom): legacy, expected_type=type(value)) @pytest.mark.parametrize("custom", [False, True], ids=["plain", "custom"]) - def test_unicode_and_surrogates_can_coexist(self, codec_mode, custom): + def test_unicode_and_surrogates_can_coexist( + self, codec_mode, ensure_ascii, custom): text = "\u0416\ud800\U0001f600\udfff\\ud800\x7f" value = ScalarPerson(text) if custom else {text: [text]} - encoded = df_dumps(value) - assert "\u0416" in encoded - assert "\U0001f600" in encoded + legacy = json.dumps(value, default=_serialize_custom_object) + assert df_dumps(value) == legacy + encoded = df_dumps(value, ensure_ascii=ensure_ascii) + if ensure_ascii: + assert encoded == legacy + else: + assert "\u0416" in encoded + assert "\U0001f600" in encoded assert "\\ud800" in encoded assert "\\udfff" in encoded assert "\\\\ud800" in encoded @@ -695,12 +725,14 @@ def test_unicode_and_surrogates_can_coexist(self, codec_mode, custom): assert encoded.encode("utf-8").decode("utf-8") == encoded assert df_loads(encoded, expected_type=type(value)) == value - def test_cyrillic_payload_utf8_size(self, codec_mode): + def test_cyrillic_payload_utf8_size(self, codec_mode, ensure_ascii): value = {"text": "\u0416" * 100000} legacy = json.dumps(value) - encoded = df_dumps(value) + assert df_dumps(value) == legacy + encoded = df_dumps(value, ensure_ascii=ensure_ascii) assert len(legacy.encode("utf-8")) == 600012 - assert len(encoded.encode("utf-8")) == 200012 + assert len(encoded.encode("utf-8")) == ( + 600012 if ensure_ascii else 200012) assert df_loads(encoded, expected_type=dict) == value diff --git a/tests/test_durable_functions_serialization.py b/tests/test_durable_functions_serialization.py index b9e2f39e..32575485 100644 --- a/tests/test_durable_functions_serialization.py +++ b/tests/test_durable_functions_serialization.py @@ -358,7 +358,7 @@ def test_encode_uses_df_dumps(self): ActivityTriggerConverter.encode({"x": 1}, expected_type=None) spy.assert_called_once_with({"x": 1}) - def test_unicode_output_is_transport_safe_in_both_modes(self): + def test_unicode_output_stays_ascii_escaped_in_both_modes(self): value = { "\u4f60": [ "\u0416", "e\u0301", "\U0001f600", @@ -370,8 +370,9 @@ def test_unicode_output_is_transport_safe_in_both_modes(self): datum = ActivityTriggerConverter.encode( value, expected_type=dict) self.assertEqual(datum.type, "json") - self.assertIn("\u0416", datum.value) - self.assertIn("\U0001f600", datum.value) + self.assertEqual(datum.value, json.dumps(value)) + self.assertIn("\\u0416", datum.value) + self.assertIn("\\ud83d\\ude00", datum.value) self.assertIn("\\ud800", datum.value) self.assertEqual( datum.value.encode("utf-8").decode("utf-8"), datum.value)