fix: add opt-in Unicode output to Durable JSON serialization

README.md

@@ -45,6 +45,35 @@ Issues and feature requests are tracked in a variety of places. To report this f
 | Portal        | User Interface or Experience Issue           | [File an Issue](https://github.com/azure/azure-functions-ux/issues)            |
 | Templates     | Code Issues with Creation Template           | [File an Issue](https://github.com/Azure/azure-functions-templates/issues)     |
 
+## Durable Functions JSON serialization
+
+`azure.functions._durable_functions.df_dumps(value, *, ensure_ascii=True)`
+preserves legacy ASCII-escaped JSON by default. Pass `ensure_ascii=False` to
+opt in to emitting ordinary Unicode literally in either loose or strict typing
+mode. Surrogate code units, including lone surrogates and explicit surrogate-pair
+strings, remain JSON-escaped so the output is safe to encode as UTF-8 and pass
+through protobuf string fields. ASCII DEL also retains its legacy `\u007f`
+escape. Unicode is not normalized. ASCII output, whitespace, custom-object
+envelopes, and strict/loose type rules are unchanged; `df_loads` reads both
+escaped and literal-Unicode payloads.
+
+For the synthetic Python value `{"text": "\u0416" * 100000}` (100,000 Cyrillic
+characters), `df_dumps(value, ensure_ascii=False)` produces **200,012 UTF-8
+bytes**, compared with **600,012 bytes** using the default or explicit
+`ensure_ascii=True`. This isolates escaping overhead; it does not measure a
+complete production request or remove aggregate transport limits.
+
+**Opt-in compatibility:** Serialized bytes change for ordinary non-ASCII text
+only when selecting `ensure_ascii=False`, even though escaped and literal JSON
+decode to the same values. Applications deriving hashes, signatures, IDs, or
+replay decisions from serialized JSON must review their byte-dependent logic
+before opting in or switching back. Existing callers, including the activity
+trigger converter, continue to use ASCII-escaped output. Literal-Unicode output
+has not been validated against live hosted backends; provider integration/replay
+coverage and real-host staging validation are tracked in
+[microsoft/durabletask-python#291](https://github.com/microsoft/durabletask-python/issues/291),
+linked from [#371](https://github.com/Azure/azure-functions-python-library/issues/371).
+
 ## Contributing
 
 This project welcomes contributions and suggestions.  Most contributions require you to agree to a

azure/functions/_durable_functions.py

@@ -4,6 +4,7 @@
 import json
 import logging
 import os
+import re
 import sys
 import warnings
 from typing import Any, Callable, Optional, Union
@@ -163,12 +164,21 @@ def _deserialize_custom_object(obj: dict) -> object:
 # ---------------------------------------------------------------------------
 
 
-def df_dumps(value: Any) -> str:
+def _escape_json_code_units(json_string: str) -> str:
+    """Preserve DEL's ASCII representation and UTF-8-safe surrogate escapes."""
+    return re.sub(
+        r"[\x7f\ud800-\udfff]",
+        lambda match: f"\\u{ord(match.group()):04x}",
+        json_string,
+    )
+
+
+def df_dumps(value: Any, *, ensure_ascii: bool = True) -> str:
     """Serialize *value* to a JSON string.
 
     In **loose mode** (default) this is equivalent to
-    ``json.dumps(value, default=_serialize_custom_object)``: nested
-    custom objects are wrapped recursively in the
+    ``json.dumps(value, default=_serialize_custom_object)``.  Nested custom
+    objects are wrapped recursively in the
     ``{"__class__", "__module__", "__data__"}`` envelope.
 
     In **strict mode** (``AZURE_FUNCTIONS_DURABLE_STRICT_TYPING`` set
@@ -177,14 +187,26 @@ def df_dumps(value: Any) -> str:
     without a ``default=`` hook.  ``to_json()`` must therefore return
     a value that is natively JSON-serializable, and ``TypeError`` is
     raised if any nested value is not.
+
+    The keyword-only ``ensure_ascii`` option defaults to ``True``, preserving
+    legacy ASCII-escaped output.  Set it to ``False`` to emit ordinary Unicode
+    literally, without normalization.  DEL and surrogate code units (including
+    explicit surrogate pairs) retain their JSON escapes for ASCII compatibility
+    and UTF-8/protobuf transport safety.  JSON whitespace is unchanged.
+    Opting in changes non-ASCII serialized bytes; byte-dependent hashes,
+    signatures, IDs and replay logic must account for this change.
     """
     if _is_strict_mode():
         if hasattr(value, "to_json"):
             envelope = _serialize_custom_object(value)
-            return json.dumps(envelope)
-        # Primitive / plain-JSON value -- serialize without default=.
-        return json.dumps(value)
-    return json.dumps(value, default=_serialize_custom_object)
+            result = json.dumps(envelope, ensure_ascii=ensure_ascii)
+        else:
+            # Primitive / plain-JSON value -- serialize without default=.
+            result = json.dumps(value, ensure_ascii=ensure_ascii)
+    else:
+        result = json.dumps(
+            value, default=_serialize_custom_object, ensure_ascii=ensure_ascii)
+    return result if ensure_ascii else _escape_json_code_units(result)
 
 
 def df_loads(s: str, expected_type: Optional[type] = None) -> Any:

spec-functions-sdk-df-serialization.md

@@ -40,11 +40,13 @@ Problems:
 Add two public functions to `azure/functions/_durable_functions.py`:
 
 ```python
-def df_dumps(value: Any) -> str:
+def df_dumps(value: Any, *, ensure_ascii: bool = True) -> str:
     """Serialize *value* to a JSON string using the Durable Functions convention.
 
     In loose mode (default), equivalent to
-    ``json.dumps(value, default=_serialize_custom_object)``.
+    ``json.dumps(value, default=_serialize_custom_object)`` by default.
+    The keyword-only ensure_ascii=False option emits ordinary Unicode literally,
+    but DEL and surrogate code units retain their legacy JSON escapes.
 
     In strict mode, the top-level custom object is wrapped in the
     legacy envelope but its ``to_json()`` output is serialized as
@@ -78,12 +80,24 @@ def df_loads(
 
 ### 2.1 Wire format — Loose mode
 
-**No change.** `df_dumps` produces the same JSON that
-`json.dumps(value, default=_serialize_custom_object)` produces today:
+`df_dumps` preserves the JSON structure produced by
+`json.dumps(value, default=_serialize_custom_object)`, including its exact
+ASCII-escaped output by default and with explicit `ensure_ascii=True`:
 - Builtins → plain JSON
 - Custom objects with `to_json` → `{"__class__": ..., "__module__": ..., "__data__": ...}`
 - Nested custom objects are recursively wrapped via the `default=` hook
 
+The keyword-only `ensure_ascii=False` option emits ordinary Unicode literally
+while retaining DEL and surrogate escapes. ASCII output and whitespace/separators
+are unchanged. Both options apply to strict-mode primitives and custom-object
+envelopes. `df_loads` reads escaped and literal-Unicode payloads, and Unicode
+is not normalized. Selecting literal output changes non-ASCII serialized bytes;
+applications using JSON bytes for hashes, signatures, IDs, or replay decisions
+must review compatibility before opting in or switching back. Existing activity
+trigger converters continue to use the default ASCII-escaped output.
+See [Durable Functions JSON serialization](README.md#durable-functions-json-serialization)
+for the synthetic UTF-8 size example and rollout-validation caveats.
+
 ### 2.1.1 Wire format — Strict mode
 
 `df_dumps` in strict mode:

tests/test_durable_functions_codec.py

@@ -17,6 +17,7 @@
 from azure.functions._durable_functions import (
     _STRICT_ENV_VAR,
     _get_serialize_default,
+    _serialize_custom_object,
     df_dumps,
     df_loads,
 )
@@ -173,6 +174,14 @@ def loose(monkeypatch):
     monkeypatch.delenv(_STRICT_ENV_VAR, raising=False)
 
 
+@pytest.fixture(params=["loose", "strict"])
+def codec_mode(request, monkeypatch):
+    """Run wire-format checks in both typing modes."""
+    monkeypatch.setenv(
+        _STRICT_ENV_VAR, "1" if request.param == "strict" else "0")
+    return request.param
+
+
 # ===================================================================
 # 1. PRIMITIVES  (str, int, float, bool, None, list, dict)
 # ===================================================================
@@ -528,8 +537,11 @@ def test_loose_mode_custom_object_still_works(self, loose):
 
 class TestWireFormat:
 
+    def test_ensure_ascii_is_keyword_only(self, codec_mode):
+        with pytest.raises(TypeError, match="positional"):
+            df_dumps("\u0416", False)
+
     def test_df_dumps_matches_legacy_json_dumps(self):
-        from azure.functions._durable_functions import _serialize_custom_object
         value = {"key": "value", "list": [1, 2, 3]}
         assert df_dumps(value) == json.dumps(value, default=_serialize_custom_object)
 
@@ -557,6 +569,173 @@ def test_nested_object_produces_plain_json_data(self):
         assert raw["__data__"] == {"item": "gadget", "hat": {"color": "blue"}}
 
 
+@pytest.mark.parametrize("ensure_ascii", [True, False])
+class TestUnicodeWireFormat:
+    """Opt in to compact Unicode without changing legacy default behavior."""
+
+    @pytest.mark.parametrize("value", [
+        None,
+        True,
+        False,
+        42,
+        -3.14,
+        "",
+        pytest.param("".join(chr(code) for code in range(128)), id="all-ascii"),
+        [1, None, {"text": "quotes \" and backslashes \\"}],
+        {1: "one", "tuple": (True, False)},
+        PlainPerson("andy", 99),
+        ScalarPerson("andy"),
+        SmartOrder("gadget", Hat("blue")),
+        pytest.param(PlainPerson("".join(chr(code) for code in range(128)), 99),
+                     id="custom-all-ascii"),
+        pytest.param(ScalarPerson("".join(chr(code) for code in range(128))),
+                     id="scalar-all-ascii"),
+    ])
+    def test_ascii_output_is_unchanged(self, codec_mode, ensure_ascii, value):
+        legacy = json.dumps(value, default=_serialize_custom_object)
+        assert df_dumps(value).encode("utf-8") == legacy.encode("utf-8")
+        assert df_dumps(value, ensure_ascii=ensure_ascii).encode(
+            "utf-8") == legacy.encode("utf-8")
+
+    @pytest.mark.parametrize("text", [
+        pytest.param("\u041f\u0440\u0438\u0432\u0435\u0442", id="cyrillic"),
+        pytest.param("\u4f60\u597d", id="chinese"),
+        pytest.param("\u65e5\u672c\u8a9e", id="japanese"),
+        pytest.param("\u0645\u0631\u062d\u0628\u0627", id="arabic"),
+        pytest.param("\u0939\u093f\u0928\u094d\u0926\u0940", id="devanagari"),
+        pytest.param("e\u0301", id="combining"),
+        pytest.param("\u00e9", id="precomposed"),
+        pytest.param("\U0001f600\U00020000", id="non-bmp"),
+        pytest.param("\u2028\u2029", id="line-separators"),
+    ])
+    def test_plain_unicode_respects_option(self, codec_mode, ensure_ascii, text):
+        for value in [text, {text: ["nested", {"text": text}]}]:
+            assert df_dumps(value) == json.dumps(value)
+            encoded = df_dumps(value, ensure_ascii=ensure_ascii)
+            assert encoded == json.dumps(value, ensure_ascii=ensure_ascii)
+            if not ensure_ascii:
+                assert text in encoded
+            assert df_loads(encoded, expected_type=type(value)) == value
+            assert encoded.encode("utf-8").decode("utf-8") == encoded
+
+    @pytest.mark.parametrize("value", [
+        PlainPerson("\u0416\u4f60\U0001f600", 99),
+        ScalarPerson("e\u0301\u0645\u0939"),
+        SmartOrder("\u65e5", Hat("\u0416")),
+    ])
+    def test_custom_envelope_respects_option(
+            self, codec_mode, ensure_ascii, value):
+        envelope = _serialize_custom_object(value)
+        assert df_dumps(value) == json.dumps(envelope)
+        encoded = df_dumps(value, ensure_ascii=ensure_ascii)
+        assert encoded == json.dumps(envelope, ensure_ascii=ensure_ascii)
+        assert json.loads(encoded) == envelope
+        assert df_loads(encoded, expected_type=type(value)) == value
+
+    @pytest.mark.parametrize("value", [
+        NaiveOrder("\u0416", Hat("\u4f60")),
+        {"\u0416": PlainPerson("\u4f60", 99)},
+        [ScalarPerson("\U0001f600")],
+    ])
+    def test_nested_custom_object_rules_are_unchanged(
+            self, codec_mode, ensure_ascii, value):
+        if codec_mode == "strict":
+            with pytest.raises(TypeError):
+                df_dumps(value, ensure_ascii=ensure_ascii)
+        else:
+            assert df_dumps(value) == json.dumps(
+                value, default=_serialize_custom_object)
+            encoded = df_dumps(value, ensure_ascii=ensure_ascii)
+            assert encoded == json.dumps(
+                value, default=_serialize_custom_object,
+                ensure_ascii=ensure_ascii)
+            assert df_loads(encoded, expected_type=type(value)) == value
+            assert df_loads(encoded) == value
+
+    def test_controls_quotes_and_backslashes_remain_escaped(
+            self, codec_mode, ensure_ascii):
+        text = "\u0416" + "".join(chr(code) for code in range(32))
+        text += '"\\ and literal escapes \\u0416 \\ud800 \\n'
+        assert df_dumps(text) == json.dumps(text)
+        encoded = df_dumps(text, ensure_ascii=ensure_ascii)
+        assert encoded == json.dumps(text, ensure_ascii=ensure_ascii)
+        assert "\\u0000" in encoded
+        assert '\\"' in encoded
+        assert "\\\\" in encoded
+        assert not any(chr(code) in encoded for code in range(32))
+        assert df_loads(encoded, expected_type=str) == text
+
+    @pytest.mark.parametrize("value", [
+        {"\u0416": ["\u4f60", "e\u0301", "\U0001f600"]},
+        PlainPerson("\u0416", 99),
+        ScalarPerson("\U0001f600"),
+        SmartOrder("\u0416", Hat("\u4f60")),
+    ])
+    def test_legacy_escaped_payloads_remain_readable(
+            self, codec_mode, ensure_ascii, value):
+        legacy = json.dumps(value, default=_serialize_custom_object)
+        assert "\\u" in legacy
+        assert df_dumps(value) == legacy
+        assert df_loads(legacy, expected_type=type(value)) == value
+        assert df_loads(
+            df_dumps(value, ensure_ascii=ensure_ascii),
+            expected_type=type(value)) == value
+        if codec_mode == "loose" or isinstance(value, dict):
+            assert df_loads(legacy) == value
+
+    @pytest.mark.parametrize("text", [
+        pytest.param("\ud800", id="first-high-surrogate"),
+        pytest.param("\udbff", id="last-high-surrogate"),
+        pytest.param("\udc00", id="first-low-surrogate"),
+        pytest.param("\udfff", id="last-low-surrogate"),
+        pytest.param("\ud83d\ude00", id="explicit-surrogate-pair"),
+        pytest.param("\udc00\ud800", id="reversed-surrogates"),
+        pytest.param("\ud800x\udfff", id="separated-surrogates"),
+    ])
+    @pytest.mark.parametrize("custom", [False, True], ids=["plain", "custom"])
+    def test_surrogates_remain_transport_safe(
+            self, codec_mode, ensure_ascii, text, custom):
+        value = ScalarPerson(text) if custom else {text: [text]}
+        legacy = json.dumps(value, default=_serialize_custom_object)
+        assert df_dumps(value) == legacy
+        encoded = df_dumps(value, ensure_ascii=ensure_ascii)
+        assert encoded == legacy
+        assert encoded.encode("utf-8").decode("utf-8") == encoded
+        # json.loads combines escaped surrogate pairs, as it did before.
+        assert df_loads(encoded, expected_type=type(value)) == df_loads(
+            legacy, expected_type=type(value))
+
+    @pytest.mark.parametrize("custom", [False, True], ids=["plain", "custom"])
+    def test_unicode_and_surrogates_can_coexist(
+            self, codec_mode, ensure_ascii, custom):
+        text = "\u0416\ud800\U0001f600\udfff\\ud800\x7f"
+        value = ScalarPerson(text) if custom else {text: [text]}
+        legacy = json.dumps(value, default=_serialize_custom_object)
+        assert df_dumps(value) == legacy
+        encoded = df_dumps(value, ensure_ascii=ensure_ascii)
+        if ensure_ascii:
+            assert encoded == legacy
+        else:
+            assert "\u0416" in encoded
+            assert "\U0001f600" in encoded
+        assert "\\ud800" in encoded
+        assert "\\udfff" in encoded
+        assert "\\\\ud800" in encoded
+        assert "\\u007f" in encoded
+        assert encoded.encode("utf-8").decode("utf-8") == encoded
+        assert df_loads(encoded, expected_type=type(value)) == value
+
+    def test_cyrillic_payload_utf8_size(self, codec_mode, ensure_ascii):
+        value = {"text": "\u0416" * 100000}
+        legacy = json.dumps(value)
+        assert df_dumps(value) == legacy
+        encoded = df_dumps(value, ensure_ascii=ensure_ascii)
+        assert len(legacy.encode("utf-8")) == 600012
+        assert len(encoded.encode("utf-8")) == (
+            600012 if ensure_ascii else 200012)
+        assert df_loads(encoded, expected_type=dict) == value
+
+
 # ===================================================================
 # 12. _get_serialize_default
 # ===================================================================

tests/test_durable_functions_serialization.py

@@ -358,6 +358,29 @@ def test_encode_uses_df_dumps(self):
             ActivityTriggerConverter.encode({"x": 1}, expected_type=None)
         spy.assert_called_once_with({"x": 1})
 
+    def test_unicode_output_stays_ascii_escaped_in_both_modes(self):
+        value = {
+            "\u4f60": [
+                "\u0416", "e\u0301", "\U0001f600",
+                "\ud800", "\ud83d\ude00", "\\ud800",
+            ],
+        }
+        for mode in ["0", "1"]:
+            with self.subTest(mode=mode), _strict_env(mode):
+                datum = ActivityTriggerConverter.encode(
+                    value, expected_type=dict)
+                self.assertEqual(datum.type, "json")
+                self.assertEqual(datum.value, json.dumps(value))
+                self.assertIn("\\u0416", datum.value)
+                self.assertIn("\\ud83d\\ude00", datum.value)
+                self.assertIn("\\ud800", datum.value)
+                self.assertEqual(
+                    datum.value.encode("utf-8").decode("utf-8"), datum.value)
+                self.assertEqual(
+                    ActivityTriggerConverter.decode(
+                        datum, trigger_metadata=None),
+                    json.loads(json.dumps(value)))
+
     def test_decode_custom_object_loaded_module(self):
         s = df_dumps(Hat("green"))
         datum = Datum(type="json", value=s)