Allow JSON null in JSON-able values

aas_core_codegen/cpp/lib/_generate_json_value_verification.py

@@ -50,8 +50,8 @@ def _generate_json_value_verificator_declaration() -> List[Stripped]:
  * \\brief Exhaustively verify that a ``nlohmann::json`` value is JSON-able.
  *
  * A JSON-able value is, recursively, exactly as JSON itself is defined:
- * a boolean, a number, a string, an array of JSON-able values or an object
- * of JSON-able values with string keys -- in particular, ``null``, a binary
+ * a ``null``, a boolean, a number, a string, an array of JSON-able values or
+ * an object of JSON-able values with string keys -- in particular, a binary
  * value and a discarded value are rejected, at any depth.
  *
  * We deliberately walk the value with an explicit stack instead of plain
@@ -188,10 +188,10 @@ def _generate_json_value_verificator_implementation() -> List[Stripped]:
 common::optional<std::wstring> CheckShallowly(
 {I}const nlohmann::json& value
 ) {{
-{I}if (value.is_null() || value.is_binary() || value.is_discarded()) {{
+{I}if (value.is_binary() || value.is_discarded()) {{
 {II}return common::make_optional<std::wstring>(
 {III}common::Concat(
-{IIII}L"Expected a JSON-able value (a boolean, a number, a string, "
+{IIII}L"Expected a JSON-able value (a null, a boolean, a number, a string, "
 {IIII}L"an array or an object), but got a value of type: ",
 {IIII}common::Utf8ToWstring(value.type_name())
 {III})

aas_core_codegen/cpp/lib/_generate_jsonization.py

@@ -1010,8 +1010,8 @@ def _generate_deserialize_json_value() -> Stripped:
     """
     Generate the function to recursively de-serialize a JSON-able value.
 
-    This rejects ``null``, a binary value and a discarded value -- none of
-    which are representable as a ``JSONValue`` -- at any depth, but does not
+    This rejects a binary value and a discarded value -- neither of which is
+    representable as a ``JSONValue`` -- at any depth, but does not
     otherwise restrict the top-level shape (unlike
     :py:func:`_generate_deserialize_json_array`/
     :py:func:`_generate_deserialize_json_object`).
@@ -1024,15 +1024,15 @@ def _generate_deserialize_json_value() -> Stripped:
 > DeserializeJsonValue(
 {I}const nlohmann::json& json
 ) {{
-{I}if (json.is_null() || json.is_binary() || json.is_discarded()) {{
+{I}if (json.is_binary() || json.is_discarded()) {{
 {II}return std::make_pair<
 {III}common::optional<nlohmann::json>,
 {III}common::optional<DeserializationError>
 {II}>(
 {III}common::nullopt,
 {III}common::make_optional<DeserializationError>(
 {IIII}common::Concat(
-{IIIII}L"Expected a JSON-able value (a boolean, a number, a string, "
+{IIIII}L"Expected a JSON-able value (a null, a boolean, a number, a string, "
 {IIIII}L"an array or an object), but got a value of type: ",
 {IIIII}common::Utf8ToWstring(json.type_name())
 {IIII})
@@ -1186,7 +1186,7 @@ def _generate_serialize_json_value() -> Stripped:
 
     This re-validates the value even though it is already a
     ``nlohmann::json`` in memory, since ``nlohmann::json`` can represent
-    shapes (``null``, a binary value, a discarded value) which are not
+    shapes (a binary value, a discarded value) which are not
     representable as a ``JSONValue`` -- the constructor of a class instance
     does not, by itself, guard against these.
     """
@@ -1198,12 +1198,12 @@ def _generate_serialize_json_value() -> Stripped:
 > SerializeJsonValue(
 {I}const nlohmann::json& value
 ) {{
-{I}if (value.is_null() || value.is_binary() || value.is_discarded()) {{
+{I}if (value.is_binary() || value.is_discarded()) {{
 {II}return std::make_pair(
 {III}common::nullopt,
 {III}common::make_optional<SerializationError>(
 {IIII}common::Concat(
-{IIIII}L"Expected a JSON-able value (a boolean, a number, a string, "
+{IIIII}L"Expected a JSON-able value (a null, a boolean, a number, a string, "
 {IIIII}L"an array or an object), but got a value of type: ",
 {IIIII}common::Utf8ToWstring(value.type_name())
 {IIII})
@@ -2952,6 +2952,29 @@ def _generate_parse_properties_of_cls_implementation(
 
         parse_expr = _json_parse_expr(prop)
 
+        # NOTE (mristin):
+        # An optional JSON-able value which is a JSON null is treated as absent,
+        # as most of the SDKs can not tell the two apart.
+        if isinstance(prop.type_annotation, intermediate.OptionalTypeAnnotation) and (
+            isinstance(prop.type_annotation.value, intermediate.JsonValueTypeAnnotation)
+        ):
+            case_blocks.append(
+                Stripped(
+                    f"""\
+case properties::{enum_name}::{literal_name}: {{
+{I}common::optional<DeserializationError> parse_error = ParseInto(
+{II}{var_name},
+{II}{indent_but_first_line(parse_expr, II)}
+{I});
+{I}if (!parse_error.has_value() && {var_name}->is_null()) {{
+{II}{var_name} = common::nullopt;
+{I}}}
+{I}return parse_error;
+}}"""
+                )
+            )
+            continue
+
         case_blocks.append(
             Stripped(
                 f"""\
@@ -5015,6 +5038,15 @@ def _generate_serialize_property(
             types_namespace=cpp_common.TYPES_NAMESPACE,
         )
 
+        # NOTE (mristin):
+        # An optional JSON-able value which is a JSON null is treated as absent,
+        # as most of the SDKs can not tell the two apart.
+        condition = (
+            f"{maybe_var}.has_value() && !{maybe_var}->is_null()"
+            if isinstance(type_anno, intermediate.JsonValueTypeAnnotation)
+            else f"{maybe_var}.has_value()"
+        )
+
         # NOTE (mristin):
         # We test the binding rather than the getter. The getter is virtual, so
         # calling it a second time here would double its cost for every optional
@@ -5024,7 +5056,7 @@ def _generate_serialize_property(
 {maybe_var_type} {maybe_var}(
 {I}that.{getter}()
 );
-if ({maybe_var}.has_value()) {{
+if ({condition}) {{
 {I}{indent_but_first_line(code, I)}
 }}"""
         )

aas_core_codegen/cpp/lib/_generate_xml_rpc.py

@@ -5,10 +5,11 @@
 # the public headers in ``include/``), just like ``xml_common`` on which it
 # depends. It implements a restricted subset of XML-RPC's own element
 # vocabulary -- ``<boolean>``, ``<double>``, ``<string>``, ``<array>``,
-# ``<data>``, ``<struct>``, ``<member>``, ``<name>`` and ``<value>`` -- to
-# de/serialize a ``nlohmann::json`` value which is JSON-able (*i.e.*,
-# recursively a boolean, a number, a string, an array of JSON-able values or
-# an object of JSON-able values with string keys; never ``null``).
+# ``<data>``, ``<struct>``, ``<member>``, ``<name>`` and ``<value>``, together
+# with the widespread ``<nil/>`` extension -- to de/serialize
+# a ``nlohmann::json`` value which is JSON-able (*i.e.*, recursively a null,
+# a boolean, a number, a string, an array of JSON-able values or an object of
+# JSON-able values with string keys).
 #
 # It is meant to be used in two different ways:
 # * Stand-alone, over a whole ``<value>`` document (see
@@ -124,8 +125,8 @@ def _generate_deserialize_definitions() -> List[Stripped]:
  * document which is already being read through \\p reader.
  *
  * \\p reader is expected to be positioned at the start element of
- * the discriminator (\\c \\<boolean\\>, \\c \\<double\\>, \\c \\<string\\>,
- * \\c \\<array\\> or \\c \\<struct\\>). On success, \\p reader is left
+ * the discriminator (\\c \\<nil\\>, \\c \\<boolean\\>, \\c \\<double\\>,
+ * \\c \\<string\\>, \\c \\<array\\> or \\c \\<struct\\>). On success, \\p reader is left
  * positioned at the node right after the discriminator's own stop element,
  * mirroring how the primitive ``Deserialize*`` functions in ``xmlization``
  * behave.
@@ -221,8 +222,8 @@ def _generate_serialize_definitions() -> List[Stripped]:
  *
  * \\param os the UTF-8-encoded output stream where the XML will be written
  * \\param value to be serialized; it must be JSON-able (recursively
- * a boolean, a number, a string, an array or an object with string keys),
- * and never \\c null, or a \\ref SerializationError is returned
+ * a null, a boolean, a number, a string, an array or an object with string
+ * keys), or a \\ref SerializationError is returned
  * \\return An error, if any.
  *
  * The XML-RPC elements reside in no namespace at all, as the XML-RPC
@@ -240,8 +241,9 @@ def _generate_serialize_definitions() -> List[Stripped]:
  * \\brief Serialize \\p value, embedded within a larger XML document which is
  * already being written through \\p writer.
  *
- * This writes exactly one discriminator element (\\c \\<boolean\\>,
- * \\c \\<double\\>, \\c \\<string\\>, \\c \\<array\\> or \\c \\<struct\\>).
+ * This writes exactly one discriminator element (\\c \\<nil\\>,
+ * \\c \\<boolean\\>, \\c \\<double\\>, \\c \\<string\\>, \\c \\<array\\> or
+ * \\c \\<struct\\>).
  *
  * \\p undeclare_namespace tells whether that element is the outermost one
  * which resides in no namespace. That one undeclares the default namespace of
@@ -529,7 +531,7 @@ def _generate_deserialize_scalar_bodies_implementation() -> List[Stripped]:
     (or, for a ``<string>``, possibly already at the stop element if the
     string is empty). None of them consume the discriminator's own stop
     element -- that is uniformly handled by ``DeserializeValueFrom`` for all
-    five discriminators alike.
+    six discriminators alike.
     """
     return [
         Stripped(
@@ -757,7 +759,7 @@ def _generate_deserialize_value_from_implementation() -> Stripped:
 {III}common::nullopt,
 {III}common::make_optional<DeserializationError>(
 {IIII}common::Concat(
-{IIIII}L"Expected one of <boolean>, <double>, <string>, <array> "
+{IIIII}L"Expected one of <nil>, <boolean>, <double>, <string>, <array> "
 {IIIII}L"or <struct>, but got ",
 {IIIII}xml_common::NodeToHumanReadableWstring(reader.node())
 {IIII})
@@ -782,7 +784,16 @@ def _generate_deserialize_value_from_implementation() -> Stripped:
 {I}common::optional<nlohmann::json> value;
 {I}common::optional<DeserializationError> error;
 
-{I}if (discriminator == "boolean") {{
+{I}if (discriminator == "nil") {{
+{II}// NOTE (mristin):
+{II}// XML-RPC itself knows no null, so we follow the widespread extension,
+{II}// and read a JSON null from an empty <nil/>. Any content other than
+{II}// whitespace is refused by the check of the stop element below.
+{II}error = SkipWhitespace(reader);
+{II}if (!error.has_value()) {{
+{III}value = nlohmann::json(nullptr);
+{II}}}
+{I}}} else if (discriminator == "boolean") {{
 {II}std::tie(value, error) = DeserializeBooleanBodyFrom(reader);
 {I}}} else if (discriminator == "double") {{
 {II}std::tie(value, error) = DeserializeDoubleBodyFrom(reader);
@@ -797,7 +808,7 @@ def _generate_deserialize_value_from_implementation() -> Stripped:
 {III}common::nullopt,
 {III}common::make_optional<DeserializationError>(
 {IIII}common::Concat(
-{IIIII}L"Expected one of <boolean>, <double>, <string>, <array> "
+{IIIII}L"Expected one of <nil>, <boolean>, <double>, <string>, <array> "
 {IIIII}L"or <struct>, but got a start element <",
 {IIIII}common::Utf8ToWstring(discriminator),
 {IIIII}L">"
@@ -1312,7 +1323,13 @@ def _generate_serialize_value_body_to_implementation() -> Stripped:
 {I}const nlohmann::json& value,
 {I}bool undeclare_namespace
 ) {{
-{I}if (value.is_boolean()) {{
+{I}if (value.is_null()) {{
+{II}// NOTE (mristin):
+{II}// XML-RPC itself knows no null, so we follow the widespread extension,
+{II}// and write a JSON null as an empty <nil/>.
+{II}writer.StartElementInNoNamespace("nil", undeclare_namespace);
+{II}writer.StopElement("nil");
+{I}}} else if (value.is_boolean()) {{
 {II}writer.StartElementInNoNamespace(
 {III}"boolean", undeclare_namespace
 {II});
@@ -1383,11 +1400,11 @@ def _generate_serialize_value_body_to_implementation() -> Stripped:
 {II}writer.StopElement("struct");
 {I}}} else {{
 {II}// NOTE (mristin):
-{II}// This covers ``null``, a binary value and a discarded value -- none of
-{II}// which are representable as a ``JSONValue``.
+{II}// This covers a binary value and a discarded value -- neither of which
+{II}// is representable as a ``JSONValue``.
 {II}return common::make_optional<SerializationError>(
 {III}common::Concat(
-{IIII}L"Expected a JSON-able value (a boolean, a number, a string, "
+{IIII}L"Expected a JSON-able value (a null, a boolean, a number, a string, "
 {IIII}L"an array or an object), but got a value of type: ",
 {IIII}common::Utf8ToWstring(value.type_name())
 {III})

aas_core_codegen/cpp/lib/_generate_xmlization.py

@@ -3532,6 +3532,29 @@ def _generate_from_sequence(
 
         var_name = cpp_naming.variable_name(Identifier(f"the_{prop.name}"))
 
+        # NOTE (mristin):
+        # An optional JSON-able value which is a JSON null is treated as absent,
+        # as most of the SDKs can not tell the two apart.
+        if isinstance(prop.type_annotation, intermediate.OptionalTypeAnnotation) and (
+            isinstance(prop.type_annotation.value, intermediate.JsonValueTypeAnnotation)
+        ):
+            case_blocks.append(
+                Stripped(
+                    f"""\
+case properties::{prop_enum_name}::{prop_literal}: {{
+{I}common::optional<DeserializationError> read_error = ReadInto(
+{II}{var_name},
+{II}{indent_but_first_line(read_expr, II)}
+{I});
+{I}if (!read_error.has_value() && {var_name}->is_null()) {{
+{II}{var_name} = common::nullopt;
+{I}}}
+{I}return read_error;
+}}"""
+                )
+            )
+            continue
+
         case_blocks.append(
             Stripped(
                 f"""\
@@ -4960,6 +4983,33 @@ def _generate_write_property_statements(prop: intermediate.Property) -> Stripped
 
     writer_exprs_joined = ",\n".join(writer_exprs)
 
+    # NOTE (mristin):
+    # An optional JSON-able value which is a JSON null is treated as absent,
+    # as most of the SDKs can not tell the two apart.
+    if isinstance(prop.type_annotation, intermediate.OptionalTypeAnnotation) and (
+        isinstance(type_anno, intermediate.JsonValueTypeAnnotation)
+    ):
+        maybe_var = cpp_naming.variable_name(Identifier(f"maybe_{prop.name}"))
+
+        return Stripped(
+            f"""\
+const common::optional<nlohmann::json>& {maybe_var}(
+{I}that.{getter_name}()
+);
+if ({maybe_var}.has_value() && !{maybe_var}->is_null()) {{
+{I}error = {function_name}(
+{II}{xml_name_literal},
+{II}*{maybe_var},
+{II}writer,
+{II}iteration::Property::{prop_literal},
+{II}{indent_but_first_line(writer_exprs_joined, II)}
+{I});
+{I}if (error.has_value()) {{
+{II}return error;
+{I}}}
+}}"""
+        )
+
     return Stripped(
         f"""\
 error = {function_name}(

aas_core_codegen/cpp/optionaling.py

@@ -185,9 +185,9 @@ def transform_index(self, node: parse_tree.Index) -> Optional[Error]:
         ):
             # NOTE (mristin):
             # An item of a JSON-able array, and a value of a JSON-able object,
-            # is itself a JSON-able value. A JSON-able value is never optional
-            # -- JSON knows a ``null``, but a ``JSONValue`` deliberately does
-            # not.
+            # is itself a JSON-able value, which is a plain ``nlohmann::json``
+            # and never a ``common::optional``. It might be a JSON null, but
+            # that is a value of ``nlohmann::json`` like any other.
             self.is_optional_map[node] = False
             return None