diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index 0aee51b26e84..33b526aa9f83 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -748,6 +748,26 @@ class common_schema_converter { optional_props.push_back("*"); } + // Also allow declared order for interleaved optional fields, preserving the required-first alternative. + bool optional_before_required = false; + for (size_t i = 0, seen_optional = 0; i < prop_names.size(); i++) { + if (!required.count(prop_names[i])) { + seen_optional = 1; + } else if (seen_optional) { + optional_before_required = true; + break; + } + } + std::string declared_order_rule; + if (optional_before_required && required.count(prop_names[0]) && !prop_kv_rule_names.count("*")) { + declared_order_rule = "\"{\" space " + prop_kv_rule_names[prop_names[0]]; + for (size_t i = 1; i < prop_names.size(); i++) { + std::string kv = " \",\" space " + prop_kv_rule_names[prop_names[i]]; + declared_order_rule += required.count(prop_names[i]) ? kv : " (" + kv + " )?"; + } + declared_order_rule += " space \"}\""; + } + std::string rule = "\"{\" space "; for (size_t i = 0; i < required_props.size(); i++) { if (i > 0) { @@ -798,6 +818,9 @@ class common_schema_converter { rule += " space \"}\""; + if (!declared_order_rule.empty()) { + return declared_order_rule + " | " + rule; + } return rule; } diff --git a/examples/json_schema_to_grammar.py b/examples/json_schema_to_grammar.py index 83abd259da57..091038f6f7c9 100755 --- a/examples/json_schema_to_grammar.py +++ b/examples/json_schema_to_grammar.py @@ -734,6 +734,17 @@ def _build_object_rule(self, properties: List[Tuple[str, Any]], required: Set[st ) optional_props.append("*") + # Also allow declared order while preserving required-first order (mirrors the C++ converter). + first_required = [k in required for k in sorted_props] + optional_before_required = any(not a and b for i, a in enumerate(first_required) for b in first_required[i + 1:]) + declared_order_rule = None + if optional_before_required and first_required[0] and "*" not in prop_kv_rule_names: + declared_order_rule = '"{" space ' + prop_kv_rule_names[sorted_props[0]] + for k in sorted_props[1:]: + kv = f' "," space {prop_kv_rule_names[k]}' + declared_order_rule += kv if k in required else f' ({kv} )?' + declared_order_rule += ' space "}"' + rule = '"{" space ' rule += ' "," space '.join(prop_kv_rule_names[k] for k in required_props) @@ -767,6 +778,8 @@ def get_recursive_refs(ks, first_is_optional): rule += ' space "}"' + if declared_order_rule: + return declared_order_rule + ' | ' + rule return rule def format_grammar(self): @@ -791,8 +804,10 @@ def main(args_in = None): help=''' comma-separated property names defining the order of precedence for object properties; properties not specified here are given lower precedence than those that are, and - are kept in their original order from the schema. Required properties are always - given precedence over optional properties. + are kept in their original order from the schema. Required properties are given + precedence over optional properties. When (in this ordering) an optional property comes + before a required one, the first property is required and additional properties are not + allowed, this ordering is also accepted, with optional properties skippable in place. ''' ) parser.add_argument( diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp index eb4b7c78f50f..2ab581a84b5a 100644 --- a/tests/test-grammar-integration.cpp +++ b/tests/test-grammar-integration.cpp @@ -1402,6 +1402,118 @@ static void test_json_schema() { } ); + test_schema( + "optional props declared before a required one: declared order and required-first both accepted", + // Schema + R"""({ + "properties": { + "name": {"type": "string"}, + "nickname": {"type": "string"}, + "age": {"type": "integer"}, + "city": {"type": "string"} + }, + "required": ["name", "city"], + "additionalProperties": false + })""", + // Passing strings + { + // both optional fields skipped + R"""({"name": "Ana", "city": "Lima"})""", + // only the first optional field + R"""({"name": "Ana", "nickname": "An", "city": "Lima"})""", + // only the second optional field + R"""({"name": "Ana", "age": 30, "city": "Lima"})""", + // both optional fields + R"""({"name": "Ana", "nickname": "An", "age": 30, "city": "Lima"})""", + // required-first order (what stock llama.cpp produced) is still accepted + R"""({"name": "Ana", "city": "Lima", "nickname": "An"})""", + R"""({"name": "Ana", "city": "Lima", "nickname": "An", "age": 30})""", + R"""({"name": "Ana", "city": "Lima", "age": 30})""", + }, + // Failing strings + { + // the same optional field twice + R"""({"name": "Ana", "nickname": "An", "city": "Lima", "nickname": "An"})""", + // optional fields out of order + R"""({"name": "Ana", "age": 30, "nickname": "An", "city": "Lima"})""", + // missing a required field + R"""({"name": "Ana", "nickname": "An"})""", + // first required field not first + R"""({"nickname": "An", "name": "Ana", "city": "Lima"})""", + } + ); + + test_schema( + "optional props before a required one, additionalProperties not set", + // Schema + R"""({ + "properties": { + "name": {"type": "string"}, + "nickname": {"type": "string"}, + "city": {"type": "string"} + }, + "required": ["name", "city"] + })""", + // Passing strings + { + R"""({"name": "Ana", "city": "Lima"})""", + R"""({"name": "Ana", "nickname": "An", "city": "Lima"})""", + R"""({"name": "Ana", "city": "Lima", "nickname": "An"})""", + }, + // Failing strings + { + R"""({"name": "Ana", "nickname": "An", "city": "Lima", "nickname": "An"})""", + R"""({"name": "Ana", "nickname": "An"})""", + } + ); + + test_schema( + "optional first property keeps required-first order (unchanged)", + // Schema + R"""({ + "properties": { + "nickname": {"type": "string"}, + "name": {"type": "string"}, + "city": {"type": "string"} + }, + "required": ["name", "city"], + "additionalProperties": false + })""", + // Passing strings + { + R"""({"name": "Ana", "city": "Lima"})""", + R"""({"name": "Ana", "city": "Lima", "nickname": "An"})""", + }, + // Failing strings + { + R"""({"nickname": "An", "name": "Ana", "city": "Lima"})""", + } + ); + + test_schema( + "optional before required with additional properties keeps required-first order (unchanged)", + // Schema + R"""({ + "properties": { + "name": {"type": "string"}, + "nickname": {"type": "string"}, + "city": {"type": "string"} + }, + "required": ["name", "city"], + "additionalProperties": true + })""", + // Passing strings + { + R"""({"name": "Ana", "city": "Lima"})""", + R"""({"name": "Ana", "city": "Lima", "nickname": "An"})""", + R"""({"name": "Ana", "city": "Lima", "extra": 1})""", + }, + // Failing strings + { + R"""({"name": "Ana", "nickname": "An", "city": "Lima"})""", + } + ); + // NOTE: Example from https://json-schema.org/learn/getting-started-step-by-step#define-required-properties test_schema( "required props", diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index 214dbe1993b8..c1006f9bb73e 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -921,6 +921,34 @@ static void test_all(const std::string & lang, std::function