diff --git a/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json index cff1d6d..b89291a 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/base_entity.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/base_entity.schema.json", "title": "base_entity", "description": "A complete entity specification within data ingest", @@ -10,7 +10,8 @@ "description": "A mapping of field names to their Python types. These will either be strings representing Python types (if there are no argumements to the type), and field specification objects otherwise", "additionalProperties": { "$ref": "field.schema.json" - } + }, + "minProperties": 1 }, "aliases": { "description": "A mapping of field name to allowed field alias", @@ -26,5 +27,7 @@ "type": "string" } } - } + }, + + "required": ["fields"] } diff --git a/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json b/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json index 2635ac4..a9a6a91 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/contact_error_details.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/contract_error_details.schema.json", "title": "base_entity", "description": "A mapping of field names to the custom error code and message required if these fields were to fail validation during the data contract phase. For nested fields, these should be specified using struct '.' notation (eg. fieldA.fieldB.fieldC)", diff --git a/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json index 24453fb..b751824 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/entity.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/entity.schema.json", "title": "entity", "description": "A concrete entity specification within data ingest", diff --git a/docs/advanced_guidance/json_schemas/contract/components/field.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field.schema.json index c8454ab..f0e7d20 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/field.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/field.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/field.schema.json", "title": "field", "description": "The type of a field, specified as a bare Python type if there are no argumements, and as an object otherwise. This can also be a reference to a type defined in the dataset", @@ -9,6 +9,10 @@ }, { "$ref": "field_specification.schema.json" + }, + { + "type": "string", + "minLength": 1 } ] } diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json index 5f00ca9..1939d29 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/field_error_detail.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/field_error_detail.schema.json", "title": "field_error_detail", "description": "The custom details to be used for a field when a validation error is raised during the data contract phase", diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json b/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json deleted file mode 100644 index 694948a..0000000 --- a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema copy.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-07/schema", - "$id": "data-ingest:contract/components/field_error_type.schema.json", - "title": "field_error_detail", - "description": "The error type for a field when a validation error is raised during the data contract phase", - "type": "object", - "properties": { - "error_type": { - "description": "The type of error the details are for", - "type": "string", - "enum": [ - "Blank", - "Bad value", - "Wrong format" - ], - "additionalProperties": { - "$ref": "field_error_detail.schema.json" - } - } - } -} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json index 694948a..13f1a26 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/field_error_type.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/field_error_type.schema.json", "title": "field_error_detail", "description": "The error type for a field when a validation error is raised during the data contract phase", diff --git a/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json b/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json index 5a57f4c..5564e18 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/field_specification.schema.json @@ -1,16 +1,16 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/field_specification.schema.json", "title": "field_specification", "description": "A complete field specification within data ingest. This is generally specified using a type, model, or callable (with optional constraints) which results in a type", "type": "object", "properties": { "human_readable_name": { - "description": "A human readable name for the field", + "description": "A human readable name for the field. Non functional.", "type": "string" }, "description": { - "description": "A human-readable description of the field's purpose", + "description": "A human-readable description of the field's purpose. Non functional.", "type": "string" }, "type": { @@ -36,108 +36,216 @@ "default": { "description": "A default value for the field, to be used if it is not provided" }, - "functions": { - "description": "Validation functions to be applied to the type", - "type": "array", - "items": { - "$ref": "validation_function.schema.json" - } - }, "examples": { - "description": "An array of example values that the field could contain", + "description": "An array of example values that the field could contain. Non functional.", "type": "array" }, "disallowed_examples": { - "description": "An array of example values that the field could not contain", + "description": "An array of example values that the field could not contain. Non functional.", "type": "array" } }, - "dependencies": { - "type": { - "allOf": [ + "allOf": [ + { + "oneOf": [ + { "required": ["type"] }, + { "required": ["model"] }, + { "required": ["callable"] } + ], + "errorMessage": "Must specify exactly one of: type, model, or callable" + }, + { + "not": { + "required": ["type", "model"] + }, + "errorMessage": "Type and model are mutually exclusive" + }, + { + "not": { + "required": ["type", "callable"] + }, + "errorMessage": "Type and callable are mutually exclusive" + }, + { + "not": { + "required": ["model", "callable"] + }, + "errorMessage": "Model and callable are mutually exclusive" + }, + { + "if": { "required": ["callable"] }, + "then": { + "required": ["constraints"], + "errorMessage": "constraints is required when using callable" + } + } + ], + "unevaluatedProperties": false, + "dependentSchemas": { + "callable": { + "oneOf": [ { - "not": { - "required": [ - "model" - ] + "properties": { + "callable": { + "const": "alphanumeric" + }, + "constraints": { + "$ref": "type_constraints/alphanumeric.schema.json" + } } }, { - "not": { - "required": [ - "callable" - ] + "properties": { + "callable": { + "const": "condate" + }, + "constraints": { + "$ref": "type_constraints/condate.schema.json" + } } }, { - "not": { - "required": [ - "constraints" - ] + "properties": { + "callable": { + "const": "condecimal" + }, + "constraints": { + "$ref": "type_constraints/condecimal.schema.json" + } } - } - ] - }, - "model": { - "allOf": [ + }, { - "not": { - "required": [ - "type" - ] + "properties": { + "callable": { + "const": "conformatteddate" + }, + "constraints": { + "$ref": "type_constraints/conformatteddate.schema.json" + } } }, { - "not": { - "required": [ - "callable" - ] + "properties": { + "callable": { + "const": "conint" + }, + "constraints": { + "$ref": "type_constraints/conint.schema.json" + } } }, { - "not": { - "required": [ - "constraints" - ] + "properties": { + "callable": { + "const": "constr" + }, + "constraints": { + "$ref": "type_constraints/constr.schema.json" + } } - } - ] - }, - "callable": { - "allOf": [ + }, + { + "properties": { + "callable": { + "const": "formatteddatetime" + }, + "constraints": { + "$ref": "type_constraints/formatteddatetime.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "formattedtime" + }, + "constraints": { + "$ref": "type_constraints/formattedtime.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "identifier" + }, + "constraints": { + "$ref": "type_constraints/identifier.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "postcode" + }, + "constraints": { + "$ref": "type_constraints/postcode.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "postcode" + }, + "constraints": { + "$ref": "type_constraints/postcode.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "nhsnumber" + }, + "constraints": { + "$ref": "type_constraints/nhsnumber.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "permissivenhsno" + }, + "constraints": { + "$ref": "type_constraints/permissivenhsno.schema.json" + } + } + }, + { + "properties": { + "callable": { + "const": "orgid" + }, + "constraints": { + "$ref": "type_constraints/orgid.schema.json" + } + } + }, { - "not": { - "required": [ - "type" - ] + "properties": { + "callable": { + "const": "reportingperiodstart" + }, + "constraints": { + "$ref": "type_constraints/reportingperiodstart.schema.json" + } } }, { - "not": { - "required": [ - "model" - ] + "properties": { + "callable": { + "const": "reportingperiodend" + }, + "constraints": { + "$ref": "type_constraints/reportingperiodend.schema.json" + } } } ] } }, - "oneOf": [ - { - "required": [ - "type" - ] - }, - { - "required": [ - "model" - ] - }, - { - "required": [ - "callable" - ] - } - ], "additionalProperties": false } diff --git a/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json b/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json index 3f0954e..fdf2565 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/readable_entity.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/readable_entity.schema.json", "title": "entity", "description": "A concrete entity specification within data ingest. This must be readable from file", @@ -13,32 +13,144 @@ "reader_config": { "description": "Reader configuration for each entity. File extensions should be used as keys.", "type": "object", - "patternProperties": { - "^\\..+$": { - "type": "object", - "properties": { - "reader": { - "type": "string", - "description": "The name of the reader class that should be used for the file extension." - }, - "kwargs": { - "type": "object", - "description": "Additional keyword constructor args for the reader class." - } - } + "properties": { + ".csv": { + "$ref": "#/$defs/csvReaderConfig", + "description": "Configuration for loading an CSV file.", + "required": ["reader"] + }, + ".json": { + "$ref": "#/$defs/jsonReaderConfig", + "description": "Configuration for loading an JSON file.", + "required": ["reader"] + }, + ".xml": { + "$ref": "#/$defs/xmlReaderConfig", + "description": "Configuration for loading an XML file.", + "required": ["reader"] } }, - "additionalProperties": false + "minProperties": 1 }, "key_field": { "description": "The key field that should be used for the entity.", "type": "string" } - }, - "required": [ - "fields", - "reader_config" + } + } + ], + "$defs": { + "csvReaderConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "reader": { + "const": "DuckDBCSVReader" + }, + "kwargs": { + "$ref": "reader_constraints/ddb_csv_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "PolarsToDuckDBCSVReader" + }, + "kwargs": { + "$ref": "reader_constraints/polars_ddb_csv_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "DuckDBCSVRepeatingHeaderReader" + }, + "kwargs": { + "$ref": "reader_constraints/ddb_csv_header_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "SparkCSVReader" + }, + "kwargs": { + "$ref": "reader_constraints/spark_csv_reader.schema.json" + } + } + } + ] + }, + "jsonReaderConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "reader": { + "const": "DuckDBJSONReader" + }, + "kwargs": { + "$ref": "reader_constraints/ddb_json_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "SparkJSONReader" + }, + "kwargs": { + "$ref": "reader_constraints/spark_json_reader.schema.json" + } + } + } + ] + }, + "xmlReaderConfig": { + "oneOf": [ + { + "type": "object", + "properties": { + "reader": { + "const": "DuckDBXMLReader" + }, + "kwargs": { + "$ref": "reader_constraints/ddb_xml_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "SparkXMLStreamReader" + }, + "kwargs": { + "$ref": "reader_constraints/spark_xml_stream_reader.schema.json" + } + } + }, + { + "type": "object", + "properties": { + "reader": { + "const": "SparkXMLReader" + }, + "kwargs": { + "$ref": "reader_constraints/spark_xml_reader.schema.json" + } + } + } ] } - ] -} + }, + "required": ["fields", "reader_config"] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json new file mode 100644 index 0000000..7a6cb48 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_header_reader.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/ddb_csv_header_reader.schema.json", + "title": "Keyword Arguments a DuckDBCSVRepeatingHeaderReader", + "description": "Arguments to contol how the DuckDB CSV Header Reader interacts with a csv file", + "type": "object", + "anyOf": [ + { + "$ref": "ddb_csv_reader.schema.json" + } + ], + "properties": { + "non_unique_header_error_code": { + "type": "string", + "description": "Allows you to customise the error code generated in the error report when there are more than 1 set of values for a header." + }, + "non_unique_header_error_message": { + "type": "string", + "description": "Allows you to customise the error message generated in the error report when there are more than 1 set of values for a header." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json new file mode 100644 index 0000000..8a531b5 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_csv_reader.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/ddb_csv_reader.schema.json", + "title": "Keyword Arguments a DuckDBCSVReader", + "description": "Arguments to contol how the DuckDB CSV Reader interacts with a csv file", + "type": "object", + "anyOf": [ + { + "$ref": "global_csv_reader_args.schema.json" + } + ], + "properties": { + "header": { + "type": "boolean", + "description": "First line of each file contains the column names." + }, + "delim": { + "type": "string", + "description": "Delimiter character used to separate columns within each line." + }, + "quotechar": { + "type": "string", + "description": "Character used to denote the start and end of a quoted item." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json new file mode 100644 index 0000000..7cb4576 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_json_reader.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/ddb_json_reader.schema.json", + "title": "Keyword Arguments for DuckDBJSONReader", + "description": "Arguments to contol how the DuckDB JSON Reader interacts with a json file", + "type": "object", + "properties": { + "json_format": { + "type": "string", + "description": "The format of the json file. Defaults to 'array'.", + "enum": [ + "auto", + "unstructured", + "newline_delimited", + "array" + ] + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json new file mode 100644 index 0000000..4872335 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/ddb_xml_reader.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/ddb_xml_reader.schema.json", + "title": "Keyword Arguments for DuckDBXMLStreamReader", + "description": "Arguments to contol how the DuckDB XML Reader interacts with a xml file", + "type": "object", + "anyOf": [ + { + "$ref": "global_xml_reader_args.schema.json" + } + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json new file mode 100644 index 0000000..4ea82ec --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_csv_reader_args.schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/global_csv_reader_args.schema.json", + "title": "Keyword Arguments used across all CSV readers", + "description": "Arguments present in all CSV readers available.", + "type": "object", + "properties": { + "field_check": { + "type": "boolean", + "description": "Enable to check for additional or missing fields." + }, + "field_check_error_code": { + "type": "string", + "description": "Allows you to customise the error code generated in the error report when the field check fails." + }, + "field_check_error_message": { + "type": "string", + "description": "Allows you to customise the error message generated in the error report when the field check fails." + }, + "null_empty_strings": { + "type": "boolean", + "description": "Converts empty string values into 'Null' values." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json new file mode 100644 index 0000000..d82ce72 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/global_xml_reader_args.schema.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/global_xml_reader_args.schema.json", + "title": "Keyword Arguments used across all XML readers", + "description": "Arguments present in all XML readers.", + "type": "object", + "properties": { + "record_tag": { + "type": "string", + "description": "A required string indicating the tag of each 'record' in the XML document." + }, + "root_tag": { + "type": "string", + "description": "A string indicating the tag to find the records in within the XML document." + }, + "trim_cells": { + "type": "string", + "description": "Removes leading and trailing whitespaces." + }, + "null_values": { + "type": "array", + "description": "A container of values to replace with null if encountered in an element.", + "items": { + "type": "string" + } + }, + "sanitise_multiline": { + "type": "boolean", + "description": "Whether to sanitise (remove newlines and multiple spaces) from multiline fields" + }, + "encoding": { + "type": "string", + "description": "Encoding of the XML file. Default: `utf-8-sig`" + }, + "n_records_to_read": { + "type": "integer", + "description": "The maximum number of records to read from a document" + }, + "xsd_location": { + "type": "string", + "description": "The Relative path (to the dischema) for the XSD document." + }, + "xsd_error_code": { + "type": "string", + "description": "Allows you to customise the error code generated in the error report when the XSD check fails." + }, + "xsd_error_message": { + "type": "string", + "description": "Allows you to customise the error message generated in the error report when the XSD check fails." + }, + "rules_location": { + "type": "string", + "description": "Allows you to prefix the directory which contains the XSD document." + } + }, + "required": [ + "record_tag" + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json new file mode 100644 index 0000000..7564660 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/polars_ddb_csv_reader.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/polars_ddb_csv_reader.schema.json", + "title": "Keyword Arguments a DuckDBCSVReader", + "description": "Arguments to contol how the DuckDB CSV Reader interacts with a csv file", + "type": "object", + "anyOf": [ + { + "$ref": "global_csv_reader_args.schema.json" + } + ], + "properties": { + "has_header": { + "type": "boolean", + "description": "First line of each file contains the column names." + }, + "separator": { + "type": "string", + "description": "Delimiter character used to separate columns within each line." + }, + "quote_char": { + "type": "string", + "description": "Character used to denote the start and end of a quoted item." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json new file mode 100644 index 0000000..e80dd7a --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_csv_reader.schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/spark_csv_reader.schema.json", + "title": "Keyword Arguments a SparkCSVReader", + "description": "Arguments to contol how the Spark CSV Reader interacts with a csv file", + "type": "object", + "anyOf": [ + { + "$ref": "global_csv_reader_args.schema.json" + } + ], + "properties": { + "delimiter": { + "type": "string", + "description": "Sets a separator for each field and value. This separator can be one or more characters." + }, + "escape_char": { + "type": "string", + "description": "DO NOT USE - NOT CURRENTLY AVAILABLE IN DVE. Sets a single character used for escaping quotes inside an already quoted value." + }, + "quote_char": { + "type": "string", + "description": "DO NOT USE - NOT CURRENTLY AVAILABLE IN DVE. Sets a single character used for escaping quoted values where the separator can be part of the value. For reading, if you would like to turn off quotations, you need to set not null but an empty string." + }, + "header": { + "type": "boolean", + "description": "For reading, uses the first line as names of columns. Note that if the given path is a RDD of Strings, this header option will remove all lines same with the header if exists. CSV built-in functions ignore this option." + }, + "multi_line": { + "type": "boolean", + "description": "Allows a row to span multiple lines, by parsing line breaks within quoted values as part of the value itself. CSV built-in functions ignore this option." + }, + "encoding": { + "type": "string", + "description": "DO NOT USE - NOT CURRENTLY AVAILABLE IN DVE. For reading, decodes the CSV files by the given encoding type. CSV built-in functions ignore this option." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json new file mode 100644 index 0000000..af783f6 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_json_reader.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/spark_json_reader.schema.json", + "title": "Keyword Arguments for SparkJSONReader", + "description": "Arguments to contol how the Spark JSON Reader interacts with a json file", + "type": "object", + "properties": { + "encoding": { + "type": "string", + "description": "For reading, allows to forcibly set one of standard basic or extended encoding for the JSON files. For example UTF-16BE, UTF-32LE." + }, + "multi_line": { + "type": "boolean", + "description": "Parse one record, which may span multiple lines, per file. JSON built-in functions ignore this option." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json new file mode 100644 index 0000000..9ab1c74 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_reader.schema.json @@ -0,0 +1,43 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/spark_xml_reader.schema.json", + "title": "Keyword Arguments for SparkXMLReader", + "description": "Arguments to contol how the Spark XML Reader interacts with a xml file", + "type": "object", + "anyOf": [ + { + "$ref": "global_xml_reader_args.schema.json" + } + ], + "properties": { + "sampling_ratio": { + "type": "integer", + "description": "Sampling ratio for inferring schema (0.0 ~ 1)." + }, + "exclude_attribute": { + "type": "boolean", + "description": "Whether you want to exclude attributes in elements or not." + }, + "mode": { + "type": "string", + "description": "The mode for dealing with corrupt records during parsing. Permissive = When it encounters a corrupted record, it sets all fields to null and puts the malformed string into a new field configured by columnNameOfCorruptRecord. When it encounters a field of the wrong datatype, it sets the offending field to null; DROPMALFORMED = ignores the whole corrupted records; FAILFAST = throws an exception when it meets corrupted records.", + "enum": [ + "PERMISSIVE", + "FAILFAST", + "DROPMALFORMED" + ] + }, + "infer_schema": { + "type": "boolean", + "description": "If true, attempts to infer an appropriate type for each resulting DataFrame column, like a boolean, numeric or date type. If false, all resulting columns are of string type." + }, + "ignore_namespace": { + "type": "boolean", + "description": "If true, namespaces prefixes on XML elements and attributes are ignored. Tags and would, for example, be treated as if both are just . Note that, at the moment, namespaces cannot be ignored on the rowTag element, only its children. Note that XML parsing is in general not namespace-aware even if false." + }, + "namespace": { + "type": "string", + "description": "Applies a namespace to each rowTag if required." + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json new file mode 100644 index 0000000..da7cdbc --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/reader_constraints/spark_xml_stream_reader.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/reader_constraints/spark_xml_stream_reader.schema.json", + "title": "Keyword Arguments for SparkXMLStreamReader", + "description": "Arguments to contol how the Spark XML Stream Reader interacts with a xml file", + "type": "object", + "anyOf": [ + { + "$ref": "global_xml_reader_args.schema.json" + } + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json new file mode 100644 index 0000000..4509b3d --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/alphanumeric.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/identifier.schema.json", + "title": "Constraints for alphanumeric value", + "description": "Constaints for an Alphanumeric type", + "type": "object", + "properties": { + "min_digits": { + "type": "integer", + "description": "The minimum number of alphanumeric characters required" + }, + "max_digits": { + "type": "integer", + "description": "The maximum number of alphanumeric characters required" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json new file mode 100644 index 0000000..a25ba1c --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condate.schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/condate.schema.json", + "title": "Constraint for Dates", + "description": "Constaints for an Date type", + "type": "object", + "properties": { + "gt": { + "type": "string", + "description": "Enforces date to be greater than the set value" + }, + "ge": { + "type": "string", + "description": "Enforces date to be greater than or equal to the set value" + }, + "lt": { + "type": "string", + "description": "Enforces date to be less than the set value" + }, + "le": { + "type": "string", + "description": "Enforces date to be less than or equal to the set value" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json new file mode 100644 index 0000000..4ea552c --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/condecimal.schema.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/condecimal.schema.json", + "title": "Constraints for Decimals", + "description": "Constaints for an Decimal type", + "type": "object", + "properties": { + "gt": { + "type": "number", + "description": "Enforces decimal to be greater than the set value" + }, + "ge": { + "type": "number", + "description": "Enforces decimal to be greater than or equal to the set value" + }, + "lt": { + "type": "number", + "description": "Enforces decimal to be less than the set value" + }, + "le": { + "type": "number", + "description": "Enforces decimal to be less than or equal to the set value" + }, + "max_digits": { + "type": "integer", + "description": "Maximum number of digits within the decimal. it does not include a zero before the decimal point or trailing decimal zeroes" + }, + "decimal_places": { + "type": "integer", + "description": "Max number of decimal places allowed. it does not include trailing decimal zeroes" + }, + "multiple_of": { + "type": "number", + "description": "Enforces decimal to be a multiple of the set value" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json new file mode 100644 index 0000000..40fe854 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conformatteddate.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/conformatteddate.schema.json", + "title": "Constraints for Constrained Formatted Date", + "description": "Constaints for an conformatteddate type", + "type": "object", + "properties": { + "date_format": { + "type": "string", + "description": "Enforces date to match a specific format." + }, + "strict": { + "type": "boolean", + "description": "Controls type coercion" + }, + "ge": { + "type": "string", + "description": "Enforces date to be greater than or equal to the set value" + }, + "le": { + "type": "string", + "description": "Enforces date to be less than the set value" + }, + "gt": { + "type": "string", + "description": "Enforces date to be greater than the set value" + }, + "lt": { + "type": "string", + "description": "Enforces date to be less than or equal to the set value" + } + }, + "required": ["date_format"] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json new file mode 100644 index 0000000..7de2c53 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/conint.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/conint.schema.json", + "title": "Constraints for Integer", + "description": "Constaints for an Integer type", + "type": "object", + "properties": { + "strict": { + "type": "boolean", + "description": "Controls type coercion" + }, + "gt": { + "type": "integer", + "description": "Enforces integer to be greater than the set value" + }, + "ge": { + "type": "integer", + "description": "Enforces integer to be greater than or equal to the set value" + }, + "lt": { + "type": "integer", + "description": "Enforces integer to be less than the set value" + }, + "le": { + "type": "integer", + "description": "Enforces integer to be less than or equal to the set value" + }, + "multiple_of": { + "type": "integer", + "description": "Enforces integer to be a multiple of the set value" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json new file mode 100644 index 0000000..d9fd393 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/constr.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/constr.schema.json", + "title": "Constraints for String", + "description": "Constaints for an String type", + "type": "object", + "properties": { + "strip_whitespace": { + "type": "boolean", + "description": "Removes leading and trailing whitespace" + }, + "to_lower": { + "type": "boolean", + "description": "Turns all characters to lowercase" + }, + "min_length": { + "type": "integer", + "description": "Minimum length of the byte string" + }, + "max_length": { + "type": "integer", + "description": "Maximum length of the byte string" + }, + "strict": { + "type": "boolean", + "description": "Controls type coercion" + }, + "regex": { + "type": "string", + "description": "Enforces string to match a specific regex pattern" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json new file mode 100644 index 0000000..35691de --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formatteddatetime.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/formatteddatetime.schema.json", + "title": "Constraints for Formatted Datetime", + "description": "Constaints for an FormattedDatetime type", + "type": "object", + "properties": { + "date_format": { + "type": "string", + "description": "Enforces date to match a specific format." + }, + "timezone_treatment": { + "type": "string", + "enum": [ + "forbid", + "permit", + "require" + ] + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json new file mode 100644 index 0000000..c827730 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/formattedtime.schema.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/formattedtime.schema.json", + "title": "Constraints for Formatted Time", + "description": "Constaints for an FormattedTime type", + "type": "object", + "properties": { + "time_format": { + "type": "string", + "description": "Enforces date to match a specific format." + }, + "timezone_treatment": { + "type": "string", + "enum": [ + "forbid", + "permit", + "require" + ] + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json new file mode 100644 index 0000000..d8323da --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/identifier.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/identifier.schema.json", + "title": "Constraints for alphanumeric identifier (including fixed set of special characters)", + "description": "Constaints for an Identifer type (alphanumeric + fixed set of special chars)", + "type": "object", + "allOf": [ + { + "$ref": "alphanumeric.schema.json" + } + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json new file mode 100644 index 0000000..50272ca --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/nhsnumber.schema.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/nhsnumber.schema.json", + "title": "Constraints for NHSNumber", + "description": "Constaints for the NHSNumber type", + "type": "object" +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json new file mode 100644 index 0000000..a7b9a33 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/orgid.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/orgid.schema.json", + "title": "Constraints for Organisation Idenitifer such as ODS Code", + "description": "Constaints for an OrgId type", + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Enforces string to match a specific regex pattern" + }, + "strip_whitespace": { + "type": "boolean", + "description": "Removes leading and trailing whitespace" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json new file mode 100644 index 0000000..4e0eeba --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/permissivenhsno.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/permissivenhsno.schema.json", + "title": "Constraints for NHSNumber", + "description": "Constaints for the NHSNumber type", + "type": "object", + "allOf": [ + { + "$ref": "constr.schema.json" + } + ], + "properties": { + "warn_on_test_numbers" : { + "type": "boolean", + "description": "Whether only warning should be raised if test numbers included" + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json new file mode 100644 index 0000000..31685fc --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/postcode.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/postcode.schema.json", + "title": "Constraints for Postcode", + "description": "Constaints for an Postcode type. Defaults to UK postcode.", + "type": "object", + "allOf": [ + { + "$ref": "constr.schema.json" + } + ], + "properties": { + "pattern": { + "type": "string", + "description": "Regex format for the postcode" + }, + "apply_normalize": { + "type": "boolean", + "description": "Whether to remove whitespace from the input value" + } + } +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json new file mode 100644 index 0000000..d2a7f0e --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodend.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/reportingperiodend.schema.json", + "title": "Constraints for Reporting Period End Date", + "description": "Constaints for an ReportingPeriod (end) type", + "type": "object", + "properties": { + "reporting_period_type": { + "type": "string", + "description": "Enforces date to be the start or end of a month.", + "enum": [ + "start", + "end" + ] + }, + "date_format": { + "type": "string", + "description": "Enforces date to match a specific format" + } + }, + "required": ["reporting_period_type"] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json new file mode 100644 index 0000000..f5749c2 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/contract/components/type_constraints/reportingperiodstart.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:contract/components/types/reportingperiodstart.schema.json", + "title": "Constraints for Reporting Period Start Date", + "description": "Constaints for an ReportingPeriod (start) type", + "type": "object", + "properties": { + "reporting_period_type": { + "type": "string", + "description": "Enforces date to be the start or end of a month.", + "enum": [ + "start", + "end" + ] + }, + "date_format": { + "type": "string", + "description": "Enforces date to match a specific format" + } + }, + "required": ["reporting_period_type"] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json b/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json index 0f331ef..660a910 100644 --- a/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/components/type_name.schema.json @@ -1,7 +1,29 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/components/type_name.schema.json", "title": "type_name", "description": "The name of an allowed type within data ingest. This should be a Python type", - "type": "string" -} + "type": "string", + "enum": [ + "str", + "int", + "float", + "date", + "datetime", + "constr", + "conint", + "condate", + "condecimal", + "postcode", + "nhsnumber", + "permissivenhsno", + "alphanumeric", + "identifier", + "orgid", + "formatteddatetime", + "formattedtime", + "conformatteddate", + "reportingperiodstart", + "reportingperiodend" + ] +} \ No newline at end of file diff --git a/docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json b/docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json deleted file mode 100644 index 69f0f85..0000000 --- a/docs/advanced_guidance/json_schemas/contract/components/validation_function.schema.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft-07/schema", - "$id": "data-ingest:contract/components/validation_function.schema.json", - "title": "validation_function", - "description": "A validation function to be applied to a field", - "type": "object", - "properties": { - "name": { - "description": "The name of the validation function to be applied", - "type": "string" - }, - "error_type": { - "description": "The type of error to emit if the function fails. This defaults to 'record_rejection'", - "type": "string", - "enum": [ - "record_rejection", - "file_rejection", - "warning" - ] - }, - "error_message": { - "description": "The message to associate with the error, if the function fails", - "type": "string" - }, - "kwargs": { - "description": "Keyword arguments to be passed to the Python function", - "type": "object" - } - }, - "required": [ - "name" - ], - "additionalProperties": false -} diff --git a/docs/advanced_guidance/json_schemas/contract/contract.schema.json b/docs/advanced_guidance/json_schemas/contract/contract.schema.json index dad5a92..f80a6e5 100644 --- a/docs/advanced_guidance/json_schemas/contract/contract.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/contract.schema.json @@ -1,8 +1,8 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:contract/contract.schema.json", "title": "contract", - "description": "A data contract within data ingest. This specifies the information required to read in the data", + "description": "A data contract within data ingest. This specifies the information required to model and read the data", "type": "object", "properties": { "types": { @@ -24,12 +24,15 @@ "type": "object", "additionalProperties": { "$ref": "components/readable_entity.schema.json" - } + }, + "minProperties": 1 }, "error_details": { - "description": "Relative path from the dischema document to reference a error details document.", - "type": "string" + "description": "Relative path from the dischema document to reference an error details document.", + "type": "string", + "pattern": "^[a-zA-Z0-9].*_data_contract_error_details\\.json$" } }, + "required": ["datasets"], "additionalProperties": true } diff --git a/docs/advanced_guidance/json_schemas/contract/error_details.schema.json b/docs/advanced_guidance/json_schemas/contract/error_details.schema.json index 58c369b..046d751 100644 --- a/docs/advanced_guidance/json_schemas/contract/error_details.schema.json +++ b/docs/advanced_guidance/json_schemas/contract/error_details.schema.json @@ -22,6 +22,7 @@ "$defs": { "errorObject": { "type": "object", + "required": ["error_code", "error_message"], "properties": { "error_code": { "type": "string", diff --git a/docs/advanced_guidance/json_schemas/dataset.schema.json b/docs/advanced_guidance/json_schemas/dataset.schema.json index 75cf4c3..4e85011 100644 --- a/docs/advanced_guidance/json_schemas/dataset.schema.json +++ b/docs/advanced_guidance/json_schemas/dataset.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:dataset.schema.json", "title": "dataset", "description": "A full dataset for data ingest. This contains the contract and the associated transformations", diff --git a/docs/advanced_guidance/json_schemas/rule_store.schema.json b/docs/advanced_guidance/json_schemas/rule_store.schema.json index cf27308..a3822bb 100644 --- a/docs/advanced_guidance/json_schemas/rule_store.schema.json +++ b/docs/advanced_guidance/json_schemas/rule_store.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:rule_store.schema.json", "title": "rule_store", "description": "A store of rules within data ingest. Property names should match the names of the contained rules.", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json index d962165..372caa5 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/business_filter.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/business_filter.schema.json", "title": "rule", "description": "Descriptions of a 'business' filter within data ingest", @@ -10,28 +10,6 @@ }, "parameters": { "type": "object" - }, - "error_code": { - "type": "string" - }, - "error_location": { - "type": "string" - }, - "reporting_entity": { - "type": "string" - }, - "reporting_field": { - "type": [ - "array", - "string" - ], - "items": { - "type": "string" - } - }, - "reporting_field_name": { - "description": "DEPRECATED: Added as a stopgap for pilot go-live, to be removed.", - "type": "string" } }, "required": [ diff --git a/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json index 6aae36d..07449d0 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/business_rule.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/business_rule.schema.json", "title": "rule", "description": "Definition of a business rule within data ingest", @@ -9,10 +9,12 @@ "description": "A templateable business rule.", "properties": { "rule_name": { - "type": "string" + "type": "string", + "description": "Name of the rule" }, "description": { - "type": "string" + "type": "string", + "description": "Description of the rule" }, "type": { "const": "transformation" @@ -21,10 +23,12 @@ "type": "object", "additionalProperties": { "type": "string" - } + }, + "description": "Descriptions of the parameters" }, "parameter_defaults": { - "type": "object" + "type": "object", + "description": "Defaults for the parameters if not explicitly supplied" }, "rule_config": { "$ref": "rule.schema.json" @@ -39,10 +43,12 @@ "description": "A templateable business filter.", "properties": { "rule_name": { - "type": "string" + "type": "string", + "description": "Name of the rule" }, "description": { - "type": "string" + "type": "string", + "description": "Description of the rule" }, "type": { "const": "filter" @@ -51,10 +57,12 @@ "type": "object", "additionalProperties": { "type": "string" - } + }, + "description": "Descriptions of the parameters" }, "parameter_defaults": { - "type": "object" + "type": "object", + "description": "Defaults for the parameters if not explicitly supplied" }, "rule_config": { "$ref": "filter.schema.json" @@ -69,10 +77,12 @@ "description": "A complex rule.", "properties": { "rule_name": { - "type": "string" + "type": "string", + "description": "Name of the rule" }, "description": { - "type": "string" + "type": "string", + "description": "Description of the rule" }, "type": { "const": "complex_rule" @@ -81,10 +91,12 @@ "type": "object", "additionalProperties": { "type": "string" - } + }, + "description": "Descriptions of the parameters" }, "parameter_defaults": { - "type": "object" + "type": "object", + "description": "Defaults for the parameters if not explicitly supplied" }, "rule_config": { "type": "object", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json index a1dfd2e..c27ff08 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/concrete_filter.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/concrete_filter.schema.json", "title": "rule", "description": "Description of a filter within data ingest. This filter uses a SQL expression, and requires a name.", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json index 409ee98..e24da53 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/core_filter.schema.json @@ -1,18 +1,21 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/core_filter.schema.json", "title": "rule", "description": "Descriptions of a filter within data ingest, implemented using a SQL expression. This filter does not require a name, but a concrete filter does.", "type": "object", "properties": { "entity": { - "type": "string" + "type": "string", + "description": "Name of the entity to perform the rule against" }, "name": { - "type": "string" + "type": "string", + "description": "Name for the filter" }, "expression": { - "type": "string" + "type": "string", + "description": "SQL expression for the filter" }, "failure_type": { "enum": [ @@ -20,10 +23,12 @@ "submission", "integrity", "group" - ] + ], + "description": "Type of failure" }, "failure_message": { - "type": "string" + "type": "string", + "description": "Customise the error message generated" }, "is_informational": { "anyOf": [ @@ -38,7 +43,8 @@ "{{ is_informational }}" ] } - ] + ], + "description": "Whether the filter is a warning or rejection" }, "error_code": { "anyOf": [ @@ -48,13 +54,16 @@ { "type": "number" } - ] + ], + "description": "Customise the error code generated" }, "error_location": { - "type": "string" + "type": "string", + "description": "Utilised where reporting_field not defined. Should represent the field/column that triggered the rule" }, "reporting_entity": { - "type": "string" + "type": "string", + "description": "Allows customisation over the name of the entity that triggered the validation rule" }, "reporting_field": { "type": [ @@ -63,17 +72,26 @@ ], "items": { "type": "string" - } + }, + "description": "The field(s) that triggered the validation rule" }, "reporting_field_name": { "description": "DEPRECATED: Added as a stopgap for pilot go-live, to be removed.", "type": "string" }, "category": { - "type": "string" + "type": "string", + "enum": [ + "Blank", + "Wrong format", + "Bad value", + "Bad file" + ], + "description": "Category of failure" }, - "other_entities": { - "type": "string" + "original_entity_override": { + "type": "string", + "description": "Allows the overriding of the entity to source the id/key field(s) from" } }, "required": [ diff --git a/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json index 54809ab..598dd23 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/filter.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/filter.schema.json", "title": "rule", "description": "Description of a 'real' filter within data ingest", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json index 44cba18..475117b 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/multiple_expressions.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/multiple_expressions.schema.json", "title": "multiple_expressions", "description": "A definition of multiple aliased expressions.", diff --git a/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json b/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json index 2e54fad..18af5a7 100644 --- a/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/components/rule.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/components/rule.schema.json", "title": "rule", "description": "Descriptions of allowed rules within data ingest", diff --git a/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json b/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json index e68e175..2c64538 100644 --- a/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json +++ b/docs/advanced_guidance/json_schemas/transformations/transformations.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "data-ingest:transformations/transformations.schema.json", "title": "transformations", "description": "Transformations within data ingest. This specifies the information required to transform and filter the data", @@ -16,31 +16,47 @@ "oneOf": [ { "type": "object", + "additionalProperties": false, "properties": { + "type": { + "const": "filename" + }, "filename": { "type": "string", - "description": "The path to the reference data (as Parquet) relative to the contract" + "description": "The path to the reference data (as Parquet) relative to the contract", + "pattern": ".*\\.(?:arrow|parquet)$" } }, "required": [ + "type", "filename" ] }, { "type": "object", + "additionalProperties": false, "properties": { + "type": { + "const": "uri" + }, "uri": { "type": "string", - "description": "The absolute URI of the reference data (as Parquet)" + "description": "The absolute URI of the reference data (as Parquet or Arrow)", + "pattern": "^[a-zA-Z][a-zA-Z0-9+.-]*:.+\\.(?:arrow|parquet)$" } }, "required": [ + "type", "uri" ] }, { "type": "object", + "additionalProperties": false, "properties": { + "type": { + "const": "database" + }, "database": { "type": "string", "description": "The database the reference data is stored in.", @@ -49,10 +65,12 @@ "table_name": { "type": "string", "description": "The table name (excluding database) containing the reference data.", - "pattern": "^(?:(?refdata)_)?(?[A-Za-z0-9]+)_(?\\w+)$" + "pattern": "^\\w+$" } }, "required": [ + "type", + "database", "table_name" ] } @@ -63,7 +81,8 @@ "description": "Stores of business rules", "type": "array", "items": { - "anyOf": [ + "type": "object", + "oneOf": [ { "properties": { "store_type": { @@ -72,17 +91,29 @@ "rules": { "$ref": "../rule_store.schema.json" } - } + }, + "required": [ + "store_type", + "rules" + ], + "additionalProperties": false }, { "properties": { "store_type": { - "type": "string" + "const": "json" }, "filename": { - "type": "string" + "type": "string", + "description": "Relative path to the current dischema document containing the rulestore", + "pattern": "^.*\\.json$" } - } + }, + "required": [ + "store_type", + "filename" + ], + "additionalProperties": false } ] } @@ -94,7 +125,8 @@ "type": "object", "properties": { "rule_name": { - "type": "string" + "type": "string", + "minLength": 1 }, "parameters": { "type": "object" @@ -143,4 +175,4 @@ }, "additionalProperties": false } -} +} \ No newline at end of file