From 81234ed669dc6cb12e17b4027b9513e44c4f46ea Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:28:35 +0000 Subject: [PATCH 1/2] docs: update docs related to v0.10.0 release --- .../entity_relationships.schema.json | 4 +-- docs/user_guidance/custom_error_report.md | 28 ++++++++++++++++ docs/user_guidance/entity_relationships.md | 33 +++++++++++++++++-- docs/user_guidance/file_transformation.md | 32 ++++++++++++++++++ docs/user_guidance/install.md | 18 +++++----- includes/jargon_and_acronyms.md | 2 ++ zensical.toml | 1 + 7 files changed, 105 insertions(+), 13 deletions(-) create mode 100644 docs/user_guidance/custom_error_report.md diff --git a/docs/advanced_guidance/json_schemas/entity_relationships.schema.json b/docs/advanced_guidance/json_schemas/entity_relationships.schema.json index 0c2abbb2..6a641173 100644 --- a/docs/advanced_guidance/json_schemas/entity_relationships.schema.json +++ b/docs/advanced_guidance/json_schemas/entity_relationships.schema.json @@ -35,11 +35,11 @@ "no_valid_records_error_message": { "type": "string" }, - "empty_entity_error_code": { + "empty_entity_post_br_error_code": { "type": "string", "minLength": 1 }, - "empty_entity_error_message": { + "empty_entity_post_br_error_message": { "type": "string", "minLength": 1 } diff --git a/docs/user_guidance/custom_error_report.md b/docs/user_guidance/custom_error_report.md new file mode 100644 index 00000000..ebbac1fa --- /dev/null +++ b/docs/user_guidance/custom_error_report.md @@ -0,0 +1,28 @@ +--- +title: Customising your Error Report +tags: + - Feedback + - Error Reports +--- + +!!! note + + This feature is WIP and functionality will be expanded over time. + +This feature will allow you to customise elements of the error report generated. To use this functionality simply add the following to your dischema document: + +```json +{ + "contract": {}, + "transformations": {}, + "error_report": { + "record_rejection_reporting": false + } +} +``` + +## Current features available + +| Feature | Purpose | Default | +| ------- | ------- | ------- | +| record_rejection_reporting | Whether to show record rejection figures in the Data Summary Tab. This will remove "Total Number of Record Rejections" from the headlines figures and it will remove the "Record Rejection" type from the errors raised per entity. | True | diff --git a/docs/user_guidance/entity_relationships.md b/docs/user_guidance/entity_relationships.md index 68fe76d7..9375bcdb 100644 --- a/docs/user_guidance/entity_relationships.md +++ b/docs/user_guidance/entity_relationships.md @@ -9,9 +9,13 @@ tags: - Rejections --- +!!! note + + This feature is performed after all business rules have been applied. A `missing_parent` check will be performed, followed by a `no_valid_records` and then finally with a `missing_parent` check again (in case this results in new missing parents). Details on `missing_parent` and `no_valid_records` can be found within this page. + Sometimes a user may choose to use the file transformation stage to `normalise` a heavily nested dataset into separate entities during the initial reading of data. This would be done by specifying different entities in the dataset section of the contract configuration in the `dischema` file. This allows for easier interaction when customising errors in the data contract or writing transformations in the business rules. -`Normalising` assets can lead to more complex validations being required. For example in the flights dataset: +`Normalising` assets can lead to structural validations being required. For example in the flights dataset: ```mermaid erDiagram @@ -21,6 +25,19 @@ erDiagram AIRPORT ||--|{ STAFF_MEMBER : "" ``` +We now have a number of entity relationships generated from the normalised model. In the case above, the ERD states... + +- A Country must have at least one valid airport +- An Airport must have at least one valid country +- An Airport must have at least one valid staff member +- An Airport may have zero to many flights +- A Staff Member must have at least one valid Airport +- A Flights may have zero to many passengers +- A Flights must have at least one valid Airport +- A Passenger must have at least one valid Flight + +This creates a scenario where you would want to apply structual integrity checks to ensure that partial and/or invalid data is not passing validation. I.e. you wouldn't want an aiport flowing beyond validation if there is no valid country. + ### Missing Parent Records It could be that an airport record is deemed invalid and removed. Due to this, any flight records that linked to the now removed airport record are themselves invalid - a situation we refer to as a `missing_parent` issue, but are now existing in an entirely different entity. @@ -31,7 +48,19 @@ It could also be the case that staff records are a mandatory field for airport r ### Dischema -In order to perform these validations, how to link normalised entities needs to be provided. This can be specified in the `entity_relationships` section of the `dischema`. +In order to perform these validations, you must specify an `entity_relationships` section within the `dischema`. For example: + +```json +{ + "contract": {...}, + "transformations": {...}, + "entity_relationships": { + "my_entity": {...} + } +} +``` + +You can find complete examples [here](https://github.com/NHSDigital/data-validation-engine/tree/main/tests/testdata/flights/). ## Entity Relationships Content diff --git a/docs/user_guidance/file_transformation.md b/docs/user_guidance/file_transformation.md index 411b1e0f..ae34ace7 100644 --- a/docs/user_guidance/file_transformation.md +++ b/docs/user_guidance/file_transformation.md @@ -157,6 +157,38 @@ The secondary use of the File Transformation stage is the ability to normalise y !!! abstract "" You can read more about the readers and kwargs [here](../advanced_guidance/package_documentation/data_contract/readers.md). +## Reader additional checks + +The file transformation section supports additional reader checks which will apply across all reader configs for a given entity. These are the checks you can currently perform: + +### Check Empty + +The `check empty` feature allows you check whether an entity is empty on submission. When this error is raised, the DVE will generate a feedback message and report. This won't process any further entities as you are effectively saying this entity cannot be empty on submission. + +Here is an example of this being implemented in a dischema document: + +```json +{ + "contract": { + "datasets": { + "my_entity": { + "fields": {...}, + "reader_config": { + ".csv": {...}, + ".xml": {...} + }, + "reader_additional_checks": { + "check_empty": { + "error_code": "Empty entity", + "error_message": "'my_entity' is empty. Please resolve" + } + } + } + } + } +} +``` + ## Supported Formats | Format | DuckDB | Spark | Version Available | diff --git a/docs/user_guidance/install.md b/docs/user_guidance/install.md index 2c86b122..34a95faf 100644 --- a/docs/user_guidance/install.md +++ b/docs/user_guidance/install.md @@ -78,12 +78,12 @@ Once you have installed the DVE you are almost ready to use it. To be able to ru ## DVE Version Compatability Matrix -| DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version | -| ------------ | -------------- | ---------------- | --------------- | ---------------- | -| >=0.10.0 | >=3.10,<1.13 | __>=1.4,<1.4.5__ | >=3.5.0,<=3.5.5 | 2.13.4 | -| >=0.9.0 | >=3.10,<3.13 | 1.1.3 | >=3.5.0,<=3.5.5 | __2.13.4__ | -| >=0.8.0 | >=3.10,<3.13 | __1.1.3__ | __3.5.2__ | 1.10.19 | -| >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | __1.10.16__ | -| >=0.6 | >=3.10,<3.12 | __1.1.*__ | __3.4.*__ | 1.10.15 | -| >=0.2,<0.6 | __>=3.10,<3.12__ | 1.1.0 | 3.4.4 | 1.10.15 | -| 0.1 | >=3.7.2,<3.8 | 1.1.0 | 3.2.1 | 1.10.15 | +| DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version | +| ------------ | -------------------- | ---------------- | --------------- | ---------------- | +| >=0.10.0 | __>=3.11,<1.13__ | __>=1.4,<1.4.5__ | >=3.5.0,<=3.5.5 | 2.13.4 | +| >=0.9.0 | >=3.10,<3.13 | 1.1.3 | >=3.5.0,<=3.5.5 | __2.13.4__ | +| >=0.8.0 | >=3.10,<3.13 | __1.1.3__ | __3.5.2__ | 1.10.19 | +| >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | __1.10.16__ | +| >=0.6 | >=3.10,<3.12 | __1.1.*__ | __3.4.*__ | 1.10.15 | +| >=0.2,<0.6 | __>=3.10,<3.12__ | 1.1.0 | 3.4.4 | 1.10.15 | +| 0.1 | >=3.7.2,<3.8 | 1.1.0 | 3.2.1 | 1.10.15 | diff --git a/includes/jargon_and_acronyms.md b/includes/jargon_and_acronyms.md index 9b42c4e6..60c5e535 100644 --- a/includes/jargon_and_acronyms.md +++ b/includes/jargon_and_acronyms.md @@ -4,3 +4,5 @@ *[kwarg]: Key Word Arguments *[constr]: Constrained String *[immutable]: Unchanging over time or unable to be changed +*[ERD]: Entity Relationship Diagram +*[WIP]: Work in progress diff --git a/zensical.toml b/zensical.toml index fc47343e..4ae3b6c7 100644 --- a/zensical.toml +++ b/zensical.toml @@ -38,6 +38,7 @@ nav = [ {"Reporting" = [ {"Feedback Messages" = "user_guidance/feedback_messages.md"}, {"Error Reports" = "user_guidance/error_reports.md"}, + {"Customising your Error Report" = "user_guidance/custom_error_report.md"}, ]} ]}, {"Advanced User Guidance" = [ From a1efaddb1a7d24ecbe57c05a94f50dc087f16771 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Thu, 29 Oct 2026 10:25:35 +0000 Subject: [PATCH 2/2] docs: fix typo in upperbound python version for v0.10.0 --- docs/user_guidance/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user_guidance/install.md b/docs/user_guidance/install.md index 34a95faf..982c96a0 100644 --- a/docs/user_guidance/install.md +++ b/docs/user_guidance/install.md @@ -80,7 +80,7 @@ Once you have installed the DVE you are almost ready to use it. To be able to ru | DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version | | ------------ | -------------------- | ---------------- | --------------- | ---------------- | -| >=0.10.0 | __>=3.11,<1.13__ | __>=1.4,<1.4.5__ | >=3.5.0,<=3.5.5 | 2.13.4 | +| >=0.10.0 | __>=3.11,<3.13__ | __>=1.4,<1.4.5__ | >=3.5.0,<=3.5.5 | 2.13.4 | | >=0.9.0 | >=3.10,<3.13 | 1.1.3 | >=3.5.0,<=3.5.5 | __2.13.4__ | | >=0.8.0 | >=3.10,<3.13 | __1.1.3__ | __3.5.2__ | 1.10.19 | | >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | __1.10.16__ |