"` for this entity.'
)
return self
diff --git a/src/dve/pipeline/pipeline.py b/src/dve/pipeline/pipeline.py
index 1737ebf8..abff55e3 100644
--- a/src/dve/pipeline/pipeline.py
+++ b/src/dve/pipeline/pipeline.py
@@ -759,6 +759,8 @@ def apply_business_rules( # pylint: disable=R0914,R0915
final_projection
)
+ self.step_implementations.clear_entity_cache() # type: ignore
+
fh.remove_prefix(
fh.joinuri(
self.processed_files_path, submission_info.submission_id, "temp_business_rules"
@@ -775,7 +777,7 @@ def apply_business_rules( # pylint: disable=R0914,R0915
self.processed_files_path,
submission_info.submission_id,
"data_contract",
- rules.global_variables.get('entity', submission_info.dataset_id)
+ rules.global_variables.get("entity", submission_info.dataset_id),
)
)
)
diff --git a/src/dve/reporting/__init__.py b/src/dve/reporting/__init__.py
index ab78e113..476e0d0d 100644
--- a/src/dve/reporting/__init__.py
+++ b/src/dve/reporting/__init__.py
@@ -1,2 +1,3 @@
"""Error reports module."""
+
# pylint: disable=R0801
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py
index ecef834b..aae56967 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py
@@ -1,6 +1,9 @@
"""Test DuckDB backend steps."""
# pylint: disable=redefined-outer-name,unused-import,line-too-long
+import datetime
+from io import StringIO
+import json
import tempfile
from pathlib import Path
from typing import Iterator, List, Optional, Set, Tuple, Type
@@ -884,3 +887,112 @@ def test_read_and_write_nested_parquet(nested_typecast_parquet):
"datetimefield": "TIMESTAMP",
"subfield": "STRUCT(id BIGINT, substrfield VARCHAR, subarrayfield DATE[])[]",
}
+
+def test_cache_management():
+ conn = DUCKDB_STEP_BACKEND.connection
+ with tempfile.NamedTemporaryFile(mode="w") as tf1, tempfile.NamedTemporaryFile(mode="w") as tf2:
+ td1 = [
+ {"greeting": "hi", "num_one": 2, "num_two": 4, "test_date": datetime.date(2020,5,1), "active": True},
+ {"greeting": "bonjour", "num_one": 3, "num_two": 9, "test_date": datetime.date(2025,7,4), "active": False},
+ ]
+ tf1.write(json.dumps(td1, default=str))
+
+ tf1.seek(0)
+
+ td1_schema = {"greeting": "STRING", "num_one": "BIGINT", "num_two" :"BIGINT", "test_date": "DATE", "active": "BOOLEAN"}
+
+ td2 = [
+ {"farewell": "aurevoir", "lots_of_nums": [1,2,3], "nested_field": {"nested_str": "test1", "nested_timestamp": datetime.datetime(2024,3,5,1,2,3)}},
+ {"farewell": "bye", "lots_of_nums": [4,5,6,7,8], "nested_field": {"nested_str": "test2", "nested_timestamp": datetime.datetime(2022,8,12,10,12,14)}},
+ ]
+
+ tf2.write(json.dumps(td2, default=str))
+
+ tf2.seek(0)
+
+ td2_schema = {"farewell": "STRING", "lots_of_nums": "BIGINT[]", "nested_field": "STRUCT(nested_str STRING, nested_timestamp TIMESTAMP)"}
+
+ em = EntityManager({})
+ em.entities["test_one"] = conn.read_json(tf1.name, columns=td1_schema)
+ em.entities["test_two"] = conn.read_json(tf2.name, columns=td2_schema)
+
+ DUCKDB_STEP_BACKEND.cache_entity("test_one", em.entities)
+
+ cached_tables = [rw["table_name"] for rw in conn.sql("SELECT table_name from duckdb_tables() WHERE database_name = 'temp'").pl().to_dicts()]
+
+ test_one_temp_name = list(filter(lambda x: x.startswith("test_one_"), cached_tables))[0]
+
+ assert "test_one" in DUCKDB_STEP_BACKEND.entity_cache_tracker
+ assert DUCKDB_STEP_BACKEND.entity_cache_tracker["test_one"] == test_one_temp_name
+ assert sorted(em.entities["test_one"].pl().to_dicts(), key=lambda x: x.get("test_date")) == td1
+ assert sorted(conn.table(test_one_temp_name).pl().to_dicts(), key=lambda x: x.get("test_date")) == td1
+ assert "test_two" not in DUCKDB_STEP_BACKEND.entity_cache_tracker
+
+ DUCKDB_STEP_BACKEND.cache_entity("test_two", em.entities)
+ DUCKDB_STEP_BACKEND._remove_cached_artifact("test_one")
+
+ cached_tables = [rw["table_name"] for rw in conn.sql("SELECT table_name from duckdb_tables() WHERE database_name = 'temp'").pl().to_dicts()]
+ test_two_temp_name = list(filter(lambda x: x.startswith("test_two_"), cached_tables))[0]
+
+ assert "test_two" in DUCKDB_STEP_BACKEND.entity_cache_tracker
+ assert DUCKDB_STEP_BACKEND.entity_cache_tracker["test_two"] in cached_tables
+ assert sorted(em.entities["test_two"].pl().to_dicts(), key=lambda x: x.get("farewell")) == td2
+ assert sorted(conn.table(test_two_temp_name).pl().to_dicts(), key=lambda x: x.get("farewell")) == td2
+ assert "test_one" not in DUCKDB_STEP_BACKEND.entity_cache_tracker
+
+ DUCKDB_STEP_BACKEND.clear_entity_cache()
+
+ cached_tables = [rw["table_name"] for rw in conn.sql("SELECT table_name from duckdb_tables() WHERE database_name = 'temp'").pl().to_dicts()]
+
+ assert not DUCKDB_STEP_BACKEND.entity_cache_tracker
+ assert not any(tbl.startswith("test_one_") for tbl in cached_tables)
+ assert not any(tbl.startswith("test_two_") for tbl in cached_tables)
+
+def test_cache_management_with_update():
+ conn = DUCKDB_STEP_BACKEND.connection
+ with tempfile.NamedTemporaryFile(mode="w") as tf, tempfile.NamedTemporaryFile(mode="w") as ef:
+ td = [
+ {"idx": 1, "lots_of_nums": [1,2,3], "nested_field": {"nested_str": "test1", "nested_timestamp": datetime.datetime(2024,3,5,1,2,3)}},
+ {"idx": 2, "lots_of_nums": [4,5,6,7,8], "nested_field": {"nested_str": "test2", "nested_timestamp": datetime.datetime(2022,8,12,10,12,14)}},
+ ]
+
+ tf.write(json.dumps(td, default=str))
+ tf.seek(0)
+ td_schema = {"idx": "BIGINT",
+ "lots_of_nums": "BIGINT[]",
+ "nested_field": "STRUCT(nested_str STRING, nested_timestamp TIMESTAMP)"}
+
+
+ em = EntityManager({})
+ em.entities["test_df"] = conn.read_json(tf.name, columns=td_schema)
+
+ DUCKDB_STEP_BACKEND.cache_entity("test_df", em.entities)
+
+ cached_tables = [rw["table_name"] for rw in conn.sql("SELECT table_name from duckdb_tables() WHERE database_name = 'temp'").pl().to_dicts()]
+
+ first_temp_name = list(filter(lambda x: x.startswith("test_df_"), cached_tables))[0]
+
+ assert "test_df" in DUCKDB_STEP_BACKEND.entity_cache_tracker
+ assert DUCKDB_STEP_BACKEND.entity_cache_tracker["test_df"] == first_temp_name
+ assert sorted(em.entities["test_df"].pl().to_dicts(), key=lambda x: x.get("idx")) == td
+
+ extra_data = [{"idx": 3, "lots_of_nums": [9], "nested_field": {"nested_str": "test3", "nested_timestamp": datetime.datetime(2024,1,9,3,2,1)}}]
+ ef.write(json.dumps(extra_data, default=str))
+ ef.seek(0)
+ em.entities["test_df"] = em.entities["test_df"].union(conn.read_json(ef.name, columns=td_schema))
+ DUCKDB_STEP_BACKEND.cache_entity("test_df", em.entities)
+
+ cached_tables = [rw["table_name"] for rw in conn.sql("SELECT table_name from duckdb_tables() WHERE database_name = 'temp'").pl().to_dicts()]
+ tables_of_interest = list(filter(lambda x: x.startswith("test_df_"), cached_tables))
+ assert len(tables_of_interest) == 1
+ assert first_temp_name not in tables_of_interest
+ second_temp_name = tables_of_interest[0]
+ assert sorted(em.entities["test_df"].pl().to_dicts(), key=lambda x: x.get("idx")) == td + extra_data
+ assert sorted(conn.table(second_temp_name).pl().to_dicts(), key=lambda x: x.get("idx")) == td + extra_data
+
+ DUCKDB_STEP_BACKEND.clear_entity_cache()
+
+ cached_tables = [rw["table_name"] for rw in conn.sql("SELECT table_name from duckdb_tables() WHERE database_name = 'temp'").pl().to_dicts()]
+
+ assert not DUCKDB_STEP_BACKEND.entity_cache_tracker
+ assert not any(tbl.startswith("test_df_") for tbl in cached_tables)
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_rules.py b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_rules.py
index 6e38c7a9..2a79d81b 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_rules.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_rules.py
@@ -1,6 +1,7 @@
"""Test Spark backend steps."""
# pylint: disable=redefined-outer-name,unused-import,line-too-long
+import datetime
from pathlib import Path
from typing import List, Optional, Set, Tuple, Type
@@ -9,6 +10,7 @@
from pyspark.sql.functions import col, lit
from pyspark.sql.types import (
ArrayType,
+ BooleanType,
DateType,
LongType,
Row,
@@ -852,3 +854,107 @@ def test_read_and_write_nested_parquet(nested_typecast_parquet):
),
]
)
+
+def test_cache_management():
+ spark = SPARK_STEP_BACKEND._spark_session
+ td1 = [
+ {"greeting": "hi", "num_one": 2, "num_two": 4, "test_date": datetime.date(2020,5,1), "active": True},
+ {"greeting": "bonjour", "num_one": 3, "num_two": 9, "test_date": datetime.date(2025,7,4), "active": False},
+ ]
+ td1_schema = StructType([
+ StructField("greeting", StringType()),
+ StructField("num_one", LongType()),
+ StructField("num_two", LongType()),
+ StructField("test_date", DateType()),
+ StructField("active", BooleanType())])
+ td2 = [
+ {"farewell": "aurevoir", "lots_of_nums": [1,2,3], "nested_field": {"nested_str": "test1", "nested_timestamp": datetime.datetime(2024,3,5,1,2,3)}},
+ {"farewell": "bye", "lots_of_nums": [4,5,6,7,8], "nested_field": {"nested_str": "test2", "nested_timestamp": datetime.datetime(2022,8,12,10,12,14)}},
+ ]
+ td2_schema = StructType(
+ [
+ StructField("farewell", StringType()),
+ StructField("lots_of_nums", ArrayType(LongType())),
+ StructField("nested_field", StructType([StructField("nested_str", StringType()), StructField("nested_timestamp", TimestampType())]))
+ ]
+ )
+ em = EntityManager({})
+ em.entities["test_one"] = spark.createDataFrame(td1, schema=td1_schema)
+ em.entities["test_two"] = spark.createDataFrame(td2, schema=td2_schema)
+
+ SPARK_STEP_BACKEND.cache_entity("test_one", em.entities)
+
+ cached_tables = (rw.tableName for rw in spark.sql("SHOW TABLES").collect())
+
+ test_one_temp_name = list(filter(lambda x: x.startswith("test_one_"), cached_tables))[0]
+
+ assert "test_one" in SPARK_STEP_BACKEND.entity_cache_tracker
+ assert SPARK_STEP_BACKEND.entity_cache_tracker["test_one"] == test_one_temp_name
+ assert sorted([rw.asDict(True) for rw in em.entities["test_one"].collect()], key=lambda x: x.get("test_date")) == td1
+ assert sorted([rw.asDict(True) for rw in spark.table(test_one_temp_name).collect()], key=lambda x: x.get("test_date")) == td1
+ assert "test_two" not in SPARK_STEP_BACKEND.entity_cache_tracker
+
+ SPARK_STEP_BACKEND.cache_entity("test_two", em.entities)
+ SPARK_STEP_BACKEND._remove_cached_artifact("test_one")
+
+ cached_tables = list(rw.tableName for rw in spark.sql("SHOW TABLES").collect())
+ test_two_temp_name = list(filter(lambda x: x.startswith("test_two_"), cached_tables))[0]
+
+ assert "test_two" in SPARK_STEP_BACKEND.entity_cache_tracker
+ assert SPARK_STEP_BACKEND.entity_cache_tracker["test_two"] in cached_tables
+ assert sorted([rw.asDict(True) for rw in em.entities["test_two"].collect()], key=lambda x: x.get("farewell")) == td2
+ assert sorted([rw.asDict(True) for rw in spark.table(test_two_temp_name).collect()], key=lambda x: x.get("farewell")) == td2
+ assert "test_one" not in SPARK_STEP_BACKEND.entity_cache_tracker
+
+ SPARK_STEP_BACKEND.clear_entity_cache()
+
+ cached_tables = list(rw.tableName for rw in spark.sql("SHOW TABLES").collect())
+
+ assert not SPARK_STEP_BACKEND.entity_cache_tracker
+ assert not any(tbl.startswith("test_one_") for tbl in cached_tables)
+ assert not any(tbl.startswith("test_two_") for tbl in cached_tables)
+
+def test_cache_management_with_update():
+ spark = SPARK_STEP_BACKEND._spark_session
+ td = [
+ {"idx": 1, "lots_of_nums": [1,2,3], "nested_field": {"nested_str": "test1", "nested_timestamp": datetime.datetime(2024,3,5,1,2,3)}},
+ {"idx": 2, "lots_of_nums": [4,5,6,7,8], "nested_field": {"nested_str": "test2", "nested_timestamp": datetime.datetime(2022,8,12,10,12,14)}},
+ ]
+ td_schema = StructType(
+ [
+ StructField("idx", LongType()),
+ StructField("lots_of_nums", ArrayType(LongType())),
+ StructField("nested_field", StructType([StructField("nested_str", StringType()), StructField("nested_timestamp", TimestampType())]))
+ ]
+ )
+ em = EntityManager({})
+ em.entities["test_df"] = spark.createDataFrame(td, schema=td_schema)
+
+ SPARK_STEP_BACKEND.cache_entity("test_df", em.entities)
+
+ cached_tables = (rw.tableName for rw in spark.sql("SHOW TABLES").collect())
+
+ first_temp_name = list(filter(lambda x: x.startswith("test_df_"), cached_tables))[0]
+
+ assert "test_df" in SPARK_STEP_BACKEND.entity_cache_tracker
+ assert SPARK_STEP_BACKEND.entity_cache_tracker["test_df"] == first_temp_name
+ assert sorted([rw.asDict(True) for rw in em.entities["test_df"].collect()], key=lambda x: x.get("idx")) == td
+
+ extra_data = [{"idx": 3, "lots_of_nums": [9], "nested_field": {"nested_str": "test3", "nested_timestamp": datetime.datetime(2024,1,9,3,2,1)}}]
+ em.entities["test_df"] = em.entities["test_df"].union(spark.createDataFrame(extra_data, schema=td_schema))
+ SPARK_STEP_BACKEND.cache_entity("test_df", em.entities)
+
+ cached_tables = (rw.tableName for rw in spark.sql("SHOW TABLES").collect())
+ tables_of_interest = list(filter(lambda x: x.startswith("test_df_"), cached_tables))
+ assert len(tables_of_interest) == 1
+ assert first_temp_name not in tables_of_interest
+ second_temp_name = tables_of_interest[0]
+ assert sorted([rw.asDict(True) for rw in em.entities["test_df"].collect()], key=lambda x: x.get("idx")) == td + extra_data
+ assert sorted([rw.asDict(True) for rw in spark.table(second_temp_name).collect()], key=lambda x: x.get("idx")) == td + extra_data
+
+ SPARK_STEP_BACKEND.clear_entity_cache()
+
+ cached_tables = list(rw.tableName for rw in spark.sql("SHOW TABLES").collect())
+
+ assert not SPARK_STEP_BACKEND.entity_cache_tracker
+ assert not any(tbl.startswith("test_df_") for tbl in cached_tables)
From df8bbe4e2f7b26b007b36a3fe144232aeee14a00 Mon Sep 17 00:00:00 2001
From: George Robertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Thu, 8 Oct 2026 11:35:26 +0100
Subject: [PATCH 27/47] feat: add orphan and group rejections to spark backend
(#174)
---
src/dve/core_engine/backends/base/reader.py | 2 +-
src/dve/core_engine/backends/base/rules.py | 19 +-
src/dve/core_engine/backends/exceptions.py | 5 +-
.../implementations/duckdb/readers/csv.py | 9 +-
.../backends/implementations/duckdb/rules.py | 15 --
.../implementations/spark/readers/xml.py | 3 +-
.../backends/implementations/spark/rules.py | 107 ++++++++-
.../core_engine/backends/readers/utilities.py | 2 +-
src/dve/pipeline/pipeline.py | 3 +-
tests/features/flights.feature | 78 +++++++
.../flights/flights_spark.dischema.json | 212 ++++++++++++++++++
11 files changed, 422 insertions(+), 33 deletions(-)
create mode 100644 tests/testdata/flights/flights_spark.dischema.json
diff --git a/src/dve/core_engine/backends/base/reader.py b/src/dve/core_engine/backends/base/reader.py
index 93b97071..eeeca7f4 100644
--- a/src/dve/core_engine/backends/base/reader.py
+++ b/src/dve/core_engine/backends/base/reader.py
@@ -11,7 +11,7 @@
from dve.core_engine.backends.exceptions import (
CriticalMessageBearingError,
MessageBearingError,
- ReaderLacksEntityTypeSupport
+ ReaderLacksEntityTypeSupport,
)
from dve.core_engine.backends.types import EntityName, EntityType
from dve.core_engine.configuration.v1 import (
diff --git a/src/dve/core_engine/backends/base/rules.py b/src/dve/core_engine/backends/base/rules.py
index 2471426c..6200e013 100644
--- a/src/dve/core_engine/backends/base/rules.py
+++ b/src/dve/core_engine/backends/base/rules.py
@@ -427,7 +427,13 @@ def process_node(node: HierarchyNode) -> bool:
self.cache_entity(node.entity_name, entities)
- return len(_messages) > 0
+ _orph_count = len(_messages)
+
+ self.logger.info(
+ f"Found {_orph_count} orphan records between {node.entity_name} and {node.parent_entity}" # pylint: disable=C0301
+ )
+
+ return _orph_count > 0
entity_issues_found: dict[EntityName, bool] = {}
@@ -454,7 +460,7 @@ def process_node(node: HierarchyNode) -> bool:
return False
self.logger.info(
- f"Identifying that mandatory entity `{node.parent_entity}` has at least 1 valid child record" # pylint: disable=C0301
+ f"Identifying that mandatory entity `{node.parent_entity}` has at least 1 valid child record in {node.entity_name}" # pylint: disable=C0301
)
join_expr = " AND ".join(
@@ -493,7 +499,14 @@ def process_node(node: HierarchyNode) -> bool:
]
msg_writer.write_queue.put(_messages)
self.cache_entity(node.parent_entity, entities)
- return len(_messages) > 0
+
+ _no_valid_child_records: int = len(_messages)
+
+ self.logger.info(
+ f"Found {_no_valid_child_records} records with no valid children in {node.parent_entity}." # pylint: disable=C0301
+ )
+
+ return _no_valid_child_records > 0
entity_issues_found: dict[EntityName, bool] = {}
diff --git a/src/dve/core_engine/backends/exceptions.py b/src/dve/core_engine/backends/exceptions.py
index f8d93079..99808c8a 100644
--- a/src/dve/core_engine/backends/exceptions.py
+++ b/src/dve/core_engine/backends/exceptions.py
@@ -32,6 +32,7 @@ def __init__(self, *args: object, messages: Messages) -> None:
self.messages = messages
"""The messages to be returned as part of the error."""
+
class CriticalMessageBearingError(BackendError):
"""
A backend error that comes with a pre-created message.
@@ -44,6 +45,7 @@ def __init__(self, *args: object, message: FeedbackMessage) -> None:
self.message = message
"""The message to be returned as part of the error."""
+
class UnableToParseCSVError(CriticalMessageBearingError):
"""An error raised when unable to parse a CSV file"""
@@ -60,7 +62,8 @@ def __init__(
failure_type="submission",
is_informational=False,
error_type="csv read",
- error_message=error_message or "Unable to parse the CSV file. Please check the structure of your CSV.", # pylint: disable=C0301
+ error_message=error_message
+ or "Unable to parse the CSV file. Please check the structure of your CSV.", # pylint: disable=C0301
error_code=error_code or "MalformedCSV",
)
)
diff --git a/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py b/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py
index dedd66bd..488f8c85 100644
--- a/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py
+++ b/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py
@@ -123,7 +123,8 @@ def read_to_relation( # pylint: disable=unused-argument
raise UnableToParseCSVError(
entity_name="csv_structure",
error_code=self.ft_error_code,
- error_message=self.ft_error_message or "Unable to parse CSV file. Structure is likely malformed.", # pylint: disable=C0301
+ error_message=self.ft_error_message
+ or "Unable to parse CSV file. Structure is likely malformed.", # pylint: disable=C0301
) from exc
if self.null_empty_strings:
@@ -183,7 +184,8 @@ def read_to_relation( # pylint: disable=unused-argument
raise UnableToParseCSVError(
entity_name="csv_structure",
error_code=self.ft_error_code,
- error_message=self.ft_error_message or "Unable to parse CSV file. Structure is likely malformed.", # pylint: disable=C0301
+ error_message=self.ft_error_message
+ or "Unable to parse CSV file. Structure is likely malformed.", # pylint: disable=C0301
) from exc
if self.null_empty_strings:
@@ -200,7 +202,8 @@ def read_to_relation( # pylint: disable=unused-argument
raise UnableToParseCSVError(
entity_name="csv_structure",
error_code=self.ft_error_code,
- error_message=self.ft_error_message or "Found zero records after loading CSV. File is likely malformed.", # pylint: disable=C0301
+ error_message=self.ft_error_message
+ or "Found zero records after loading CSV. File is likely malformed.", # pylint: disable=C0301
)
return entity
diff --git a/src/dve/core_engine/backends/implementations/duckdb/rules.py b/src/dve/core_engine/backends/implementations/duckdb/rules.py
index 24910d33..766142aa 100644
--- a/src/dve/core_engine/backends/implementations/duckdb/rules.py
+++ b/src/dve/core_engine/backends/implementations/duckdb/rules.py
@@ -421,12 +421,6 @@ def identify_orphans(
.set_alias("orphan")
)
- if relation_is_empty(orphaned_rel):
- self.logger.info(
- f"Found 0 orphan records between {config.entity_name} and {config.target_name}"
- )
- return []
-
message_rel = (
entities[config.entity_name]
.set_alias(config.entity_name)
@@ -474,15 +468,6 @@ def check_mandatory_group(
missing_children_rel = joined_rel.filter("fk IS NULL")
filtered_rel = joined_rel.filter("fk IS NOT NULL").select(StarExpression(exclude=["fk"]))
- _no_valid_child_records: tuple[int] = missing_children_rel.count("*").fetchone() # type: ignore # pylint: disable=C0301
- if _no_valid_child_records:
- _no_valid_children = _no_valid_child_records[0]
- else:
- _no_valid_children = 0
- self.logger.info(
- f"Found {_no_valid_children} records with no valid children in {config.entity_name}."
- ) # pylint: disable=C0301
-
entities[config.entity_name] = filtered_rel
return duckdb_rel_to_dictionaries(missing_children_rel)
diff --git a/src/dve/core_engine/backends/implementations/spark/readers/xml.py b/src/dve/core_engine/backends/implementations/spark/readers/xml.py
index 275182e8..769c480e 100644
--- a/src/dve/core_engine/backends/implementations/spark/readers/xml.py
+++ b/src/dve/core_engine/backends/implementations/spark/readers/xml.py
@@ -44,6 +44,7 @@ def read_to_dataframe(
resource: URI,
entity_name: EntityName,
schema: type[BaseModel],
+ all_model_fields: Optional[set[str]] = None,
) -> DataFrame:
"""Stream an XML file into a Spark data frame"""
if not self.spark:
@@ -51,7 +52,7 @@ def read_to_dataframe(
spark_schema = get_type_from_annotation(schema)
return self.add_record_index(
self.spark.createDataFrame( # type: ignore
- list(self.read_to_py_iterator(resource, entity_name, schema)),
+ list(self.read_to_py_iterator(resource, entity_name, schema, all_model_fields)),
schema=spark_schema,
)
)
diff --git a/src/dve/core_engine/backends/implementations/spark/rules.py b/src/dve/core_engine/backends/implementations/spark/rules.py
index 5f218ec6..8cdc8a1b 100644
--- a/src/dve/core_engine/backends/implementations/spark/rules.py
+++ b/src/dve/core_engine/backends/implementations/spark/rules.py
@@ -1,7 +1,7 @@
"""Step implementations in Spark."""
# pylint: disable=R0801
-from collections.abc import Callable, Iterator
+from collections.abc import Callable, Iterable, Iterator
from typing import Optional
from uuid import uuid4
@@ -13,6 +13,7 @@
from dve.core_engine.backends.exceptions import ConstraintError
from dve.core_engine.backends.implementations.spark.spark_helpers import (
create_udf,
+ df_is_empty,
get_all_registered_udfs,
object_to_spark_literal,
spark_filter_contract_errors,
@@ -48,6 +49,7 @@
SemiJoin,
TableUnion,
)
+from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
from dve.core_engine.functions import implementations as functions
from dve.core_engine.message import FeedbackMessage
from dve.core_engine.templating import template_object
@@ -341,16 +343,107 @@ def union(self, entities: SparkEntities, *, config: TableUnion) -> Messages:
return []
def identify_orphans(
- self, entities: SparkEntities, *, config: OrphanIdentification
- ) -> tuple[Messages, int]:
- # TODO - adjust this to new setup of identify and remove orphans
- raise NotImplementedError
+ self,
+ entities: SparkEntities,
+ *,
+ config: OrphanIdentification,
+ ) -> Iterable:
+ """Identify records in an entity which don't have at least one corresponding
+ match in the target. A new boolean column will be added to `entity` ('IsOrphaned')
+ indicating whether the condition matched.
+
+ If there is already an 'IsOrphaned' column in the entity, this will be set to the
+ logical OR of its current value and the value it would have been set to otherwise.
+
+ """
+ source_df: DataFrame = entities[config.entity_name]
+ source_df = source_df.alias(config.entity_name)
+
+ if df_is_empty(source_df):
+ self.logger.info(f"{config.entity_name} is empty. Skipping orphan check.")
+ return
+
+ target_df: DataFrame = entities[config.target_name]
+ match_name = f"matched_{uuid4().hex}"
+ target_df = target_df.select("*", sf.lit(1).alias(match_name)).alias(config.target_name)
+
+ orphaned_df: DataFrame = (
+ source_df.join(target_df, on=sf.expr(config.join_condition), how="left")
+ .groupBy(f"{config.entity_name}.{config.id}")
+ .agg(
+ sf.first(f"{config.entity_name}.{RECORD_INDEX_COLUMN_NAME}").alias(
+ RECORD_INDEX_COLUMN_NAME
+ ), # pylint: disable=C0301
+ (sf.coalesce(sf.count(match_name), sf.lit(0)) == sf.lit(0)).alias("IsOrphaned"),
+ )
+ .filter(sf.col("IsOrphaned"))
+ .select(RECORD_INDEX_COLUMN_NAME)
+ .alias("orphan")
+ )
+
+ message_df = (
+ entities[config.entity_name]
+ .alias(config.entity_name)
+ .join(
+ orphaned_df,
+ sf.expr(
+ f"{config.entity_name}.{RECORD_INDEX_COLUMN_NAME} = orphan.{RECORD_INDEX_COLUMN_NAME}", # pylint: disable=C0301
+ ),
+ "semi",
+ )
+ )
+ filtered_rel = (
+ entities[config.entity_name]
+ .alias(config.entity_name)
+ .join(
+ orphaned_df,
+ sf.expr(
+ f"{config.entity_name}.{RECORD_INDEX_COLUMN_NAME} = orphan.{RECORD_INDEX_COLUMN_NAME}", # pylint: disable=C0301
+ ),
+ "anti",
+ )
+ )
+
+ entities[config.entity_name] = filtered_rel
+
+ for r in message_df.toLocalIterator():
+ yield r.asDict()
def check_mandatory_group(
self, entities: SparkEntities, *, config: GroupIdentification
) -> Iterator:
- # TODO - implement for spark
- raise NotImplementedError
+ """
+ Check that a mandatory key in an entity has at least one valid entry in the all the
+ child entities.
+ """
+ source_df: DataFrame = entities[config.entity_name]
+ source_df = source_df.alias(config.entity_name)
+ target_df: DataFrame = entities[config.target_name]
+ target_df = target_df.alias(config.target_name)
+
+ source_columns = [f"{config.entity_name}.{c.strip()}" for c in source_df.columns]
+ _pk, fk = config.join_condition.split("=")
+
+ joined_df = source_df.join(target_df, sf.expr(config.join_condition), "left").select(
+ *source_columns,
+ sf.col(fk.strip()).alias("fk"),
+ )
+
+ missing_children_df = joined_df.filter("fk IS NULL")
+ filtered_df = joined_df.filter("fk IS NOT NULL").select("*").drop(sf.col("fk"))
+
+ if not df_is_empty(missing_children_df):
+ _no_valid_children = missing_children_df.count()
+ else:
+ _no_valid_children = 0
+ self.logger.info(
+ f"Found {_no_valid_children} records with no valid children in {config.entity_name}."
+ )
+
+ entities[config.entity_name] = filtered_df
+
+ for r in missing_children_df.toLocalIterator():
+ yield r.asDict()
def filter(self, entities: SparkEntities, *, config: ImmediateFilter) -> Messages:
"""Filter an entity immediately, and do not emit any messages.
diff --git a/src/dve/core_engine/backends/readers/utilities.py b/src/dve/core_engine/backends/readers/utilities.py
index 99ca6abc..86347dce 100644
--- a/src/dve/core_engine/backends/readers/utilities.py
+++ b/src/dve/core_engine/backends/readers/utilities.py
@@ -66,7 +66,7 @@ def raise_message_bearing_error_on_header_differences(
reporting_field="csv_header",
error_code=field_check_error_code,
error_message=field_check_error_message,
- )
+ ),
)
diff --git a/src/dve/pipeline/pipeline.py b/src/dve/pipeline/pipeline.py
index abff55e3..2cf00553 100644
--- a/src/dve/pipeline/pipeline.py
+++ b/src/dve/pipeline/pipeline.py
@@ -764,7 +764,8 @@ def apply_business_rules( # pylint: disable=R0914,R0915
fh.remove_prefix(
fh.joinuri(
self.processed_files_path, submission_info.submission_id, "temp_business_rules"
- )
+ ),
+ recursive=True,
)
self.check_mandatory_entities_have_records(
diff --git a/tests/features/flights.feature b/tests/features/flights.feature
index bcfad686..3bf2ee06 100644
--- a/tests/features/flights.feature
+++ b/tests/features/flights.feature
@@ -286,3 +286,81 @@ Feature: Pipeline tests using the flights dataset
And the latest audit record for the submission is marked with processing status error_report
When I run the error report phase
Then An error report is produced
+
+ Scenario: A flights submission with a rejection on a node with two mandatory nodes (spark)
+ Given I submit the flights file multi_node_file_rejection.xml for processing
+ And A spark pipeline is configured with schema file 'flights_spark.dischema.json'
+ And I add initial audit entries for the submission
+ Then the latest audit record for the submission is marked with processing status file_transformation
+ When I run the file transformation phase
+ Then the country entity is stored as a parquet after the file_transformation phase
+ And the airport entity is stored as a parquet after the file_transformation phase
+ And the flights entity is stored as a parquet after the file_transformation phase
+ And the passengers entity is stored as a parquet after the file_transformation phase
+ And the latest audit record for the submission is marked with processing status data_contract
+ When I run the data contract phase
+ Then there are no file rejections from the data_contract phase
+ And there are no record rejections from the data_contract phase
+ When I run the business rules phase
+ Then there are errors with the following details and associated error_count from the business_rules phase
+ | ErrorType | Status | ErrorCode | error_count |
+ | record | error | StaffIDMissing | 7 |
+ | record | error | AirportHasNoStaff | 1 |
+ | record | error | FlightHasNoAirport | 1 |
+ | record | error | PassengerHasNoFlight | 1 |
+ And the final entities have the following row counts
+ | entity_name | row_count |
+ | country | 1 |
+ | airport | 1 |
+ | staff | 1 |
+ | flights | 1 |
+ | passengers | 1 |
+ When I run the error report phase
+ Then An error report is produced
+ And The statistics entry for the submission shows the following information
+ | parameter | value |
+ | record_count | 1 |
+ | number_submission_rejections | 0 |
+ | number_record_rejections | 10 |
+ | number_warnings | 0 |
+
+ Scenario: A flights submission with many types of rejections in a single submission (spark)
+ Given I submit the flights file flights_full_regression.xml for processing
+ And A spark pipeline is configured with schema file 'flights_spark.dischema.json'
+ And I add initial audit entries for the submission
+ Then the latest audit record for the submission is marked with processing status file_transformation
+ When I run the file transformation phase
+ Then the country entity is stored as a parquet after the file_transformation phase
+ And the airport entity is stored as a parquet after the file_transformation phase
+ And the flights entity is stored as a parquet after the file_transformation phase
+ And the passengers entity is stored as a parquet after the file_transformation phase
+ And the latest audit record for the submission is marked with processing status data_contract
+ When I run the data contract phase
+ Then there are errors with the following details and associated error_count from the data_contract phase
+ | FailureType | ErrorCode | error_count |
+ | record | AirportIdIsMissing | 1 |
+ When I run the business rules phase
+ Then there are errors with the following details and associated error_count from the business_rules phase
+ | ErrorType | Status | ErrorCode | error_count |
+ | record | error | InvalidFlightDestination | 1 |
+ | record | error | PassengerNameMissing | 1 |
+ | record | error | StaffIDMissing | 4 |
+ | record | error | PassengerHasNoFlight | 3 |
+ | record | error | StaffHasNoAirport | 1 |
+ | record | error | FlightHasNoAirport | 2 |
+ | record | error | AirportHasNoStaff | 1 |
+ And the final entities have the following row counts
+ | entity_name | row_count |
+ | country | 1 |
+ | airport | 3 |
+ | staff | 3 |
+ | flights | 3 |
+ | passengers | 2 |
+ When I run the error report phase
+ Then An error report is produced
+ And The statistics entry for the submission shows the following information
+ | parameter | value |
+ | record_count | 1 |
+ | number_submission_rejections | 0 |
+ | number_record_rejections | 14 |
+ | number_warnings | 0 |
diff --git a/tests/testdata/flights/flights_spark.dischema.json b/tests/testdata/flights/flights_spark.dischema.json
new file mode 100644
index 00000000..b318f823
--- /dev/null
+++ b/tests/testdata/flights/flights_spark.dischema.json
@@ -0,0 +1,212 @@
+{
+ "contract": {
+ "schemas": {
+ "passengers": {
+ "fields": {
+ "flight_id": "int",
+ "passenger_id": "int",
+ "passenger_name": "str"
+ }
+ }
+ },
+ "error_details": "flights_data_contract_error_details.json",
+ "datasets": {
+ "country": {
+ "fields": {
+ "country_id": "int",
+ "country_name": "str"
+ },
+ "reader_config": {
+ ".xml": {
+ "reader": "SparkXMLStreamReader",
+ "kwargs": {
+ "record_tag": "country",
+ "root_tag": "country"
+ }
+ }
+ },
+ "key_field": "country_id",
+ "mandatory_fields": [
+ "country_id",
+ "country_name"
+ ]
+ },
+ "airport": {
+ "fields": {
+ "country_id": "int",
+ "airport_id": "int",
+ "airport_name": "str",
+ "postcode": "str"
+ },
+ "reader_config": {
+ ".xml": {
+ "reader": "SparkXMLStreamReader",
+ "kwargs": {
+ "record_tag": "airport",
+ "root_tag": "country"
+ }
+ }
+ },
+ "key_field": "airport_id",
+ "mandatory_fields": [
+ "airport_id"
+ ]
+ },
+ "staff": {
+ "fields": {
+ "airport_id": "int",
+ "staff_id": "int",
+ "staff_name": "str",
+ "role": "str"
+ },
+ "reader_config": {
+ ".xml": {
+ "reader": "SparkXMLStreamReader",
+ "kwargs": {
+ "record_tag": "staff_member",
+ "root_tag": "country"
+ }
+ }
+ },
+ "key_field": "staff_id"
+ },
+ "flights": {
+ "fields": {
+ "airport_id": "int",
+ "flight_id": "int",
+ "destination": "str"
+ },
+ "reader_config": {
+ ".xml": {
+ "reader": "SparkXMLStreamReader",
+ "kwargs": {
+ "record_tag": "flight",
+ "root_tag": "country"
+ }
+ }
+ },
+ "key_field": "flight_id"
+ },
+ "passengers": {
+ "fields": {
+ "flight_id": "int",
+ "passenger_id": "int",
+ "passenger_name": "str"
+ },
+ "reader_config": {
+ ".xml": {
+ "reader": "SparkXMLStreamReader",
+ "kwargs": {
+ "record_tag": "passenger",
+ "root_tag": "country"
+ }
+ }
+ },
+ "key_field": "passenger_id"
+ }
+ }
+ },
+ "transformations": {
+ "parameters": {
+ "entity": "country"
+ },
+ "filters": [
+ {
+ "entity": "flights",
+ "name": "flight_missing_id",
+ "expression": "flight_id IS NOT NULL",
+ "failure_type": "record",
+ "failure_message": "Record Rejected - Flight is missing an id",
+ "reporting_field": "flight_id",
+ "reporting_entity": "flights",
+ "category": "Blank",
+ "error_code": "FlightIDMissing"
+ },
+ {
+ "entity": "flights",
+ "name": "invalid_destination",
+ "expression": "lower(destination) IN ('paris', 'madrid', 'new york', 'amsterdam', 'rome', 'dubai', 'dublin', 'lisbon', 'toronto')",
+ "failure_type": "record",
+ "failure_message": "Record Rejected - {{ destination }} is not a valid destination",
+ "reporting_field": "flight_id",
+ "reporting_entity": "flights",
+ "category": "Bad value",
+ "error_code": "InvalidFlightDestination"
+ },
+ {
+ "entity": "passengers",
+ "name": "passenger_name_is_null",
+ "expression": "passenger_name IS NOT NULL",
+ "failure_type": "record",
+ "failure_message": "Record Rejected - Passenger Name is missing",
+ "reporting_field": "passenger_name",
+ "reporting_entity": "passengers",
+ "category": "Blank",
+ "error_code": "PassengerNameMissing"
+ },
+ {
+ "entity": "staff",
+ "name": "staff_id_is_null",
+ "expression": "staff_id IS NOT NULL",
+ "failure_type": "record",
+ "failure_message": "Record Rejected - staff_id is missing",
+ "reporting_field": "passenger_name",
+ "reporting_entity": "passengers",
+ "category": "Blank",
+ "error_code": "StaffIDMissing"
+ }
+ ]
+ },
+ "entity_relationships": {
+ "country": {
+ "is_root_entity": true,
+ "mandatory": true,
+ "empty_entity_error_code": "NoValidCountries",
+ "empty_entity_error_message": "File Rejected - There are no valid country records"
+ },
+ "airport": {
+ "parent_entity": "country",
+ "join_fields": {
+ "country_id": "country_id"
+ },
+ "mandatory": true,
+ "missing_parent_id_error_code": "AirportHasNoCountry",
+ "missing_parent_id_error_message": "Record rejected - No valid country id found for airport",
+ "no_valid_records_error_code": "CountryHasNoAirport",
+ "no_valid_records_error_message": "Group rejected - Unable to find any valid airports",
+ "empty_entity_error_code": "NoValidAirports",
+ "empty_entity_error_message": "File Rejected - There are no valid airport records"
+ },
+ "staff": {
+ "parent_entity": "airport",
+ "join_fields": {
+ "airport_id": "airport_id"
+ },
+ "mandatory": true,
+ "missing_parent_id_error_code": "StaffHasNoAirport",
+ "missing_parent_id_error_message": "Record rejected - No valid airport id found for staff. Airport ID = {{ airport_id }}, Staff ID = {{ staff_id }}",
+ "no_valid_records_error_code": "AirportHasNoStaff",
+ "no_valid_records_error_message": "Group rejected - Airport has no valid staff. Airport ID = {{ airport_id }}",
+ "empty_entity_error_code": "NoValidStaff",
+ "empty_entity_error_message": "File Rejected - There are no valid staff records"
+ },
+ "flights": {
+ "parent_entity": "airport",
+ "join_fields": {
+ "airport_id": "airport_id"
+ },
+ "mandatory": false,
+ "missing_parent_id_error_code": "FlightHasNoAirport",
+ "missing_parent_id_error_message": "Record Rejected - No valid airport found for flight"
+ },
+ "passengers": {
+ "parent_entity": "flights",
+ "join_fields": {
+ "flight_id": "flight_id"
+ },
+ "mandatory": false,
+ "missing_parent_id_error_code": "PassengerHasNoFlight",
+ "missing_parent_id_error_message": "Record rejected - No valid flight found for passenger"
+ }
+ }
+}
\ No newline at end of file
From a342af77cdb6cc2d02b843d84c4fde2814311831 Mon Sep 17 00:00:00 2001
From: George Robertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Thu, 8 Oct 2026 14:00:26 +0100
Subject: [PATCH 28/47] fix: group rejection not filtering out invalid records
correctly (#175)
---
src/dve/core_engine/backends/base/rules.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/src/dve/core_engine/backends/base/rules.py b/src/dve/core_engine/backends/base/rules.py
index 6200e013..9f19c024 100644
--- a/src/dve/core_engine/backends/base/rules.py
+++ b/src/dve/core_engine/backends/base/rules.py
@@ -513,7 +513,9 @@ def process_node(node: HierarchyNode) -> bool:
for tree in entity_hierarchy.entity_trees.values():
for node in tree.iterate_lowest_descendent_up():
if node.parent_entity and node.mandatory:
- entity_issues_found[node.parent_entity] = process_node(node)
+ result = process_node(node)
+ if result:
+ entity_issues_found[node.parent_entity] = result
return [], entity_issues_found
From ddfdf64ac473a8202e06fbf18970124589177168 Mon Sep 17 00:00:00 2001
From: George Robertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Fri, 9 Oct 2026 10:43:02 +0100
Subject: [PATCH 29/47] build: upgrade urllib3 from 2.7 to 2.8 (#178)
---
poetry.lock | 8 ++++----
pyproject.toml | 2 +-
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/poetry.lock b/poetry.lock
index 18f789ec..78d904f7 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -3347,14 +3347,14 @@ files = [
[[package]]
name = "urllib3"
-version = "2.7.0"
+version = "2.8.0"
description = "HTTP library with thread-safe connection pooling, file post, and more."
optional = false
python-versions = ">=3.10"
groups = ["main", "dev", "test"]
files = [
- {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"},
- {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"},
+ {file = "urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3"},
+ {file = "urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63"},
]
[package.extras]
@@ -3638,4 +3638,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""]
[metadata]
lock-version = "2.1"
python-versions = ">=3.10,<3.13"
-content-hash = "6d996888c9d149407fb885b454745bf763f9cba56404a29148ffe0b9fd57b0de"
+content-hash = "2345c7671a4598e4421d8e71efdf2ad53343494a72bbec7703e26b461e73d794"
diff --git a/pyproject.toml b/pyproject.toml
index 5cac0282..49b4f30a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -47,7 +47,7 @@ pyarrow = "23.0.1"
pydantic = "2.13.4"
pyspark = ">=3.5.0,<=3.5.5"
typing_extensions = "4.15.0"
-urllib3 = "2.7.0" # dependency of boto3 & botocore
+urllib3 = "2.8.0" # dependency of boto3 & botocore
[tool.poetry.group.dev]
optional = true
From a32a01b339b4d29a9eea5b3ba75300716827aaee Mon Sep 17 00:00:00 2001
From: George Robertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Fri, 9 Oct 2026 11:20:21 +0100
Subject: [PATCH 30/47] fix: refactor the join_header method in duckdb to deal
with out of bound date values (#177)
---
.../backends/implementations/duckdb/rules.py | 22 ++++++++-----
.../test_duckdb/test_rules.py | 32 +++++++++++++++++++
2 files changed, 46 insertions(+), 8 deletions(-)
diff --git a/src/dve/core_engine/backends/implementations/duckdb/rules.py b/src/dve/core_engine/backends/implementations/duckdb/rules.py
index 766142aa..2dd6a048 100644
--- a/src/dve/core_engine/backends/implementations/duckdb/rules.py
+++ b/src/dve/core_engine/backends/implementations/duckdb/rules.py
@@ -10,6 +10,7 @@
ConstantExpression,
DuckDBPyConnection,
DuckDBPyRelation,
+ FunctionExpression,
StarExpression,
)
from duckdb.typing import DuckDBPyType
@@ -22,7 +23,6 @@
)
from dve.core_engine.backends.exceptions import ConstraintError
from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import (
- DDBStruct,
ddb_filter_contract_errors,
duckdb_get_entity_count,
duckdb_read_parquet,
@@ -361,8 +361,7 @@ def join_header(self, entities: DuckDBEntities, *, config: HeaderJoin) -> Messag
target_rel: DuckDBPyRelation = entities[config.target_name]
target_rel = target_rel.set_alias(config.target_name)
- target_rows = target_rel.pl().to_struct("header").to_list()
- n_target_rows = len(target_rows)
+ n_target_rows = target_rel.shape[0]
if n_target_rows != 1:
raise ConstraintError(
f"Unable to join header {config.target_name!r} to {config.entity_name!r} "
@@ -373,11 +372,18 @@ def join_header(self, entities: DuckDBEntities, *, config: HeaderJoin) -> Messag
),
)
- target_schema = DDBStruct(dict(zip(target_rel.columns, target_rel.dtypes)))() # type: ignore # pylint:disable=C0301
-
- joined_rel = source_rel.select(
- StarExpression(exclude=[]),
- ConstantExpression(target_rows[0]).cast(target_schema).alias(config.header_column_name),
+ joined_rel = (
+ source_rel
+ .cross(target_rel)
+ .select(
+ StarExpression(
+ exclude=[f"{config.target_name}.{c}" for c in target_rel.columns]
+ ),
+ FunctionExpression(
+ "struct_pack",
+ *[ColumnExpression(f"{config.target_name}.{c}") for c in target_rel.columns],
+ ).alias(config.header_column_name)
+ )
)
entities[config.new_entity_name or config.entity_name] = joined_rel
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py
index aae56967..ea1629ca 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_rules.py
@@ -70,6 +70,13 @@ def value_literal_1_header(duckdb_connection) -> Iterator[DuckDBPyRelation]:
yield duckdb_connection.sql("SELECT 1 AS Value")
+@pytest.fixture(scope="function")
+def value_mixed_header(duckdb_connection) -> Iterator[DuckDBPyRelation]:
+ yield duckdb_connection.sql(
+ "SELECT make_date(0000,1,1) as test_date, 'test' as test_str, 123 as test_int, make_date(10000,1,1) as test_date2"
+ )
+
+
def test_column_addition(planets_rel: DuckDBPyRelation):
"""Test that columns can be added to entities."""
entities = EntityManager({"planets": planets_rel})
@@ -566,6 +573,31 @@ def test_header_join_planets(
assert planet_row == actual_row, "More than header changed as result of header join"
+def test_header_join_with_out_of_range_date(
+ planets_rel: DuckDBPyRelation,
+ value_mixed_header: DuckDBPyRelation
+):
+ """Test that a header with an invalid date does not error"""
+ header_join = HeaderJoin(
+ entity_name="planets",
+ target_name="header",
+ new_entity_name="planets_header",
+ header_column_name="_Header"
+ )
+ entities = EntityManager({
+ "planets": planets_rel,
+ "header": value_mixed_header,
+ })
+ DUCKDB_STEP_BACKEND.evaluate(entities, config=header_join)
+
+ assert entities.entities["planets_header"].select("_header").distinct().shape[0] == 1
+ assert entities.entities["planets_header"].shape[0] == 9
+ assert (
+ entities.entities["planets_header"].select("typeof(_header)").fetchone()[0]
+ == 'STRUCT(test_date DATE, test_str VARCHAR, test_int INTEGER, test_date2 DATE)'
+ )
+
+
def test_header_multi_rows_raises(
planets_rel: DuckDBPyRelation, value_literal_1_header: DuckDBPyRelation
):
From 06eaee659b06df993e2bc3b4eccc83f0a18fc33a Mon Sep 17 00:00:00 2001
From: George Robertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Fri, 9 Oct 2026 11:33:35 +0100
Subject: [PATCH 31/47] build: bump virtualenv version to 21.7.13 to resolve
vulns (#179)
---
poetry.lock | 23 +++++++++--------------
pyproject.toml | 3 ++-
2 files changed, 11 insertions(+), 15 deletions(-)
diff --git a/poetry.lock b/poetry.lock
index 78d904f7..e647ec18 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -2838,23 +2838,18 @@ six = ">=1.5"
[[package]]
name = "python-discovery"
-version = "1.2.0"
+version = "1.6.0"
description = "Python interpreter discovery"
optional = false
python-versions = ">=3.8"
groups = ["dev"]
files = [
- {file = "python_discovery-1.2.0-py3-none-any.whl", hash = "sha256:1e108f1bbe2ed0ef089823d28805d5ad32be8e734b86a5f212bf89b71c266e4a"},
- {file = "python_discovery-1.2.0.tar.gz", hash = "sha256:7d33e350704818b09e3da2bd419d37e21e7c30db6e0977bb438916e06b41b5b1"},
+ {file = "python_discovery-1.6.0-py3-none-any.whl", hash = "sha256:d4e244cf17b8b29819ed78003d55fbacf86eda23425b075454fff9271b79377a"},
+ {file = "python_discovery-1.6.0.tar.gz", hash = "sha256:6393b4eae1be8b2182670635e7baff89ac21cb9f8e86fd1ff40c7b1144febb4c"},
]
[package.dependencies]
filelock = ">=3.15.4"
-platformdirs = ">=4.3.6,<5"
-
-[package.extras]
-docs = ["furo (>=2025.12.19)", "sphinx (>=9.1)", "sphinx-autodoc-typehints (>=3.6.3)", "sphinxcontrib-mermaid (>=2)"]
-testing = ["covdefaults (>=2.3)", "coverage (>=7.5.4)", "pytest (>=8.3.5)", "pytest-mock (>=3.14)", "setuptools (>=75.1)"]
[[package]]
name = "pytokens"
@@ -3365,21 +3360,21 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""]
[[package]]
name = "virtualenv"
-version = "21.2.1"
+version = "21.7.13"
description = "Virtual Python Environment builder"
optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.9"
groups = ["dev"]
files = [
- {file = "virtualenv-21.2.1-py3-none-any.whl", hash = "sha256:bd16b49c53562b28cf1a3ad2f36edb805ad71301dee70ddc449e5c88a9f919a2"},
- {file = "virtualenv-21.2.1.tar.gz", hash = "sha256:b66ffe81301766c0d5e2208fc3576652c59d44e7b731fc5f5ed701c9b537fa78"},
+ {file = "virtualenv-21.7.13-py3-none-any.whl", hash = "sha256:1bea5af7463f59c4719db48fe739579a2a4f569c96f26c086edda85c96da9f59"},
+ {file = "virtualenv-21.7.13.tar.gz", hash = "sha256:0355558b6f33619aab31347e43643b0ebc97f61ea3acf617b2b69e1f8a843d11"},
]
[package.dependencies]
distlib = ">=0.3.7,<1"
filelock = {version = ">=3.24.2,<4", markers = "python_version >= \"3.10\""}
platformdirs = ">=3.9.1,<5"
-python-discovery = ">=1"
+python-discovery = ">=1.6"
typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\""}
[[package]]
@@ -3638,4 +3633,4 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""]
[metadata]
lock-version = "2.1"
python-versions = ">=3.10,<3.13"
-content-hash = "2345c7671a4598e4421d8e71efdf2ad53343494a72bbec7703e26b461e73d794"
+content-hash = "2090fe698ede57b0910f35a1701b8fe66bbcd69ffa72412597afcab95f4876b8"
diff --git a/pyproject.toml b/pyproject.toml
index 49b4f30a..6970281f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -60,8 +60,9 @@ include-groups = [
commitizen = "4.9.1"
pre-commit = "4.3.0"
charset-normalizer = "3.4.6"
-python-discovery = "1.2.0"
+python-discovery = "1.6.0"
idna = "3.15" # dependency of requests
+virtualenv = "21.7.13"
[tool.poetry.group.test]
optional = true
From 5bc3eda87d3f7e6db7e02f6814d5e3c2eae5a323 Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Mon, 12 Oct 2026 11:37:37 +0100
Subject: [PATCH 32/47] docs: add udfs to docs
refactor the advanced docs section
---
.../package_documentation/auditing.md | 1 +
.../business_rules/index.md | 31 +++++++++++++++
.../{ => business_rules}/operations.md | 0
.../{ => business_rules}/refdata_loaders.md | 0
.../reference_data_types.md} | 0
.../business_rules/udfs.md | 10 +++++
.../{ => data_contract}/domain_types.md | 0
.../data_contract/index.md | 20 ++++++++++
.../{ => data_contract}/readers.md | 0
.../package_documentation/entity_hierarchy.md | 2 +-
.../feedback/feedback_messages.md | 9 +++++
.../package_documentation/feedback/index.md | 13 +++++++
.../feedback_messages.md | 12 ------
.../package_documentation/index.md | 38 +++++++++++++++++--
.../package_documentation/other/index.md | 13 +++++++
.../{ => other}/models.md | 4 +-
docs/user_guidance/business_rules.md | 24 +++++++++++-
docs/user_guidance/data_contract.md | 2 +-
docs/user_guidance/feedback_messages.md | 2 +-
docs/user_guidance/file_transformation.md | 2 +-
docs/user_guidance/implementations/duckdb.md | 6 +--
docs/user_guidance/implementations/spark.md | 2 +-
zensical.toml | 31 ++++++++-------
23 files changed, 182 insertions(+), 40 deletions(-)
create mode 100644 docs/advanced_guidance/package_documentation/business_rules/index.md
rename docs/advanced_guidance/package_documentation/{ => business_rules}/operations.md (100%)
rename docs/advanced_guidance/package_documentation/{ => business_rules}/refdata_loaders.md (100%)
rename docs/advanced_guidance/package_documentation/{refence_data_types.md => business_rules/reference_data_types.md} (100%)
create mode 100644 docs/advanced_guidance/package_documentation/business_rules/udfs.md
rename docs/advanced_guidance/package_documentation/{ => data_contract}/domain_types.md (100%)
create mode 100644 docs/advanced_guidance/package_documentation/data_contract/index.md
rename docs/advanced_guidance/package_documentation/{ => data_contract}/readers.md (100%)
create mode 100644 docs/advanced_guidance/package_documentation/feedback/feedback_messages.md
create mode 100644 docs/advanced_guidance/package_documentation/feedback/index.md
delete mode 100644 docs/advanced_guidance/package_documentation/feedback_messages.md
create mode 100644 docs/advanced_guidance/package_documentation/other/index.md
rename docs/advanced_guidance/package_documentation/{ => other}/models.md (51%)
diff --git a/docs/advanced_guidance/package_documentation/auditing.md b/docs/advanced_guidance/package_documentation/auditing.md
index a043e9ae..78230195 100644
--- a/docs/advanced_guidance/package_documentation/auditing.md
+++ b/docs/advanced_guidance/package_documentation/auditing.md
@@ -11,3 +11,4 @@
- get_all_business_rule_submissions
- get_all_error_report_submissions
- get_current_processing_info
+ show_root_toc_entry: false
diff --git a/docs/advanced_guidance/package_documentation/business_rules/index.md b/docs/advanced_guidance/package_documentation/business_rules/index.md
new file mode 100644
index 00000000..e64ed66e
--- /dev/null
+++ b/docs/advanced_guidance/package_documentation/business_rules/index.md
@@ -0,0 +1,31 @@
+---
+title: Business Rules Documentation
+---
+
+
+
+- :material-language-python:{ .lg .middle } __Operations__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the `Operation` objects here](operations.md)
+
+- :material-language-python:{ .lg .middle } __Reference Data Loaders__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the `Business Rules` objects here](refdata_loaders.md)
+
+- :material-language-python:{ .lg .middle } __Reference Data Types__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the `Data Contract` objects here](reference_data_types.md)
+
+- :material-language-python:{ .lg .middle } __User Defined Functions (UDFs)__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the `Submission Feedback` objects here](udfs.md)
+
+
\ No newline at end of file
diff --git a/docs/advanced_guidance/package_documentation/operations.md b/docs/advanced_guidance/package_documentation/business_rules/operations.md
similarity index 100%
rename from docs/advanced_guidance/package_documentation/operations.md
rename to docs/advanced_guidance/package_documentation/business_rules/operations.md
diff --git a/docs/advanced_guidance/package_documentation/refdata_loaders.md b/docs/advanced_guidance/package_documentation/business_rules/refdata_loaders.md
similarity index 100%
rename from docs/advanced_guidance/package_documentation/refdata_loaders.md
rename to docs/advanced_guidance/package_documentation/business_rules/refdata_loaders.md
diff --git a/docs/advanced_guidance/package_documentation/refence_data_types.md b/docs/advanced_guidance/package_documentation/business_rules/reference_data_types.md
similarity index 100%
rename from docs/advanced_guidance/package_documentation/refence_data_types.md
rename to docs/advanced_guidance/package_documentation/business_rules/reference_data_types.md
diff --git a/docs/advanced_guidance/package_documentation/business_rules/udfs.md b/docs/advanced_guidance/package_documentation/business_rules/udfs.md
new file mode 100644
index 00000000..6a994a27
--- /dev/null
+++ b/docs/advanced_guidance/package_documentation/business_rules/udfs.md
@@ -0,0 +1,10 @@
+
+!!! warning
+
+ Consider limiting how many UDFs you utilise in your rules as they can cause significant performance degredation when used on large datasets.
+
+::: dve.core_engine.functions.implementations
+ handler: python
+ options:
+ show_root_heading: true
+ heading_level: 1
diff --git a/docs/advanced_guidance/package_documentation/domain_types.md b/docs/advanced_guidance/package_documentation/data_contract/domain_types.md
similarity index 100%
rename from docs/advanced_guidance/package_documentation/domain_types.md
rename to docs/advanced_guidance/package_documentation/data_contract/domain_types.md
diff --git a/docs/advanced_guidance/package_documentation/data_contract/index.md b/docs/advanced_guidance/package_documentation/data_contract/index.md
new file mode 100644
index 00000000..df4bf23b
--- /dev/null
+++ b/docs/advanced_guidance/package_documentation/data_contract/index.md
@@ -0,0 +1,20 @@
+---
+title: Data Contract Documentation
+---
+
+
+
+- :material-language-python:{ .lg .middle } __Readers__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the `Operation` objects here](readers.md)
+
+- :material-language-python:{ .lg .middle } __Domain Types__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the `Business Rules` objects here](domain_types.md)
+
+
+
\ No newline at end of file
diff --git a/docs/advanced_guidance/package_documentation/readers.md b/docs/advanced_guidance/package_documentation/data_contract/readers.md
similarity index 100%
rename from docs/advanced_guidance/package_documentation/readers.md
rename to docs/advanced_guidance/package_documentation/data_contract/readers.md
diff --git a/docs/advanced_guidance/package_documentation/entity_hierarchy.md b/docs/advanced_guidance/package_documentation/entity_hierarchy.md
index 4080dcbe..b4f3ba48 100644
--- a/docs/advanced_guidance/package_documentation/entity_hierarchy.md
+++ b/docs/advanced_guidance/package_documentation/entity_hierarchy.md
@@ -2,4 +2,4 @@
handler: python
options:
show_root_heading: true
- heading_level: 2
+ heading_level: 1
diff --git a/docs/advanced_guidance/package_documentation/feedback/feedback_messages.md b/docs/advanced_guidance/package_documentation/feedback/feedback_messages.md
new file mode 100644
index 00000000..cfc279a6
--- /dev/null
+++ b/docs/advanced_guidance/package_documentation/feedback/feedback_messages.md
@@ -0,0 +1,9 @@
+
+::: dve.core_engine.message
+ handler: python
+ options:
+ heading_level: 2
+ members:
+ - FeedbackMessage
+ - UserMessage
+ show_root_toc_entry: false
diff --git a/docs/advanced_guidance/package_documentation/feedback/index.md b/docs/advanced_guidance/package_documentation/feedback/index.md
new file mode 100644
index 00000000..e0d810a2
--- /dev/null
+++ b/docs/advanced_guidance/package_documentation/feedback/index.md
@@ -0,0 +1,13 @@
+---
+title: User Feedback Documentation
+---
+
+
+
+- :material-language-python:{ .lg .middle } __Feedback Messages__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the `Operation` objects here](feedback_messages.md)
+
+
\ No newline at end of file
diff --git a/docs/advanced_guidance/package_documentation/feedback_messages.md b/docs/advanced_guidance/package_documentation/feedback_messages.md
deleted file mode 100644
index ede7aad8..00000000
--- a/docs/advanced_guidance/package_documentation/feedback_messages.md
+++ /dev/null
@@ -1,12 +0,0 @@
-
-::: dve.core_engine.message.FeedbackMessage
- handler: python
- options:
- show_root_heading: true
- heading_level: 2
-
-::: dve.core_engine.message.UserMessage
- handler: python
- options:
- show_root_heading: true
- heading_level: 2
diff --git a/docs/advanced_guidance/package_documentation/index.md b/docs/advanced_guidance/package_documentation/index.md
index c2e76c50..3c02ad0e 100644
--- a/docs/advanced_guidance/package_documentation/index.md
+++ b/docs/advanced_guidance/package_documentation/index.md
@@ -4,16 +4,46 @@ title: Package Documentation
-- :material-language-python:{ .lg .middle } __Pipeline__
+- :material-database-plus:{ .lg .middle } __Auditing__
---
- [:octicons-arrow-right-24: Read about the `Pipeline` objects here](pipeline.md)
+ [:octicons-arrow-right-24: Read about the `Auditing` objects here](auditing.md)
+
+- :material-clipboard-check:{ .lg .middle } __Business Rules__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the `Business Rules` objects here](./business_rules/index.md)
+
+- :material-file-document-check:{ .lg .middle } __Data Contract__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the `Data Contract` objects here](./data_contract/index.md)
+
+- :material-relation-one-to-many:{ .lg .middle } __Entity Relationships__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the `Entity Reliationships` objects here](entity_hierarchy.md)
+
+- :material-message-alert:{ .lg .middle } __Submission Feedback__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the `Submission Feedback` objects here](./feedback/index.md)
-- :material-set-all:{ .lg .middle } __Reference Data Loader__
+- :material-package-variant-plus:{ .lg .middle } __Additional Objects__
---
- [:octicons-arrow-right-24: Read about the `Reference Data Loader` objects here](refdata_loaders.md)
+ [:octicons-arrow-right-24: Read about the additional objects here](./other/index.md)
+
+- :material-pipe:{ .lg .middle } __Pipeline__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the `Pipeline` objects here](pipeline.md)
\ No newline at end of file
diff --git a/docs/advanced_guidance/package_documentation/other/index.md b/docs/advanced_guidance/package_documentation/other/index.md
new file mode 100644
index 00000000..bfb3a4e2
--- /dev/null
+++ b/docs/advanced_guidance/package_documentation/other/index.md
@@ -0,0 +1,13 @@
+---
+title: Additional Objects Documentation
+---
+
+
+
+- :material-code-braces-box:{ .lg .middle } __Models__
+
+ ---
+
+ [:octicons-arrow-right-24: Read about the Model objects here](./models.md)
+
+
diff --git a/docs/advanced_guidance/package_documentation/models.md b/docs/advanced_guidance/package_documentation/other/models.md
similarity index 51%
rename from docs/advanced_guidance/package_documentation/models.md
rename to docs/advanced_guidance/package_documentation/other/models.md
index 42e94202..0fc02050 100644
--- a/docs/advanced_guidance/package_documentation/models.md
+++ b/docs/advanced_guidance/package_documentation/other/models.md
@@ -2,5 +2,7 @@
::: dve.core_engine.models
handler: python
options:
- show_root_heading: true
heading_level: 2
+ members:
+ - SubmissionInfo
+ show_root_toc_entry: false
diff --git a/docs/user_guidance/business_rules.md b/docs/user_guidance/business_rules.md
index 0f4093f5..9ea21a12 100644
--- a/docs/user_guidance/business_rules.md
+++ b/docs/user_guidance/business_rules.md
@@ -209,7 +209,27 @@ The difference between modifiying the existing entity and adding a new one is si
### Operations
-For a full list of operations that you can perform during the pre-steps see [Advanced User Guidance: Operations](../advanced_guidance/package_documentation/operations.md).
+For a full list of operations that you can perform during the pre-steps see [Advanced User Guidance: Operations](../advanced_guidance/package_documentation/business_rules/operations.md).
+
+### User Defined Functions (UDFs)
+
+!!! warning
+
+ Consider limiting how many UDFs you utilise in your rules as they can cause significant performance degredation when used on large datasets.
+
+
+The DVE currently has a number of UDFs available to be called during a rule check. For instance, if you wanted to use the [NHS Number](https://www.datadictionary.nhs.uk/attributes/nhs_number.html#ariaid-title2) mod 11 check then you can simply call the udf during the expression definition - such as:
+
+```json title="example.dischema.json"
+ {
+ "entity": "example",
+ "name": "check NHS number passes mod 11 check",
+ "expression": "nhsno_mod11_check()"
+ ...
+ }
+```
+
+A full list of UDF functions available can be found [here](../advanced_guidance/package_documentation/business_rules/udfs.md).
## Post Rule
@@ -358,7 +378,7 @@ If your Business Rules are reliant on reference data, then you can add the `"ref
- Refdata entities are also immutable. So, if you need to modify them in any way, you will always need to create a new entity from it
-For latest supported reference data types, see [Advanced User Guidance: Reference Data Types](../advanced_guidance/package_documentation/refence_data_types.md).
+For latest supported reference data types, see [Advanced User Guidance: Reference Data Types](../advanced_guidance/package_documentation/business_rules/reference_data_types.md).
## Complex Rules
diff --git a/docs/user_guidance/data_contract.md b/docs/user_guidance/data_contract.md
index 13e72efa..937fed10 100644
--- a/docs/user_guidance/data_contract.md
+++ b/docs/user_guidance/data_contract.md
@@ -278,7 +278,7 @@ As you can see, we can set the "type" for several fields to `MyConstrainedString
#### Domain Types
-Domain types are custom Pydantic model types available with the DVE. Current Domain types available are `Postcode`, `NHSNumber`, `FormattedDatetime` etc. You can find the full list of Domain Types [here](../advanced_guidance/package_documentation/domain_types.md).
+Domain types are custom Pydantic model types available with the DVE. Current Domain types available are `Postcode`, `NHSNumber`, `FormattedDatetime` etc. You can find the full list of Domain Types [here](../advanced_guidance/package_documentation/data_contract/domain_types.md).
### Complex Types
diff --git a/docs/user_guidance/feedback_messages.md b/docs/user_guidance/feedback_messages.md
index ee812886..01128d27 100644
--- a/docs/user_guidance/feedback_messages.md
+++ b/docs/user_guidance/feedback_messages.md
@@ -28,4 +28,4 @@ If the DVE is crashing and the error message is either unreadable or is crashing
## Feedback Message Models
-Please refer to [Advanced User Guidance: Feedback Messages](../advanced_guidance/package_documentation/feedback_messages.md).
+Please refer to [Advanced User Guidance: Feedback Messages](../advanced_guidance/package_documentation/feedback/feedback_messages.md).
diff --git a/docs/user_guidance/file_transformation.md b/docs/user_guidance/file_transformation.md
index 41dc7de4..411b1e0f 100644
--- a/docs/user_guidance/file_transformation.md
+++ b/docs/user_guidance/file_transformation.md
@@ -155,7 +155,7 @@ The secondary use of the File Transformation stage is the ability to normalise y
```
!!! abstract ""
- You can read more about the readers and kwargs [here](../advanced_guidance/package_documentation/readers.md).
+ You can read more about the readers and kwargs [here](../advanced_guidance/package_documentation/data_contract/readers.md).
## Supported Formats
diff --git a/docs/user_guidance/implementations/duckdb.md b/docs/user_guidance/implementations/duckdb.md
index ac75c587..48448e35 100644
--- a/docs/user_guidance/implementations/duckdb.md
+++ b/docs/user_guidance/implementations/duckdb.md
@@ -36,7 +36,7 @@ Now you have the DuckDB connection object setup, you are ready to setup the requ
## Generating SubmissionInfo objects
-Before we utilise the DVE, we need to generate an iterable object containing `SubmissionInfo` objects. These objects effectively contain the necessery metadata for the DVE to work with a given submission. Here is an example function used to generate [SubmissionInfo](../../advanced_guidance/package_documentation/models.md#dve.core_engine.models.SubmissionInfo) objects from a given path:
+Before we utilise the DVE, we need to generate an iterable object containing `SubmissionInfo` objects. These objects effectively contain the necessery metadata for the DVE to work with a given submission. Here is an example function used to generate [SubmissionInfo](../../advanced_guidance/package_documentation/other/models.md#dve.core_engine.models.SubmissionInfo) objects from a given path:
```py
import glob
from datetime import date, datetime
@@ -112,7 +112,7 @@ DuckDBRefDataLoader.dataset_config_uri = Path("path", "to", "my", "rules").as_po
The connection passed into the `DuckDBRefDataLoader` object will then be able to use various DuckDB readers to load data from an existing table on the connection OR loading data from reference data persisted in either `parquet` or `pyarrow` format.
-If you want to learn more about the reference data loaders, you can view the advanced user guidance [here](../../advanced_guidance/package_documentation/refdata_loaders.md).
+If you want to learn more about the reference data loaders, you can view the advanced user guidance [here](../../advanced_guidance/package_documentation/business_rules/refdata_loaders.md).
Now we can move onto setting up the DuckDB DVE Pipeline object.
@@ -171,4 +171,4 @@ error_reports = dve_pipeline.cluster_pipeline_run()
For further details on the objects referenced above, you can use the following links to read more about the objects:
- [Pipeline Docs](../../advanced_guidance/package_documentation/pipeline.md)
-- [Reference Data Docs](../../advanced_guidance/package_documentation/refdata_loaders.md)
+- [Reference Data Docs](../../advanced_guidance/package_documentation/business_rules/refdata_loaders.md)
diff --git a/docs/user_guidance/implementations/spark.md b/docs/user_guidance/implementations/spark.md
index 6bbdb300..866460e0 100644
--- a/docs/user_guidance/implementations/spark.md
+++ b/docs/user_guidance/implementations/spark.md
@@ -187,4 +187,4 @@ error_reports = dve_pipeline.cluster_pipeline_run()
For further details on the objects referenced above, you can use the following links to read more about the objects:
- [Pipeline Docs](../../advanced_guidance/package_documentation/pipeline.md)
-- [Reference Data Docs](../../advanced_guidance/package_documentation/refdata_loaders.md)
+- [Reference Data Docs](../../advanced_guidance/package_documentation/business_rules/refdata_loaders.md)
diff --git a/zensical.toml b/zensical.toml
index 0141b6df..fc47343e 100644
--- a/zensical.toml
+++ b/zensical.toml
@@ -44,28 +44,33 @@ nav = [
"advanced_guidance/index.md",
{"DVE Package Documentation" = [
"advanced_guidance/package_documentation/index.md",
- {"Pipeline" = "advanced_guidance/package_documentation/pipeline.md"},
{"Auditing" = "advanced_guidance/package_documentation/auditing.md"},
- {"Data Contract" = [
- {"Readers" = "advanced_guidance/package_documentation/readers.md"},
- {"Domain Types" = "advanced_guidance/package_documentation/domain_types.md"},
- ]},
{"Business Rules" = [
+ "advanced_guidance/package_documentation/business_rules/index.md",
{"Rules" = [
- {"Operations" = "advanced_guidance/package_documentation/operations.md"},
+ {"Operations" = "advanced_guidance/package_documentation/business_rules/operations.md"},
]},
{"Refdata" = [
- {"Refdata Types" = "advanced_guidance/package_documentation/refence_data_types.md"},
- {"Refdata Loaders" = "advanced_guidance/package_documentation/refdata_loaders.md"},
+ {"Refdata Types" = "advanced_guidance/package_documentation/business_rules/reference_data_types.md"},
+ {"Refdata Loaders" = "advanced_guidance/package_documentation/business_rules/refdata_loaders.md"},
]},
- {"Entity Relationships" = [
- {"Entity Hierarchy" = "advanced_guidance/package_documentation/entity_hierarchy.md"}
- ]}
+ {"User Defined Functions (UDFs)" = "advanced_guidance/package_documentation/business_rules/udfs.md"},
]},
+ {"Data Contract" = [
+ "advanced_guidance/package_documentation/data_contract/index.md",
+ {"Readers" = "advanced_guidance/package_documentation/data_contract/readers.md"},
+ {"Domain Types" = "advanced_guidance/package_documentation/data_contract/domain_types.md"},
+ ]},
+ {"Entity Relationships" = "advanced_guidance/package_documentation/entity_hierarchy.md"},
{"Feedback" = [
- {"Feedback Messages" = "advanced_guidance/package_documentation/feedback_messages.md"},
+ "advanced_guidance/package_documentation/feedback/index.md",
+ {"Feedback Messages" = "advanced_guidance/package_documentation/feedback/feedback_messages.md"},
+ ]},
+ {"Other" = [
+ "advanced_guidance/package_documentation/other/index.md",
+ {"Core Models" = "advanced_guidance/package_documentation/other/models.md"},
]},
- {"Models" = "advanced_guidance/package_documentation/models.md"},
+ {"Pipeline" = "advanced_guidance/package_documentation/pipeline.md"},
]},
{"DVE Developer Guidance" = [
{"Implementing a new backend" = "advanced_guidance/new_backend.md"},
From 72d15bed72af818ce5ba2d28c377d9fe910bb60a Mon Sep 17 00:00:00 2001
From: stevenhsd <56357022+stevenhsd@users.noreply.github.com>
Date: Mon, 12 Oct 2026 12:01:35 +0100
Subject: [PATCH 33/47] * fix: ensure that null records are not included in
record counts when determining if entity empty (#182)
* test: additional tests with record index for entity empty check functions
---
.../implementations/duckdb/duckdb_helpers.py | 12 +++++---
.../backends/implementations/duckdb/rules.py | 18 ++++-------
.../implementations/spark/spark_helpers.py | 14 +++++++--
.../test_duckdb/test_duckdb_helpers.py | 13 ++++++++
.../test_spark/test_spark_helpers.py | 30 +++++++++++++++++++
5 files changed, 69 insertions(+), 18 deletions(-)
diff --git a/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py b/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py
index 55db0945..c25387fb 100644
--- a/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py
+++ b/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py
@@ -110,9 +110,13 @@ def get_all_existing_ddb_tables(connection: DuckDBPyConnection) -> tuple[str]:
def relation_is_empty(relation: DuckDBPyRelation) -> bool:
"""Check if a duckdb relation is empty"""
- if relation.limit(1).shape[0] > 0:
- return False
- return True
+ if relation.limit(1).shape[0] == 0:
+ return True
+
+ cl_all_null_check_exprs = [
+ f'bool_and("{cl}" is NULL)' for cl in relation.columns if cl != RECORD_INDEX_COLUMN_NAME
+ ]
+ return relation.aggregate(" AND ".join(cl_all_null_check_exprs)).fetchone()[0] # type: ignore
def get_duckdb_type_from_annotation(type_annotation: Any) -> DuckDBPyType:
@@ -329,7 +333,7 @@ def duckdb_get_entity_count(cls):
def _duckdb_check_entity_empty(self, entity: DuckDBPyRelation) -> bool: # pylint: disable=W0613
- return entity.shape[0] == 0
+ return relation_is_empty(entity)
def duckdb_check_entity_empty(cls):
diff --git a/src/dve/core_engine/backends/implementations/duckdb/rules.py b/src/dve/core_engine/backends/implementations/duckdb/rules.py
index 2dd6a048..187a880d 100644
--- a/src/dve/core_engine/backends/implementations/duckdb/rules.py
+++ b/src/dve/core_engine/backends/implementations/duckdb/rules.py
@@ -372,18 +372,12 @@ def join_header(self, entities: DuckDBEntities, *, config: HeaderJoin) -> Messag
),
)
- joined_rel = (
- source_rel
- .cross(target_rel)
- .select(
- StarExpression(
- exclude=[f"{config.target_name}.{c}" for c in target_rel.columns]
- ),
- FunctionExpression(
- "struct_pack",
- *[ColumnExpression(f"{config.target_name}.{c}") for c in target_rel.columns],
- ).alias(config.header_column_name)
- )
+ joined_rel = source_rel.cross(target_rel).select(
+ StarExpression(exclude=[f"{config.target_name}.{c}" for c in target_rel.columns]),
+ FunctionExpression(
+ "struct_pack",
+ *[ColumnExpression(f"{config.target_name}.{c}") for c in target_rel.columns],
+ ).alias(config.header_column_name),
)
entities[config.new_entity_name or config.entity_name] = joined_rel
diff --git a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py
index 1257306e..f3ce03f4 100644
--- a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py
+++ b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py
@@ -359,7 +359,17 @@ def object_to_spark_literal(obj: SparkLiteralType) -> Column:
def df_is_empty(dataframe: DataFrame) -> bool:
"""Return whether a DataFrame is empty."""
- return not dataframe.select(lit(1)).head(1)
+
+ if not dataframe.select(lit(1)).head(1):
+ return True
+
+ return (
+ not dataframe.dropna(
+ how="all", subset=[cl for cl in dataframe.columns if cl != RECORD_INDEX_COLUMN_NAME]
+ )
+ .select(lit(1))
+ .head(1)
+ )
def _spark_read_parquet(self, path: URI, **kwargs) -> DataFrame:
@@ -457,7 +467,7 @@ def spark_get_entity_count(cls):
def _spark_check_entity_empty(self, entity: DataFrame) -> bool: # pylint: disable=W0613
- return entity.count() == 0
+ return df_is_empty(entity)
def spark_check_entity_empty(cls):
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py
index 0e81cf46..74b7b3a8 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py
@@ -25,6 +25,7 @@
relation_is_empty
)
from dve.core_engine.backends.utilities import datetime_format_to_regex
+from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
from dve.metadata_parser.domain_types import formatteddatetime
@pytest.fixture
@@ -266,6 +267,18 @@ def test_relation_is_empty(temp_ddb_conn: DuckDBPyConnection):
_, con = temp_ddb_conn
rel = con.sql("SELECT 'abc' AS test").filter("test IS NULL")
assert relation_is_empty(rel)
+
+ rel2 = con.sql("SELECT NULL as fld1, NULL as fld2 UNION ALL SELECT NULL as fld1, NULL as fld2")
+ assert relation_is_empty(rel2)
+
+ rel3 = con.sql("SELECT 1 as fld1, NULL as fld2 UNION ALL SELECT NULL as fld1, NULL as fld2")
+ assert not relation_is_empty(rel3)
+
+ rel4 = con.sql(f"SELECT 1 as {RECORD_INDEX_COLUMN_NAME}, NULL as fld1, NULL as fld2 UNION ALL SELECT 2, NULL, NULL")
+ assert relation_is_empty(rel4)
+
+ rel5 = con.sql(f"SELECT 1 as {RECORD_INDEX_COLUMN_NAME}, NULL as fld1, NULL as fld2 UNION ALL SELECT 2, NULL, 'hello'")
+ assert not relation_is_empty(rel5)
@pytest.mark.parametrize("date_format, expected_regex", [("%Y-%m-%d", r"^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$"),
("%Y-%m-%dT%H:%M:%S", r"^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$"),
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py
index 41c9e93e..34ecb6ca 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py
@@ -30,7 +30,9 @@
get_type_from_annotation,
object_to_spark_literal,
python_to_java_datetime_format,
+ df_is_empty
)
+from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
from dve.metadata_parser.domain_types import conformatteddate
from .....fixtures import spark # pylint: disable=unused-import
@@ -354,3 +356,31 @@ def test_spark_filter_contract_errors(spark: SparkSession, example_data_contract
("%Y-%m-%dT%H:%M:%S", "yyyy-MM-dd'T'HH:mm:ss")])
def test_python_to_java_datetime_format(python_dt, expected_java_dt):
assert python_to_java_datetime_format(python_dt) == expected_java_dt
+
+
+def test_df_is_empty(spark):
+ df = spark.createDataFrame([], schema=StructType([StructField("col1", StringType()), StructField("col2", LongType())]))
+ assert df_is_empty(df)
+ df2 = spark.createDataFrame([(None, None), (None, None), (None, None)],
+ schema=StructType([StructField("col1", StringType()), StructField("col2", LongType())]))
+ assert df_is_empty(df2)
+ df3 = spark.createDataFrame([(None, None), (None, None), (None, 3)],
+ schema=StructType([StructField("col1", StringType()), StructField("col2", LongType())]))
+ assert not df_is_empty(df3)
+
+ _schema_w_ri = schema=StructType([
+ StructField(RECORD_INDEX_COLUMN_NAME, LongType()),
+ StructField("col1", StringType()),
+ StructField("col2", LongType())
+ ])
+
+ df4 = spark.createDataFrame([(1, None, None), (2, None, None), (3, None, None)],
+ schema=_schema_w_ri)
+ assert df_is_empty(df4)
+
+ df5 = df4.union(spark.createDataFrame([(4, None, 8)], schema=_schema_w_ri))
+ assert not df_is_empty(df5)
+
+
+
+
From 562ca2bcbbebc0c42112ccc6272ad80d7c6fa855 Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Mon, 12 Oct 2026 12:19:55 +0100
Subject: [PATCH 34/47] docs: fix incorrect text in some of the indexes
---
.../package_documentation/business_rules/index.md | 2 +-
.../package_documentation/data_contract/index.md | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/docs/advanced_guidance/package_documentation/business_rules/index.md b/docs/advanced_guidance/package_documentation/business_rules/index.md
index e64ed66e..0b1f3aea 100644
--- a/docs/advanced_guidance/package_documentation/business_rules/index.md
+++ b/docs/advanced_guidance/package_documentation/business_rules/index.md
@@ -26,6 +26,6 @@ title: Business Rules Documentation
---
- [:octicons-arrow-right-24: Read about the `Submission Feedback` objects here](udfs.md)
+ [:octicons-arrow-right-24: Read about the `UDFs` objects here](udfs.md)
\ No newline at end of file
diff --git a/docs/advanced_guidance/package_documentation/data_contract/index.md b/docs/advanced_guidance/package_documentation/data_contract/index.md
index df4bf23b..a801650d 100644
--- a/docs/advanced_guidance/package_documentation/data_contract/index.md
+++ b/docs/advanced_guidance/package_documentation/data_contract/index.md
@@ -8,13 +8,13 @@ title: Data Contract Documentation
---
- [:octicons-arrow-right-24: Read about the `Operation` objects here](readers.md)
+ [:octicons-arrow-right-24: Read about the `Readers` objects here](readers.md)
- :material-language-python:{ .lg .middle } __Domain Types__
---
- [:octicons-arrow-right-24: Read about the `Business Rules` objects here](domain_types.md)
+ [:octicons-arrow-right-24: Read about the `Domain Types` objects here](domain_types.md)
\ No newline at end of file
From 3c71d18d2bc4a46cbc74ec499705a5a508902945 Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Mon, 12 Oct 2026 12:30:53 +0100
Subject: [PATCH 35/47] refactor: removed ods udfs as they were functionally
useless
---
.../core_engine/functions/implementations.py | 110 ------------------
1 file changed, 110 deletions(-)
diff --git a/src/dve/core_engine/functions/implementations.py b/src/dve/core_engine/functions/implementations.py
index bae6d98b..9d5928d2 100644
--- a/src/dve/core_engine/functions/implementations.py
+++ b/src/dve/core_engine/functions/implementations.py
@@ -42,116 +42,6 @@ def date_in_current_financial_year(test_date: datetime.date) -> bool:
return test_date >= current_fiscal_year_start
-def is_valid_ods_code(check_ods_code: str) -> bool:
- """A test ODS code lookup, contains some actual valid codes"""
- if (not check_ods_code) or (not isinstance(check_ods_code, str)):
- return False
- valid_ods_codes = [
- "EE142976",
- "EE144430",
- "EE143473",
- "EE148112",
- "EE142863",
- "EE147862",
- "EE142472",
- "EE141208",
- "EE143149",
- "EE140862",
- "EE140319",
- "EE144899",
- "EE144475",
- "EE141850",
- "EE147934",
- "EE141068",
- "EE143825",
- "EE147805",
- "EE143489",
- "EE146813",
- "EE145703",
- "EE148295",
- "EE140156",
- "EE145502",
- "EE148396",
- "EE144126",
- "EE145590",
- "EE141566",
- "EE142081",
- "EE143640",
- "EE144911",
- "EE145935",
- "EE145279",
- "EE143156",
- "EE146556",
- "EE140781",
- "EE144734",
- "EE144841",
- "EE140419",
- "EE140040",
- "EE147342",
- "EE143330",
- "EE140926",
- "EE140926",
- "EE146438",
- "EE142137",
- "EE143856",
- "EE141067",
- "EE148534",
- "EE141310",
- "EE146899",
- "EE146996",
- "EE147487",
- "EE148447",
- "EE144311",
- "EE142147",
- "EE147605",
- "EE142117",
- "EE144087",
- "EE147326",
- "EE147614",
- "EE143703",
- "EE146135",
- "EE140782",
- "EE143603",
- "EE143554",
- "EE146659",
- "EE140321",
- "EE141185",
- "EE147648",
- "EE144527",
- "EE142680",
- "EE141620",
- "EE145274",
- "EE146251",
- "EE148209",
- "EE142574",
- "EE148162",
- "EE143118",
- "EE142977",
- "EE147798",
- "EE147902",
- "EE145780",
- "EE146992",
- "EE142916",
- "EE144777",
- "EE146935",
- "EE145586",
- "EE144570",
- "EE147122",
- "EE140874",
- "EE141338",
- "EE143244",
- ]
- return check_ods_code in valid_ods_codes
-
-
-def is_valid_national_org(check_org_code: str) -> bool:
- """Simple test org code lookup"""
- if (not check_org_code) or (not isinstance(check_org_code, str)):
- return False
- valid_org_codes = ["ORG01", "ORG02"]
- return check_org_code in valid_org_codes
-
-
def check_correct_numeric_signage(
val: Union[float, int, Decimal], expected_sign: str = "+/-"
) -> Optional[bool]:
From 54f704bfe6c3ceb8b6cd1474613cd1db7f007e09 Mon Sep 17 00:00:00 2001
From: stevenhsd <56357022+stevenhsd@users.noreply.github.com>
Date: Mon, 12 Oct 2026 22:00:13 +0100
Subject: [PATCH 36/47] fix: adjust regex check for timezone offsets, allowing
: to be included (#181)
---
.../implementations/spark/spark_helpers.py | 2 +-
src/dve/core_engine/backends/utilities.py | 3 +-
.../test_duckdb/test_duckdb_helpers.py | 29 +++++++++++++++----
.../test_spark/test_spark_helpers.py | 18 +++++++++++-
4 files changed, 43 insertions(+), 9 deletions(-)
diff --git a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py
index f3ce03f4..5041095b 100644
--- a/src/dve/core_engine/backends/implementations/spark/spark_helpers.py
+++ b/src/dve/core_engine/backends/implementations/spark/spark_helpers.py
@@ -112,7 +112,7 @@ def __post_init__(self):
"%H": "HH",
"%M": "mm",
"%S": "ss",
- "%z": "XX",
+ "%z": "XXX",
"%Z": "z",
}
"""A mapping of python to java datetime component formats. Not exhaustive but will hopefully support
diff --git a/src/dve/core_engine/backends/utilities.py b/src/dve/core_engine/backends/utilities.py
index af2f7573..7cb625d4 100644
--- a/src/dve/core_engine/backends/utilities.py
+++ b/src/dve/core_engine/backends/utilities.py
@@ -40,7 +40,7 @@
"H": r"[0-9]{2}",
"M": r"[0-9]{2}",
"S": r"[0-9]{2}",
- "z": r"(\+|\-)?[0-9]+(\.[0-9]*)?",
+ "z": r"[+-][0-9]+(:[0-9]{2})?(\.[0-9]*)?",
"Z": r"[A-Z]{0,3}",
}
@@ -54,7 +54,6 @@ def datetime_format_to_regex(format_str: str) -> str:
Helper function to convert python datetime formats to regexp string checks for casting
purposes.
"""
-
tokens = list(format_str.replace("%", ""))
for idx, tkn in enumerate(tokens):
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py
index 74b7b3a8..58dd423a 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py
@@ -26,7 +26,7 @@
)
from dve.core_engine.backends.utilities import datetime_format_to_regex
from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
-from dve.metadata_parser.domain_types import formatteddatetime
+from dve.metadata_parser.domain_types import conformatteddate, formatteddatetime
@pytest.fixture
def casting_test_table(temp_ddb_conn):
@@ -225,8 +225,8 @@ def test_duckdb_rel_to_dictionaries(temp_ddb_conn: DuckDBPyConnection,
("date_test", datetime.date,"TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"date_test\"), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"date_test\"), '%Y-%m-%d') ELSE NULL END as DATE)"),
("timestamp_test", datetime.datetime, "TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"timestamp_test\"), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"timestamp_test\"), '%Y-%m-%dT%H:%M:%S') ELSE NULL END as TIMESTAMP)"),
("list_int_field", list[int], "TRY_CAST(LIST_TRANSFORM(\"list_int_field\", x -> TRIM(\"x\")) as BIGINT[])"),
- ("basic_model", BasicModel, "TRY_CAST(STRUCT_PACK(\"str_field\":= TRIM(\"basic_model\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"basic_model\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"basic_model\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)) as STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP))"),
- ("another_model", AnotherModel, "TRY_CAST(STRUCT_PACK(\"unique_id\":= TRIM(\"another_model\".unique_id),\"basic_models\":= LIST_TRANSFORM(\"another_model\".basic_models, x -> STRUCT_PACK(\"str_field\":= TRIM(\"x\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"x\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"x\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)))) as STRUCT(unique_id BIGINT, basic_models STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP)[]))")])
+ ("basic_model", BasicModel, "TRY_CAST(STRUCT_PACK(\"str_field\":= TRIM(\"basic_model\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"basic_model\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]+(:[0-9]{2})?(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"basic_model\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)) as STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP))"),
+ ("another_model", AnotherModel, "TRY_CAST(STRUCT_PACK(\"unique_id\":= TRIM(\"another_model\".unique_id),\"basic_models\":= LIST_TRANSFORM(\"another_model\".basic_models, x -> STRUCT_PACK(\"str_field\":= TRIM(\"x\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"x\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]+(:[0-9]{2})?(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"x\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)))) as STRUCT(unique_id BIGINT, basic_models STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP)[]))")])
def test_get_duckdb_cast_statement_from_annotation(field_name, field_type, cast_statement):
assert get_duckdb_cast_statement_from_annotation(field_name, field_type) == cast_statement
@@ -283,6 +283,25 @@ def test_relation_is_empty(temp_ddb_conn: DuckDBPyConnection):
@pytest.mark.parametrize("date_format, expected_regex", [("%Y-%m-%d", r"^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$"),
("%Y-%m-%dT%H:%M:%S", r"^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$"),
("%m/%d/%Y", r"^[0-9]{2}/[0-9]{2}/[0-9]{4}$"),
- ("%Y/%m/%d %H:%M:%S%z", r"^[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(\.[0-9]*)?$")])
+ ("%Y/%m/%d %H:%M:%S%z", r"^[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]+(:[0-9]{2})?(\.[0-9]*)?$")])
+
def test_datetime_format_to_regex(date_format, expected_regex):
- assert datetime_format_to_regex(date_format) == expected_regex
\ No newline at end of file
+ assert datetime_format_to_regex(date_format) == expected_regex
+
+
+@pytest.mark.parametrize("datetime_str, datetime_format, expected_value", [("2020-06-11", "%Y-%m-%d", datetime.date(2020,6,11)),
+ ("15/11/2018", "%d/%m/%Y", datetime.date(2018,11,15)),
+ ("27/14/2027", "%d/%m/%Y", None),
+ ("2014-05-27", "%d/%m/%y", None),
+ ("2020-03-15T13:24:54+0000", "%Y-%m-%dT%H:%M:%S%z", datetime.datetime(2020, 3, 15, 13, 24, 54)),
+ ("2020-03-15T13:24:54+00:00", "%Y-%m-%dT%H:%M:%S%z", datetime.datetime(2020, 3, 15, 13, 24, 54))])
+def test_datetime_casting(temp_ddb_conn: DuckDBPyConnection, datetime_str, datetime_format, expected_value):
+ _, con = temp_ddb_conn
+ if isinstance(expected_value, datetime.datetime):
+ type_ = formatteddatetime(date_format=datetime_format)
+ else:
+ type_ = conformatteddate(date_format=datetime_format)
+ cast_statement = get_duckdb_cast_statement_from_annotation("test", type_)
+ rel = con.sql(f"SELECT '{datetime_str}' as test")
+ check = rel.project(cast_statement).fetchone()[0]
+ assert check == expected_value
\ No newline at end of file
diff --git a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py
index 34ecb6ca..9f7c9282 100644
--- a/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py
+++ b/tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py
@@ -33,7 +33,7 @@
df_is_empty
)
from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
-from dve.metadata_parser.domain_types import conformatteddate
+from dve.metadata_parser.domain_types import conformatteddate, formatteddatetime
from .....fixtures import spark # pylint: disable=unused-import
@@ -380,6 +380,22 @@ def test_df_is_empty(spark):
df5 = df4.union(spark.createDataFrame([(4, None, 8)], schema=_schema_w_ri))
assert not df_is_empty(df5)
+
+@pytest.mark.parametrize("datetime_str, datetime_format, expected_value", [("2020-06-11", "%Y-%m-%d", dt.date(2020,6,11)),
+ ("15/11/2018", "%d/%m/%Y", dt.date(2018,11,15)),
+ ("27/14/2027", "%d/%m/%Y", None),
+ ("2014-05-27", "%d/%m/%y", None),
+ ("2020-03-15T13:24:54+00:00", "%Y-%m-%dT%H:%M:%S%z", dt.datetime(2020, 3, 15, 13, 24, 54))])
+def test_datetime_casting(spark, datetime_str, datetime_format, expected_value):
+
+ if isinstance(expected_value, dt.datetime):
+ type_ = formatteddatetime(date_format=datetime_format)
+ else:
+ type_ = conformatteddate(date_format=datetime_format)
+ cast_statement = get_spark_cast_statement_from_annotation("test", type_)
+ df = spark.createDataFrame([(datetime_str,)], schema=StructType([StructField("test", StringType())]))
+ check = df.select(cast_statement.alias("test")).first().test
+ assert check == expected_value
From 665e1fad7ee23a31623bf0b20c001f843f3c715d Mon Sep 17 00:00:00 2001
From: stevenhsd <56357022+stevenhsd@users.noreply.github.com>
Date: Tue, 13 Oct 2026 15:30:17 +0100
Subject: [PATCH 37/47] refactor: rename of error code and messages related to
empty entities post business rules (#187)
---
docs/user_guidance/entity_relationships.md | 4 +--
.../core_engine/configuration/v1/__init__.py | 4 +--
.../core_engine/configuration/v1/hierarchy.py | 8 ++---
src/dve/pipeline/pipeline.py | 4 +--
tests/test_core_engine/test_hierarchy.py | 32 +++++++++----------
tests/testdata/flights/flights.dischema.json | 12 +++----
.../flights/flights_spark.dischema.json | 8 ++---
7 files changed, 36 insertions(+), 36 deletions(-)
diff --git a/docs/user_guidance/entity_relationships.md b/docs/user_guidance/entity_relationships.md
index bf0f6c19..68fe76d7 100644
--- a/docs/user_guidance/entity_relationships.md
+++ b/docs/user_guidance/entity_relationships.md
@@ -48,8 +48,8 @@ There is also the functionality to customise errors related to either missing pa
- `missing_parent_id_error_message`: the error message to display if a record is rejected as it has no valid parent record
- `no_valid_records_error_code`: the error code to display if parent records are removed due to no valid children in a mandatory field
- `no_valid_records_error_message`: the error message to display if parent records are removed due to no valid children in a mandatory field
-- `empty_entity_error_code`: the error code to display if a __mandatory__ entity has any records post filtering
-- `empty_entity_error_message`: the error message to display if a __mandatory__ entity has any records post filtering
+- `empty_entity_post_br_error_code`: the error code to display if a __mandatory__ entity has any records post filtering
+- `empty_entity_post_br_error_message`: the error message to display if a __mandatory__ entity has any records post filtering
!!! note
Specifying root entities is __optional__. Root entities will be inferred based on their absence.
diff --git a/src/dve/core_engine/configuration/v1/__init__.py b/src/dve/core_engine/configuration/v1/__init__.py
index 634cc112..dbccdab6 100644
--- a/src/dve/core_engine/configuration/v1/__init__.py
+++ b/src/dve/core_engine/configuration/v1/__init__.py
@@ -116,9 +116,9 @@ class _LinkageConfig(BaseModel):
"Records removed due to no valid parent record"
)
"""The error code to emit if the entity contains records that are orphaned by parent record rejections""" # pylint: disable=C0301
- empty_entity_error_code: ErrorCode = "EmptyEntity"
+ empty_entity_post_br_error_code: ErrorCode = "EmptyEntity"
"""The error code to emit if a mandatory entity has no valid remaining records"""
- empty_entity_error_message: ErrorMessage = "no valid records remaining"
+ empty_entity_post_br_error_message: ErrorMessage = "no valid records remaining"
"""The error message to emit if a mandatory entity has no valid remaining records"""
@model_validator(mode="after")
diff --git a/src/dve/core_engine/configuration/v1/hierarchy.py b/src/dve/core_engine/configuration/v1/hierarchy.py
index 35997eb1..57b64506 100644
--- a/src/dve/core_engine/configuration/v1/hierarchy.py
+++ b/src/dve/core_engine/configuration/v1/hierarchy.py
@@ -26,8 +26,8 @@ class HierarchyNode(BaseModel):
missing_parent_id_error_message: Optional[ErrorMessage] = (
"Records removed due to no valid parent record"
)
- empty_entity_error_code: ErrorCode = "EmptyEntity"
- empty_entity_error_message: ErrorMessage = "no valid records remaining"
+ empty_entity_post_br_error_code: ErrorCode = "EmptyEntity"
+ empty_entity_post_br_error_message: ErrorMessage = "no valid records remaining"
@model_validator(mode="after")
def validate_empty_error_details(self):
@@ -36,8 +36,8 @@ def validate_empty_error_details(self):
non mandatory nodes/entities
"""
if not self.mandatory:
- self.empty_entity_error_code = None
- self.empty_entity_error_message = None
+ self.empty_entity_post_br_error_code = None
+ self.empty_entity_post_br_error_message = None
return self
def get_descendents(self) -> list["HierarchyNode"]:
diff --git a/src/dve/pipeline/pipeline.py b/src/dve/pipeline/pipeline.py
index 2cf00553..1d29b1b2 100644
--- a/src/dve/pipeline/pipeline.py
+++ b/src/dve/pipeline/pipeline.py
@@ -581,10 +581,10 @@ def check_mandatory_entities_have_records(
entity=entity_name,
record=None,
error_location=entity_name,
- error_message=node.empty_entity_error_message,
+ error_message=node.empty_entity_post_br_error_message,
failure_type="submission",
error_type="submission",
- error_code=node.empty_entity_error_code,
+ error_code=node.empty_entity_post_br_error_code,
category="Empty entity",
)
)
diff --git a/tests/test_core_engine/test_hierarchy.py b/tests/test_core_engine/test_hierarchy.py
index bbce74f0..43a0a8f7 100644
--- a/tests/test_core_engine/test_hierarchy.py
+++ b/tests/test_core_engine/test_hierarchy.py
@@ -269,8 +269,8 @@ def test_linkage_config_load():
"no_valid_records_error_message": "parent record removed as no valid child records",
"missing_parent_id_error_code": null,
"missing_parent_id_error_message": null,
- "empty_entity_error_code": null,
- "empty_entity_error_message": null,
+ "empty_entity_post_br_error_code": null,
+ "empty_entity_post_br_error_message": null,
"children": {
"ds_003": {
"parent_entity": "ds_001",
@@ -282,8 +282,8 @@ def test_linkage_config_load():
"no_valid_records_error_message": "parent record removed as no valid child records",
"missing_parent_id_error_code": "DS003NoParent",
"missing_parent_id_error_message": "record removed as no parent",
- "empty_entity_error_code": null,
- "empty_entity_error_message": null,
+ "empty_entity_post_br_error_code": null,
+ "empty_entity_post_br_error_message": null,
"children": {}
},
"ds_101": {
@@ -296,8 +296,8 @@ def test_linkage_config_load():
"no_valid_records_error_message": "{{ ds_001_id }} removed as no valid ds_101 records",
"missing_parent_id_error_code": "DS101NoParent",
"missing_parent_id_error_message": "record removed as no parent",
- "empty_entity_error_code": null,
- "empty_entity_error_message": null,
+ "empty_entity_post_br_error_code": null,
+ "empty_entity_post_br_error_message": null,
"children": {
"ds_201": {
"parent_entity": "ds_101",
@@ -309,8 +309,8 @@ def test_linkage_config_load():
"no_valid_records_error_message": "parent record removed as no valid child records",
"missing_parent_id_error_code": "DS201NoParent",
"missing_parent_id_error_message": "record removed as no parent",
- "empty_entity_error_code": null,
- "empty_entity_error_message": null,
+ "empty_entity_post_br_error_code": null,
+ "empty_entity_post_br_error_message": null,
"children": {
"ds_202": {
"parent_entity": "ds_201",
@@ -322,8 +322,8 @@ def test_linkage_config_load():
"no_valid_records_error_message": "parent record removed as no valid child records",
"missing_parent_id_error_code": "MissingParentRecord",
"missing_parent_id_error_message": "Records removed due to no valid parent record",
- "empty_entity_error_code": "EmptyEntity",
- "empty_entity_error_message": "no valid records remaining",
+ "empty_entity_post_br_error_code": "EmptyEntity",
+ "empty_entity_post_br_error_message": "no valid records remaining",
"children": {}
}
}
@@ -351,8 +351,8 @@ def test_linkage_config_load():
"no_valid_records_error_message": "{{ ds_001_id }} removed as no valid ds_101 records",
"missing_parent_id_error_code": "DS101NoParent",
"missing_parent_id_error_message": "record removed as no parent",
- "empty_entity_error_code": null,
- "empty_entity_error_message": null,
+ "empty_entity_post_br_error_code": null,
+ "empty_entity_post_br_error_message": null,
"children": {
"ds_201": {
"parent_entity": "ds_101",
@@ -364,8 +364,8 @@ def test_linkage_config_load():
"no_valid_records_error_message": "parent record removed as no valid child records",
"missing_parent_id_error_code": "DS201NoParent",
"missing_parent_id_error_message": "record removed as no parent",
- "empty_entity_error_code": null,
- "empty_entity_error_message": null,
+ "empty_entity_post_br_error_code": null,
+ "empty_entity_post_br_error_message": null,
"children": {
"ds_202": {
"parent_entity": "ds_201",
@@ -377,8 +377,8 @@ def test_linkage_config_load():
"no_valid_records_error_message": "parent record removed as no valid child records",
"missing_parent_id_error_code": "MissingParentRecord",
"missing_parent_id_error_message": "Records removed due to no valid parent record",
- "empty_entity_error_code": "EmptyEntity",
- "empty_entity_error_message": "no valid records remaining",
+ "empty_entity_post_br_error_code": "EmptyEntity",
+ "empty_entity_post_br_error_message": "no valid records remaining",
"children": {}
}
}
diff --git a/tests/testdata/flights/flights.dischema.json b/tests/testdata/flights/flights.dischema.json
index a0342cc0..a4ef67e1 100644
--- a/tests/testdata/flights/flights.dischema.json
+++ b/tests/testdata/flights/flights.dischema.json
@@ -161,8 +161,8 @@
"country": {
"is_root_entity": true,
"mandatory": true,
- "empty_entity_error_code": "NoValidCountries",
- "empty_entity_error_message": "File Rejected - There are no valid country records"
+ "empty_entity_post_br_error_code": "NoValidCountries",
+ "empty_entity_post_br_error_message": "File Rejected - There are no valid country records"
},
"airport": {
"parent_entity": "country",
@@ -174,8 +174,8 @@
"missing_parent_id_error_message": "Record rejected - No valid country id found for airport",
"no_valid_records_error_code": "CountryHasNoAirport",
"no_valid_records_error_message": "Group rejected - Unable to find any valid airports",
- "empty_entity_error_code": "NoValidAirports",
- "empty_entity_error_message": "File Rejected - There are no valid airport records"
+ "empty_entity_post_br_error_code": "NoValidAirports",
+ "empty_entity_post_br_error_message": "File Rejected - There are no valid airport records"
},
"staff": {
"parent_entity": "airport",
@@ -187,8 +187,8 @@
"missing_parent_id_error_message": "Record rejected - No valid airport id found for staff. Airport ID = {{ airport_id }}, Staff ID = {{ staff_id }}",
"no_valid_records_error_code": "AirportHasNoStaff",
"no_valid_records_error_message": "Group rejected - Airport has no valid staff. Airport ID = {{ airport_id }}",
- "empty_entity_error_code": "NoValidStaff",
- "empty_entity_error_message": "File Rejected - There are no valid staff records"
+ "empty_entity_post_br_error_code": "NoValidStaff",
+ "empty_entity_post_br_error_message": "File Rejected - There are no valid staff records"
},
"flights": {
"parent_entity": "airport",
diff --git a/tests/testdata/flights/flights_spark.dischema.json b/tests/testdata/flights/flights_spark.dischema.json
index b318f823..d94e07d3 100644
--- a/tests/testdata/flights/flights_spark.dischema.json
+++ b/tests/testdata/flights/flights_spark.dischema.json
@@ -174,8 +174,8 @@
"missing_parent_id_error_message": "Record rejected - No valid country id found for airport",
"no_valid_records_error_code": "CountryHasNoAirport",
"no_valid_records_error_message": "Group rejected - Unable to find any valid airports",
- "empty_entity_error_code": "NoValidAirports",
- "empty_entity_error_message": "File Rejected - There are no valid airport records"
+ "empty_entity_post_br_error_code": "NoValidAirports",
+ "empty_entity_post_br_error_message": "File Rejected - There are no valid airport records"
},
"staff": {
"parent_entity": "airport",
@@ -187,8 +187,8 @@
"missing_parent_id_error_message": "Record rejected - No valid airport id found for staff. Airport ID = {{ airport_id }}, Staff ID = {{ staff_id }}",
"no_valid_records_error_code": "AirportHasNoStaff",
"no_valid_records_error_message": "Group rejected - Airport has no valid staff. Airport ID = {{ airport_id }}",
- "empty_entity_error_code": "NoValidStaff",
- "empty_entity_error_message": "File Rejected - There are no valid staff records"
+ "empty_entity_post_br_error_code": "NoValidStaff",
+ "empty_entity_post_br_error_message": "File Rejected - There are no valid staff records"
},
"flights": {
"parent_entity": "airport",
From 17dc21b41d4c630a7258a05911ccdf82b8bddffd Mon Sep 17 00:00:00 2001
From: George Robertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Thu, 22 Oct 2026 11:14:33 +0100
Subject: [PATCH 38/47] refactor: uppercase all entity names in error report
(#190)
---
src/dve/pipeline/pipeline.py | 2 +-
tests/test_pipeline/test_spark_pipeline.py | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/src/dve/pipeline/pipeline.py b/src/dve/pipeline/pipeline.py
index 1d29b1b2..d118e4c2 100644
--- a/src/dve/pipeline/pipeline.py
+++ b/src/dve/pipeline/pipeline.py
@@ -917,7 +917,7 @@ def _get_error_dataframes(self, submission_id: str):
.alias("error_type") # type: ignore
)
df = df.select(
- pl.col("ReportingEntity").alias("Table"), # type: ignore
+ pl.col("ReportingEntity").str.to_uppercase().alias("Table"), # type: ignore
pl.col("error_type").alias("Type"), # type: ignore
pl.col("ErrorCode").alias("Error_Code"), # type: ignore
pl.col("ReportingField").alias("Data_Item"), # type: ignore
diff --git a/tests/test_pipeline/test_spark_pipeline.py b/tests/test_pipeline/test_spark_pipeline.py
index 642a7ead..cbb95b60 100644
--- a/tests/test_pipeline/test_spark_pipeline.py
+++ b/tests/test_pipeline/test_spark_pipeline.py
@@ -449,7 +449,7 @@ def test_error_report_where_report_is_expected( # pylint: disable=redefined-out
OrderedDict(
**{
"Type": "Record Rejection",
- "Group": "planets",
+ "Group": "PLANETS",
"Data Item Submission Name": "orbitalPeriod",
"Category": "Bad value",
"Error Code": "LONG_ORBIT",
@@ -459,7 +459,7 @@ def test_error_report_where_report_is_expected( # pylint: disable=redefined-out
OrderedDict(
**{
"Type": "Record Rejection",
- "Group": "planets",
+ "Group": "PLANETS",
"Data Item Submission Name": "gravity",
"Category": "Bad value",
"Error Code": "STRONG_GRAVITY",
@@ -477,7 +477,7 @@ def test_error_report_where_report_is_expected( # pylint: disable=redefined-out
assert error_data_records == [
OrderedDict(
**{
- "Group": "planets",
+ "Group": "PLANETS",
"Type": "Record Rejection",
"Error Code": "LONG_ORBIT",
"Data Item Submission Name": "orbitalPeriod",
@@ -490,7 +490,7 @@ def test_error_report_where_report_is_expected( # pylint: disable=redefined-out
),
OrderedDict(
**{
- "Group": "planets",
+ "Group": "PLANETS",
"Type": "Record Rejection",
"Error Code": "STRONG_GRAVITY",
"Data Item Submission Name": "gravity",
From ec64b426f4a2b1f6eef9097cf3e75516c9a8ddc5 Mon Sep 17 00:00:00 2001
From: George Robertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Thu, 22 Oct 2026 13:39:58 +0100
Subject: [PATCH 39/47] feat: add error_report config to toggle record
rejection reporting (#189)
Co-authored-by: vaibhavtiwari16 <249828730+vaibhavtiwari16@users.noreply.github.com>
---
.../core_engine/configuration/v1/__init__.py | 8 +++++
src/dve/pipeline/foundry_ddb_pipeline.py | 7 +++--
src/dve/pipeline/pipeline.py | 31 +++++++++++++++----
tests/features/animals.feature | 4 +--
tests/test_pipeline/test_duckdb_pipeline.py | 2 +-
tests/test_pipeline/test_spark_pipeline.py | 2 +-
tests/testdata/animals/animals.dischema.json | 3 ++
7 files changed, 45 insertions(+), 12 deletions(-)
diff --git a/src/dve/core_engine/configuration/v1/__init__.py b/src/dve/core_engine/configuration/v1/__init__.py
index dbccdab6..e860c91c 100644
--- a/src/dve/core_engine/configuration/v1/__init__.py
+++ b/src/dve/core_engine/configuration/v1/__init__.py
@@ -248,6 +248,12 @@ class V1TransformationConfig(BaseModel):
"""Complex rules."""
+class V1ErrorReportConfig(BaseModel):
+ """Configuration for the error report component of the dataset."""
+
+ record_rejection_reporting: bool = True
+
+
class V1EngineConfig(BaseEngineConfig):
"""A loader for the first version of the configuration language."""
@@ -255,6 +261,8 @@ class V1EngineConfig(BaseEngineConfig):
"""The data contract configuration."""
transformations: V1TransformationConfig = Field(default_factory=V1TransformationConfig)
"""The transformation/rules configuration."""
+ error_report: V1ErrorReportConfig = Field(default_factory=V1ErrorReportConfig)
+ """The error report configuration."""
_rule_store_rules: dict[RuleName, BusinessComponentSpecConfigUnion] = PrivateAttr(
default_factory=dict
)
diff --git a/src/dve/pipeline/foundry_ddb_pipeline.py b/src/dve/pipeline/foundry_ddb_pipeline.py
index a5c24638..56d202a7 100644
--- a/src/dve/pipeline/foundry_ddb_pipeline.py
+++ b/src/dve/pipeline/foundry_ddb_pipeline.py
@@ -10,6 +10,7 @@
duckdb_get_entity_count,
duckdb_write_parquet,
)
+from dve.core_engine.configuration.v1 import V1ErrorReportConfig
from dve.core_engine.exceptions import CriticalProcessingError
from dve.core_engine.models import SubmissionInfo
from dve.core_engine.type_hints import URI
@@ -101,10 +102,12 @@ def apply_business_rules(
return submission_info, SubmissionStatus(processing_failed=True)
def error_report(
- self, submission_info: SubmissionInfo, submission_status: Optional[SubmissionStatus] = None
+ self, submission_info: SubmissionInfo,
+ submission_status: Optional[SubmissionStatus] = None ,
+ error_report_config: Optional[V1ErrorReportConfig] = None,
):
try:
- return super().error_report(submission_info, submission_status)
+ return super().error_report(submission_info, submission_status,error_report_config)
except Exception as exc: # pylint: disable=W0718
self._logger.exception("Error reports raised exception:")
sub_stats = None
diff --git a/src/dve/pipeline/pipeline.py b/src/dve/pipeline/pipeline.py
index d118e4c2..8d9cfdb6 100644
--- a/src/dve/pipeline/pipeline.py
+++ b/src/dve/pipeline/pipeline.py
@@ -7,7 +7,6 @@
from collections import defaultdict
from collections.abc import Generator, Iterable, Iterator
from concurrent.futures import Executor, Future, ThreadPoolExecutor
-from functools import lru_cache
from itertools import starmap
from threading import Lock
from typing import Any, Optional, Union
@@ -35,6 +34,7 @@
from dve.core_engine.backends.readers.utilities import get_all_model_fields
from dve.core_engine.backends.types import EntityType
from dve.core_engine.backends.utilities import stringify_model
+from dve.core_engine.configuration.v1 import V1ErrorReportConfig
from dve.core_engine.configuration.v1.hierarchy import EntityHierarchy
from dve.core_engine.exceptions import CriticalProcessingError
from dve.core_engine.loggers import get_logger
@@ -886,8 +886,7 @@ def _publish_error_aggregates(self, submission_id: str, aggregates_df: pl.DataFr
aggregates_df.write_parquet(output_uri)
return output_uri
- @lru_cache() # noqa: B019
- def _get_error_dataframes(self, submission_id: str):
+ def _get_error_dataframes(self, submission_id: str, include_record_rejections: bool = True):
if not self.processed_files_path:
raise AttributeError("processed files path not provided")
@@ -935,12 +934,19 @@ def _get_error_dataframes(self, submission_id: str):
errors_dfs.append(df)
errors_df = pl.concat(errors_dfs, how="align") # type: ignore
+ if not include_record_rejections:
+ errors_df = errors_df.filter(
+ pl.col("Type") != ErrorReportCategories.RECORD_REJECTION.reporting_name
+ )
aggregates = calculate_aggregates(errors_df)
return errors_df, aggregates
def error_report(
- self, submission_info: SubmissionInfo, submission_status: Optional[SubmissionStatus] = None
+ self,
+ submission_info: SubmissionInfo,
+ submission_status: Optional[SubmissionStatus] = None,
+ error_report_config: Optional[V1ErrorReportConfig] = None,
) -> tuple[
SubmissionInfo, SubmissionStatus, Optional[SubmissionStatisticsRecord], Optional[URI]
]:
@@ -962,7 +968,15 @@ def error_report(
raise AttributeError("processed files path not provided")
self._logger.info("Reading error dataframes")
- errors_df, aggregates = self._get_error_dataframes(submission_info.submission_id)
+ if error_report_config is None:
+ if not self.rules_path:
+ raise AttributeError("rules path not provided")
+ _, config, _ = load_config(submission_info.dataset_id, self.rules_path)
+ error_report_config = config.error_report
+ errors_df, aggregates = self._get_error_dataframes(
+ submission_info.submission_id,
+ error_report_config.record_rejection_reporting,
+ )
if not submission_status.number_of_records:
sub_stats = None
@@ -993,7 +1007,12 @@ def error_report(
summary_items = er.SummaryItems(
submission_status=submission_status,
summary_dict=summary_dict,
- row_headings=[e.reporting_name for e in ErrorReportCategories],
+ row_headings=[
+ e.reporting_name
+ for e in ErrorReportCategories
+ if error_report_config.record_rejection_reporting
+ or e != ErrorReportCategories.RECORD_REJECTION
+ ],
)
workbook = er.ExcelFormat(
diff --git a/tests/features/animals.feature b/tests/features/animals.feature
index d68ddbf1..1a2efc17 100644
--- a/tests/features/animals.feature
+++ b/tests/features/animals.feature
@@ -24,7 +24,7 @@ Feature: Pipeline tests using the animal dataset
And The statistics entry for the submission shows the following information
| parameter | value |
| record_count | 5 |
- | number_record_rejections | 2 |
+ | number_record_rejections | 0 |
| number_warnings | 0 |
Scenario: Validate XML data with a mixture of error types in (duckdb)
@@ -55,5 +55,5 @@ Feature: Pipeline tests using the animal dataset
| parameter | value |
| record_count | 7 |
| number_submission_rejections | 1 |
- | number_record_rejections | 2 |
+ | number_record_rejections | 0 |
| number_warnings | 1 |
diff --git a/tests/test_pipeline/test_duckdb_pipeline.py b/tests/test_pipeline/test_duckdb_pipeline.py
index 4e41af0e..f3dcf462 100644
--- a/tests/test_pipeline/test_duckdb_pipeline.py
+++ b/tests/test_pipeline/test_duckdb_pipeline.py
@@ -189,7 +189,7 @@ def test_error_report_step(
audit_tables=audit_manager,
job_run_id=1,
connection=conn,
- rules_path=None,
+ rules_path=PLANETS_RULES_PATH,
submitted_files_path=None,
)
diff --git a/tests/test_pipeline/test_spark_pipeline.py b/tests/test_pipeline/test_spark_pipeline.py
index cbb95b60..7ddb524e 100644
--- a/tests/test_pipeline/test_spark_pipeline.py
+++ b/tests/test_pipeline/test_spark_pipeline.py
@@ -516,7 +516,7 @@ def test_error_report_step(
processed_files_path=processed_files_path,
audit_tables=audit_manager,
job_run_id=1,
- rules_path=None,
+ rules_path=PLANETS_RULES_PATH,
submitted_files_path=None,
spark=spark,
)
diff --git a/tests/testdata/animals/animals.dischema.json b/tests/testdata/animals/animals.dischema.json
index 0e1eda14..7180d093 100644
--- a/tests/testdata/animals/animals.dischema.json
+++ b/tests/testdata/animals/animals.dischema.json
@@ -50,5 +50,8 @@
"is_informational": true
}
]
+ },
+ "error_report": {
+ "record_rejection_reporting": false
}
}
\ No newline at end of file
From 2dc7a48e033166d3ac03760558c2d76847fec524 Mon Sep 17 00:00:00 2001
From: George Robertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Mon, 26 Oct 2026 17:18:58 +0000
Subject: [PATCH 40/47] build: update various packages for security patching
(#193)
---
.mise.toml | 4 +-
.tool-versions | 4 +-
poetry.lock | 156 +++++++++++++++++++------------------------------
poetry.toml | 1 +
pyproject.toml | 15 ++---
5 files changed, 73 insertions(+), 107 deletions(-)
diff --git a/.mise.toml b/.mise.toml
index 0213a692..b0205030 100644
--- a/.mise.toml
+++ b/.mise.toml
@@ -1,4 +1,4 @@
[tools]
-python="3.12"
+python="3.12.16"
poetry="2.4.1"
-java="zulu-17.60.17"
+java="zulu-17.70.111"
diff --git a/.tool-versions b/.tool-versions
index eb68911b..e32b0537 100644
--- a/.tool-versions
+++ b/.tool-versions
@@ -1,3 +1,3 @@
-python 3.12.12
+python 3.12.16
poetry 2.4.1
-java zulu-17.60.17
+java zulu-17.70.111
diff --git a/poetry.lock b/poetry.lock
index e647ec18..6614bbf2 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -39,9 +39,6 @@ files = [
{file = "astroid-3.3.9.tar.gz", hash = "sha256:622cc8e3048684aa42c820d9d218978021c3c3d174fb03a9f0d615921744f550"},
]
-[package.dependencies]
-typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""}
-
[[package]]
name = "behave"
version = "1.3.3"
@@ -61,7 +58,6 @@ cucumber-tag-expressions = ">=4.1.0"
parse = ">=1.18.0"
parse-type = ">=0.6.0"
six = ">=1.15.0"
-tomli = {version = ">=1.1.0", markers = "python_version >= \"3.0\" and python_version < \"3.11\""}
[package.extras]
develop = ["PyHamcrest (<2.0) ; python_version < \"3.0\"", "PyHamcrest (>=2.0.2) ; python_version >= \"3.0\"", "build (>=0.5.1)", "coverage (>=5.0)", "invoke (>=1.7.0) ; python_version >= \"3.6\"", "invoke (>=1.7.0,<2.0) ; python_version < \"3.6\"", "mock (<4.0) ; python_version < \"3.6\"", "mock (>=4.0) ; python_version >= \"3.6\"", "modernize (>=0.5)", "path (>=13.1.0) ; python_version >= \"3.5\"", "path.py (>=11.5.0) ; python_version < \"3.5\"", "pycmd", "pylint", "pytest (>=4.2,<5.0) ; python_version < \"3.0\"", "pytest (>=5.0) ; python_version >= \"3.0\"", "pytest-cov", "pytest-html (>=1.19.0,<2.0) ; python_version < \"3.0\"", "pytest-html (>=2.0) ; python_version >= \"3.0\"", "ruff ; python_version >= \"3.7\"", "tox (>=3.28.0,<4.0)", "twine (>=1.13.0)", "virtualenv (<20.22.0) ; python_version < \"3.7\"", "virtualenv (>=20.26.6) ; python_version >= \"3.7\""]
@@ -71,39 +67,39 @@ testing = ["PyHamcrest (<2.0) ; python_version < \"3.0\"", "PyHamcrest (>=2.0.2)
[[package]]
name = "black"
-version = "26.3.1"
+version = "26.10.1"
description = "The uncompromising code formatter."
optional = false
python-versions = ">=3.10"
groups = ["dev", "lint"]
files = [
- {file = "black-26.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:86a8b5035fce64f5dcd1b794cf8ec4d31fe458cf6ce3986a30deb434df82a1d2"},
- {file = "black-26.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5602bdb96d52d2d0672f24f6ffe5218795736dd34807fd0fd55ccd6bf206168b"},
- {file = "black-26.3.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c54a4a82e291a1fee5137371ab488866b7c86a3305af4026bdd4dc78642e1ac"},
- {file = "black-26.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:6e131579c243c98f35bce64a7e08e87fb2d610544754675d4a0e73a070a5aa3a"},
- {file = "black-26.3.1-cp310-cp310-win_arm64.whl", hash = "sha256:5ed0ca58586c8d9a487352a96b15272b7fa55d139fc8496b519e78023a8dab0a"},
- {file = "black-26.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:28ef38aee69e4b12fda8dba75e21f9b4f979b490c8ac0baa7cb505369ac9e1ff"},
- {file = "black-26.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bf9bf162ed91a26f1adba8efda0b573bc6924ec1408a52cc6f82cb73ec2b142c"},
- {file = "black-26.3.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:474c27574d6d7037c1bc875a81d9be0a9a4f9ee95e62800dab3cfaadbf75acd5"},
- {file = "black-26.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:5e9d0d86df21f2e1677cc4bd090cd0e446278bcbbe49bf3659c308c3e402843e"},
- {file = "black-26.3.1-cp311-cp311-win_arm64.whl", hash = "sha256:9a5e9f45e5d5e1c5b5c29b3bd4265dcc90e8b92cf4534520896ed77f791f4da5"},
- {file = "black-26.3.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5e6f89631eb88a7302d416594a32faeee9fb8fb848290da9d0a5f2903519fc1"},
- {file = "black-26.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:41cd2012d35b47d589cb8a16faf8a32ef7a336f56356babd9fcf70939ad1897f"},
- {file = "black-26.3.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f76ff19ec5297dd8e66eb64deda23631e642c9393ab592826fd4bdc97a4bce7"},
- {file = "black-26.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:ddb113db38838eb9f043623ba274cfaf7d51d5b0c22ecb30afe58b1bb8322983"},
- {file = "black-26.3.1-cp312-cp312-win_arm64.whl", hash = "sha256:dfdd51fc3e64ea4f35873d1b3fb25326773d55d2329ff8449139ebaad7357efb"},
- {file = "black-26.3.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:855822d90f884905362f602880ed8b5df1b7e3ee7d0db2502d4388a954cc8c54"},
- {file = "black-26.3.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8a33d657f3276328ce00e4d37fe70361e1ec7614da5d7b6e78de5426cb56332f"},
- {file = "black-26.3.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f1cd08e99d2f9317292a311dfe578fd2a24b15dbce97792f9c4d752275c1fa56"},
- {file = "black-26.3.1-cp313-cp313-win_amd64.whl", hash = "sha256:c7e72339f841b5a237ff14f7d3880ddd0fc7f98a1199e8c4327f9a4f478c1839"},
- {file = "black-26.3.1-cp313-cp313-win_arm64.whl", hash = "sha256:afc622538b430aa4c8c853f7f63bc582b3b8030fd8c80b70fb5fa5b834e575c2"},
- {file = "black-26.3.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2d6bfaf7fd0993b420bed691f20f9492d53ce9a2bcccea4b797d34e947318a78"},
- {file = "black-26.3.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:f89f2ab047c76a9c03f78d0d66ca519e389519902fa27e7a91117ef7611c0568"},
- {file = "black-26.3.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b07fc0dab849d24a80a29cfab8d8a19187d1c4685d8a5e6385a5ce323c1f015f"},
- {file = "black-26.3.1-cp314-cp314-win_amd64.whl", hash = "sha256:0126ae5b7c09957da2bdbd91a9ba1207453feada9e9fe51992848658c6c8e01c"},
- {file = "black-26.3.1-cp314-cp314-win_arm64.whl", hash = "sha256:92c0ec1f2cc149551a2b7b47efc32c866406b6891b0ee4625e95967c8f4acfb1"},
- {file = "black-26.3.1-py3-none-any.whl", hash = "sha256:2bd5aa94fc267d38bb21a70d7410a89f1a1d318841855f698746f8e7f51acd1b"},
- {file = "black-26.3.1.tar.gz", hash = "sha256:2c50f5063a9641c7eed7795014ba37b0f5fa227f3d408b968936e24bc0566b07"},
+ {file = "black-26.10.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51d5e417e700fe6ec0b0ecdc408c6f6cb5def80328f31f724993d82c6486b746"},
+ {file = "black-26.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:70ccbd175b7f6be29d2b727ee7ca6b4c54053df59da653a6df80b175d20a94fa"},
+ {file = "black-26.10.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5bd3518d8e97138fef295230b1e9804076d69fa4e3594071494a8c68abe6266"},
+ {file = "black-26.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:0ce08b367307b0fd91c9dd1d4084e62b05b3055475f951f0f34a46b6e2393b64"},
+ {file = "black-26.10.1-cp310-cp310-win_arm64.whl", hash = "sha256:19fa8f5beb5e77c54c9c7e21d00cc93ed6c8b6228ee385616906d6befe081143"},
+ {file = "black-26.10.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a0219b29cd70e49f920acb7081e6ce5025c719008447c521d0200dcad93206a"},
+ {file = "black-26.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7bdade400bfe24d78a7762896acc2f9a8e1a17fb0fd0536bf6b7c7097cf3eec7"},
+ {file = "black-26.10.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ff57f63029aa1353fa8b1b0c8971fd88a6c92dc766608d2eee33ad2deb23270e"},
+ {file = "black-26.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:3414a0c52901964dceabd98c7c56beac0f964115a116ecedcce7247359b14017"},
+ {file = "black-26.10.1-cp311-cp311-win_arm64.whl", hash = "sha256:1935b32f5326028019856e18cb42b4da63db23765dc84464cec723e0de478a9b"},
+ {file = "black-26.10.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fe85fc4019bee59bc495c0f2a8ee76c5cd02c7015508d94a967ba2376f39a52c"},
+ {file = "black-26.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:182f6c32be38074b16d378498c498b32cb51928178ee611485344972c35ec9c6"},
+ {file = "black-26.10.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b5347d760f0c02bb00dd249384cab71c3bf828b4f68d5b401eb116e0390f147d"},
+ {file = "black-26.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:4d9a90516db1d99c25dbb20cc0998e0e01531dd903466c7744e56d66f864220a"},
+ {file = "black-26.10.1-cp312-cp312-win_arm64.whl", hash = "sha256:2ffbc023a12d0c729408823b8f10514490bd0baa301d0d4e21a7240249f9507f"},
+ {file = "black-26.10.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b6272cfd7e1e8e271f5b0e0207259fe2834687e5cb9b5f620b34a44db9754993"},
+ {file = "black-26.10.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:978113a40223a6aaefc17364176a809a320e6b288683841427fff04c6d7b4130"},
+ {file = "black-26.10.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03c0ddd93bb392e71209903a691767eb366fe1a76deb9509ccbaae9e1f14bb52"},
+ {file = "black-26.10.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6dba8138cdc99061ef07b958ac082d2aa057b6961d1936f9717c350f02bab5f"},
+ {file = "black-26.10.1-cp313-cp313-win_arm64.whl", hash = "sha256:d42dd2fac7c342ae67e64ee99c9532e20b2a84e92c79ed3317fa2ef54c801d93"},
+ {file = "black-26.10.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:8375962579d537364cc0efa19b1474481915d3a793f9fc0774901814c5e5b5f4"},
+ {file = "black-26.10.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d8b3a9074a680b3c5749633714e9ae3992a1e5a23343a97ad61cd9b119b444d2"},
+ {file = "black-26.10.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:289282aa2e09d3162312a3be1788ff21b08e9ea9cc4a81e656024728b32428fb"},
+ {file = "black-26.10.1-cp314-cp314-win_amd64.whl", hash = "sha256:5cd88fd7b444ca51f3fc883b6f6657ea53a258b0b2eef6d9f2dfcfa17ce0e27b"},
+ {file = "black-26.10.1-cp314-cp314-win_arm64.whl", hash = "sha256:2520037aa62f8a1454d0811b8f5c88b444445b03a4bfba480d8d220893b64c34"},
+ {file = "black-26.10.1-py3-none-any.whl", hash = "sha256:28842f9a8207cc1df6eb983a35a14c5a0dfcd603d214fe82d84bef552afd2e3a"},
+ {file = "black-26.10.1.tar.gz", hash = "sha256:5f9f83beae62437e060dafd53d7f1fc327e3d3494f74d72ee5c2b73eb90fc4e7"},
]
[package.dependencies]
@@ -113,8 +109,6 @@ packaging = ">=22.0"
pathspec = ">=1.0.0"
platformdirs = ">=2"
pytokens = ">=0.4.0,<0.5.0"
-tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
-typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""}
[package.extras]
colorama = ["colorama (>=0.4.3)"]
@@ -547,14 +541,14 @@ types-awscrt = "*"
[[package]]
name = "certifi"
-version = "2026.6.17"
+version = "2026.10.23"
description = "Python package for providing Mozilla's CA Bundle."
optional = false
python-versions = ">=3.7"
groups = ["dev", "test"]
files = [
- {file = "certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db"},
- {file = "certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432"},
+ {file = "certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1"},
+ {file = "certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd"},
]
[[package]]
@@ -859,7 +853,6 @@ pyyaml = ">=3.8"
questionary = ">=2.0,<3.0"
termcolor = ">=1.1.0,<4.0.0"
tomlkit = ">=0.5.3,<1.0.0"
-typing-extensions = {version = ">=4.0.1,<5.0.0", markers = "python_version < \"3.11\""}
[[package]]
name = "coverage"
@@ -1024,7 +1017,6 @@ files = [
[package.dependencies]
cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\""}
-typing-extensions = {version = ">=4.13.2", markers = "python_full_version < \"3.11.0\""}
[package.extras]
ssh = ["bcrypt (>=3.1.5)"]
@@ -1208,35 +1200,16 @@ files = [
{file = "et_xmlfile-2.0.0.tar.gz", hash = "sha256:dab3f4764309081ce75662649be815c4c9081e88f0837825f90fd28317d4da54"},
]
-[[package]]
-name = "exceptiongroup"
-version = "1.3.1"
-description = "Backport of PEP 654 (exception groups)"
-optional = false
-python-versions = ">=3.7"
-groups = ["dev", "test"]
-markers = "python_version == \"3.10\""
-files = [
- {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"},
- {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"},
-]
-
-[package.dependencies]
-typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""}
-
-[package.extras]
-test = ["pytest (>=6)"]
-
[[package]]
name = "filelock"
-version = "3.29.5"
+version = "4.2.1"
description = "A platform independent file lock."
optional = false
-python-versions = ">=3.10"
+python-versions = ">=3.11"
groups = ["dev"]
files = [
- {file = "filelock-3.29.5-py3-none-any.whl", hash = "sha256:8af830889ba3a0ffcefbd6c7d2af8a54012058103771f2e10848222f476a1693"},
- {file = "filelock-3.29.5.tar.gz", hash = "sha256:6e6034c57a00a020e767f2614a5539863f056de7e7991d6d1473aef7ff73f156"},
+ {file = "filelock-4.2.1-py3-none-any.whl", hash = "sha256:7be2662e9e826b9bba28afc0f843a123280d799a2ea44bb5d35ccec67905ae44"},
+ {file = "filelock-4.2.1.tar.gz", hash = "sha256:c4c937ccac2a813cee448e662a56366cfb374c0f8702e75f295c412cc1c5d5a3"},
]
[[package]]
@@ -1879,7 +1852,6 @@ files = [
griffelib = ">=2.0"
mkdocs-autorefs = ">=1.4"
mkdocstrings = ">=0.30"
-typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""}
[[package]]
name = "moto"
@@ -1986,7 +1958,6 @@ files = [
librt = {version = ">=0.8.0", markers = "platform_python_implementation != \"PyPy\""}
mypy_extensions = ">=1.0.0"
pathspec = ">=1.0.0"
-tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
typing_extensions = {version = ">=4.6.0", markers = "python_version < \"3.15\""}
[package.extras]
@@ -2168,14 +2139,14 @@ et-xmlfile = "*"
[[package]]
name = "packaging"
-version = "26.2"
+version = "26.3"
description = "Core utilities for Python packages"
optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.9"
groups = ["dev", "docs", "lint", "test"]
files = [
- {file = "packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e"},
- {file = "packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661"},
+ {file = "packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c"},
+ {file = "packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79"},
]
[[package]]
@@ -2245,7 +2216,6 @@ files = [
[package.dependencies]
numpy = [
- {version = ">=1.22.4", markers = "python_version < \"3.11\""},
{version = ">=1.23.2", markers = "python_version == \"3.11\""},
{version = ">=1.26.0", markers = "python_version >= \"3.12\""},
]
@@ -2728,14 +2698,12 @@ files = [
astroid = ">=3.3.8,<=3.4.0.dev0"
colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""}
dill = [
- {version = ">=0.2", markers = "python_version < \"3.11\""},
{version = ">=0.3.6", markers = "python_version == \"3.11\""},
{version = ">=0.3.7", markers = "python_version >= \"3.12\""},
]
isort = ">=4.2.5,<5.13 || >5.13,<7"
mccabe = ">=0.6,<0.8"
platformdirs = ">=2.2"
-tomli = {version = ">=1.1", markers = "python_version < \"3.11\""}
tomlkit = ">=0.10.1"
[package.extras]
@@ -2796,12 +2764,10 @@ files = [
[package.dependencies]
colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""}
-exceptiongroup = {version = ">=1", markers = "python_version < \"3.11\""}
iniconfig = ">=1.0.1"
packaging = ">=22"
pluggy = ">=1.5,<2"
pygments = ">=2.7.2"
-tomli = {version = ">=1", markers = "python_version < \"3.11\""}
[package.extras]
dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"]
@@ -2838,14 +2804,14 @@ six = ">=1.5"
[[package]]
name = "python-discovery"
-version = "1.6.0"
+version = "1.6.1"
description = "Python interpreter discovery"
optional = false
python-versions = ">=3.8"
groups = ["dev"]
files = [
- {file = "python_discovery-1.6.0-py3-none-any.whl", hash = "sha256:d4e244cf17b8b29819ed78003d55fbacf86eda23425b075454fff9271b79377a"},
- {file = "python_discovery-1.6.0.tar.gz", hash = "sha256:6393b4eae1be8b2182670635e7baff89ac21cb9f8e86fd1ff40c7b1144febb4c"},
+ {file = "python_discovery-1.6.1-py3-none-any.whl", hash = "sha256:d43fcdef879fe795352bd13ccf8d185ba5a9f86f36cfcd00529f596e737442b3"},
+ {file = "python_discovery-1.6.1.tar.gz", hash = "sha256:cf87d3627dfb4412437fdd5b13eae402607722998d21567993aedbc59b23c15e"},
]
[package.dependencies]
@@ -3125,7 +3091,7 @@ version = "2.4.1"
description = "A lil' TOML parser"
optional = false
python-versions = ">=3.8"
-groups = ["dev", "docs", "lint", "test"]
+groups = ["docs"]
files = [
{file = "tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30"},
{file = "tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a"},
@@ -3175,7 +3141,6 @@ files = [
{file = "tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe"},
{file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"},
]
-markers = {dev = "python_version == \"3.10\"", lint = "python_version == \"3.10\"", test = "python_version == \"3.10\""}
[[package]]
name = "tomlkit"
@@ -3302,16 +3267,15 @@ files = [
[[package]]
name = "typing-extensions"
-version = "4.15.0"
+version = "4.16.0"
description = "Backported and Experimental Type Hints for Python 3.9+"
optional = false
python-versions = ">=3.9"
-groups = ["main", "dev", "docs", "lint", "test"]
+groups = ["main", "dev", "lint"]
files = [
- {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"},
- {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"},
+ {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"},
+ {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"},
]
-markers = {docs = "python_version == \"3.10\"", test = "python_version == \"3.10\""}
[[package]]
name = "typing-inspection"
@@ -3360,22 +3324,22 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""]
[[package]]
name = "virtualenv"
-version = "21.7.13"
+version = "21.14.9"
description = "Virtual Python Environment builder"
optional = false
python-versions = ">=3.9"
groups = ["dev"]
files = [
- {file = "virtualenv-21.7.13-py3-none-any.whl", hash = "sha256:1bea5af7463f59c4719db48fe739579a2a4f569c96f26c086edda85c96da9f59"},
- {file = "virtualenv-21.7.13.tar.gz", hash = "sha256:0355558b6f33619aab31347e43643b0ebc97f61ea3acf617b2b69e1f8a843d11"},
+ {file = "virtualenv-21.14.9-py3-none-any.whl", hash = "sha256:ac51744da66ae56ff814b087d65096c94de698ec520adc624b108b679655809b"},
+ {file = "virtualenv-21.14.9.tar.gz", hash = "sha256:4d1bee354cd81c820da3bee56e38be3426490f8d1fc54e10ea5b2072ebad8ef8"},
]
[package.dependencies]
-distlib = ">=0.3.7,<1"
-filelock = {version = ">=3.24.2,<4", markers = "python_version >= \"3.10\""}
-platformdirs = ">=3.9.1,<5"
-python-discovery = ">=1.6"
-typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\""}
+distlib = ">=0.4.3,<1"
+filelock = {version = ">=3.24.2,<5", markers = "python_version >= \"3.10\""}
+packaging = ">=26.3"
+platformdirs = ">=4.4,<5"
+python-discovery = ">=1.6.1"
[[package]]
name = "watchdog"
@@ -3434,14 +3398,14 @@ files = [
[[package]]
name = "werkzeug"
-version = "3.1.6"
+version = "3.1.9"
description = "The comprehensive WSGI web application library."
optional = false
python-versions = ">=3.9"
groups = ["dev", "test"]
files = [
- {file = "werkzeug-3.1.6-py3-none-any.whl", hash = "sha256:7ddf3357bb9564e407607f988f683d72038551200c704012bb9a4c523d42f131"},
- {file = "werkzeug-3.1.6.tar.gz", hash = "sha256:210c6bede5a420a913956b4791a7f4d6843a43b6fcee4dfa08a65e93007d0d25"},
+ {file = "werkzeug-3.1.9-py3-none-any.whl", hash = "sha256:6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab"},
+ {file = "werkzeug-3.1.9.tar.gz", hash = "sha256:55ca7c70a75689be937aa27f8ff4b018f06ff4838fc73045560bf0f5a1291060"},
]
[package.dependencies]
@@ -3632,5 +3596,5 @@ type = ["pytest-mypy (>=1.0.1) ; platform_python_implementation != \"PyPy\""]
[metadata]
lock-version = "2.1"
-python-versions = ">=3.10,<3.13"
-content-hash = "2090fe698ede57b0910f35a1701b8fe66bbcd69ffa72412597afcab95f4876b8"
+python-versions = ">=3.11,<3.13"
+content-hash = "889e54ad662e9460ae037f9a541c4f4f3993ee7062e81f58d312622fd615b4a7"
diff --git a/poetry.toml b/poetry.toml
index 93385b5d..8a6f547a 100644
--- a/poetry.toml
+++ b/poetry.toml
@@ -7,6 +7,7 @@ no-binary = ["pyspark"]
[solver]
min-release-age = 10
+min-release-age-exclude = ["certifi", "filelock", "virtualenv"]
[virtualenvs]
in-project = true
diff --git a/pyproject.toml b/pyproject.toml
index 6970281f..8e129cbd 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -9,7 +9,6 @@ readme = "README.md"
classifiers = [
"Development Status :: 4 - Beta",
"Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Operating System :: POSIX :: Linux",
@@ -32,7 +31,7 @@ packages = [
]
[tool.poetry.dependencies]
-python = ">=3.10,<3.13" # breaking changes beyond 3.12
+python = ">=3.11,<3.13" # breaking changes beyond 3.12
boto3 = ">=1.34.162,<1.36" # breaking change beyond 1.36
botocore = ">=1.34.162,<1.36" # breaking change beyond 1.36
delta-spark = ">=3.0.0,<=3.2.0"
@@ -46,7 +45,7 @@ polars = "0.20.31"
pyarrow = "23.0.1"
pydantic = "2.13.4"
pyspark = ">=3.5.0,<=3.5.5"
-typing_extensions = "4.15.0"
+typing_extensions = "4.16.0"
urllib3 = "2.8.0" # dependency of boto3 & botocore
[tool.poetry.group.dev]
@@ -60,9 +59,11 @@ include-groups = [
commitizen = "4.9.1"
pre-commit = "4.3.0"
charset-normalizer = "3.4.6"
-python-discovery = "1.6.0"
+python-discovery = "1.6.1"
idna = "3.15" # dependency of requests
-virtualenv = "21.7.13"
+virtualenv = "21.14.9"
+certifi = "2026.10.23"
+filelock = "4.2.1"
[tool.poetry.group.test]
optional = true
@@ -73,7 +74,7 @@ coverage = "7.11.0"
moto = {extras = ["s3"], version = "4.2.14"}
cryptography = "50.0.0" # dependency of `moto`
requests = "2.33.0" # dependency of `moto`
-Werkzeug = "3.1.6"
+Werkzeug = "3.1.9"
pytest = "9.0.3"
pytest-lazy-fixtures = "1.4.0" # switched from https://github.com/TvoroG/pytest-lazy-fixture as it's no longer supported
xlsx2csv = "0.8.4" # polars requirement
@@ -82,7 +83,7 @@ xlsx2csv = "0.8.4" # polars requirement
optional = true
[tool.poetry.group.lint.dependencies]
-black = "26.3.1"
+black = "26.10.1"
astroid = "3.3.9"
isort = "5.13.2"
pylint = "3.3.9"
From 56be7c8ae2997967fe5610d093e147926c889d6b Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Mon, 26 Oct 2026 17:40:13 +0000
Subject: [PATCH 41/47] fix: total no records rejected showing when it
shouldn't
---
src/dve/pipeline/pipeline.py | 1 +
src/dve/reporting/excel_report.py | 19 ++++++++++++-------
2 files changed, 13 insertions(+), 7 deletions(-)
diff --git a/src/dve/pipeline/pipeline.py b/src/dve/pipeline/pipeline.py
index 8d9cfdb6..4eba0492 100644
--- a/src/dve/pipeline/pipeline.py
+++ b/src/dve/pipeline/pipeline.py
@@ -1013,6 +1013,7 @@ def error_report(
if error_report_config.record_rejection_reporting
or e != ErrorReportCategories.RECORD_REJECTION
],
+ error_report_config=error_report_config,
)
workbook = er.ExcelFormat(
diff --git a/src/dve/reporting/excel_report.py b/src/dve/reporting/excel_report.py
index e0c48917..71eba728 100644
--- a/src/dve/reporting/excel_report.py
+++ b/src/dve/reporting/excel_report.py
@@ -16,6 +16,7 @@
from polars import DataFrame
from polars.exceptions import ColumnNotFoundError
+from dve.core_engine.configuration.v1 import V1ErrorReportConfig
from dve.pipeline.utils import SubmissionStatus
from dve.reporting.constants import ErrorReportCategories, ErrorReportStatus
@@ -39,6 +40,8 @@ class SummaryItems:
"""List of aggregations to apply to the grouped up dataframe"""
additional_columns: Optional[list] = None
"""any additional columns to add to the summary table"""
+ error_report_config: V1ErrorReportConfig = field(default_factory=V1ErrorReportConfig)
+ """user specificed config to change elements of the error report"""
def create_summary_sheet(
self,
@@ -160,13 +163,15 @@ def _add_submission_info(self, status: str, summary: Worksheet):
_records_rejected = self.submission_status.number_of_records
else:
_records_rejected = self.submission_status.number_of_records_rejected
- summary.append(
- [
- "",
- "Total Number of Records Rejected",
- _records_rejected,
- ]
- )
+
+ if self.error_report_config.record_rejection_reporting:
+ summary.append(
+ [
+ "",
+ "Total Number of Records Rejected",
+ _records_rejected,
+ ]
+ )
summary.append(["", ""])
From 910fc149ab606932816e165ab703a3e64aacb29f Mon Sep 17 00:00:00 2001
From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Mon, 26 Oct 2026 17:53:09 +0000
Subject: [PATCH 42/47] docs: add error_reports to json schemas
---
.../json_schemas/dataset.schema.json | 3 +++
.../json_schemas/error_report.schema.json | 15 +++++++++++++++
2 files changed, 18 insertions(+)
create mode 100644 docs/advanced_guidance/json_schemas/error_report.schema.json
diff --git a/docs/advanced_guidance/json_schemas/dataset.schema.json b/docs/advanced_guidance/json_schemas/dataset.schema.json
index af8b620f..909cbb12 100644
--- a/docs/advanced_guidance/json_schemas/dataset.schema.json
+++ b/docs/advanced_guidance/json_schemas/dataset.schema.json
@@ -13,6 +13,9 @@
},
"entity_relationships": {
"$ref": "entity_relationships.schema.json"
+ },
+ "error_report": {
+ "$ref": "error_report.schema.json"
}
},
"required": [
diff --git a/docs/advanced_guidance/json_schemas/error_report.schema.json b/docs/advanced_guidance/json_schemas/error_report.schema.json
new file mode 100644
index 00000000..546d4365
--- /dev/null
+++ b/docs/advanced_guidance/json_schemas/error_report.schema.json
@@ -0,0 +1,15 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "data-ingest:error_report.schema.json",
+ "title": "error_report",
+ "description": "Configuration for generating the error report.",
+ "type": "object",
+ "properties": {
+ "record_rejection_reporting": {
+ "description": "Whether to show/remove record rejection from the error report generated.",
+ "type": "boolean",
+ "default": true
+ }
+ },
+ "additionalProperties": false
+}
From 4652abe602e98df4e746aed1307069e871a0d761 Mon Sep 17 00:00:00 2001
From: stevenhsd <56357022+stevenhsd@users.noreply.github.com>
Date: Wed, 28 Oct 2026 13:27:39 +0000
Subject: [PATCH 43/47] fix: ensure error summary sheet populates entities in
deterministic way (#191)
---
src/dve/pipeline/foundry_ddb_pipeline.py | 7 ++-----
src/dve/pipeline/pipeline.py | 11 +++++------
2 files changed, 7 insertions(+), 11 deletions(-)
diff --git a/src/dve/pipeline/foundry_ddb_pipeline.py b/src/dve/pipeline/foundry_ddb_pipeline.py
index 56d202a7..a5c24638 100644
--- a/src/dve/pipeline/foundry_ddb_pipeline.py
+++ b/src/dve/pipeline/foundry_ddb_pipeline.py
@@ -10,7 +10,6 @@
duckdb_get_entity_count,
duckdb_write_parquet,
)
-from dve.core_engine.configuration.v1 import V1ErrorReportConfig
from dve.core_engine.exceptions import CriticalProcessingError
from dve.core_engine.models import SubmissionInfo
from dve.core_engine.type_hints import URI
@@ -102,12 +101,10 @@ def apply_business_rules(
return submission_info, SubmissionStatus(processing_failed=True)
def error_report(
- self, submission_info: SubmissionInfo,
- submission_status: Optional[SubmissionStatus] = None ,
- error_report_config: Optional[V1ErrorReportConfig] = None,
+ self, submission_info: SubmissionInfo, submission_status: Optional[SubmissionStatus] = None
):
try:
- return super().error_report(submission_info, submission_status,error_report_config)
+ return super().error_report(submission_info, submission_status)
except Exception as exc: # pylint: disable=W0718
self._logger.exception("Error reports raised exception:")
sub_stats = None
diff --git a/src/dve/pipeline/pipeline.py b/src/dve/pipeline/pipeline.py
index 4eba0492..53185a06 100644
--- a/src/dve/pipeline/pipeline.py
+++ b/src/dve/pipeline/pipeline.py
@@ -946,7 +946,6 @@ def error_report(
self,
submission_info: SubmissionInfo,
submission_status: Optional[SubmissionStatus] = None,
- error_report_config: Optional[V1ErrorReportConfig] = None,
) -> tuple[
SubmissionInfo, SubmissionStatus, Optional[SubmissionStatisticsRecord], Optional[URI]
]:
@@ -968,11 +967,10 @@ def error_report(
raise AttributeError("processed files path not provided")
self._logger.info("Reading error dataframes")
- if error_report_config is None:
- if not self.rules_path:
- raise AttributeError("rules path not provided")
- _, config, _ = load_config(submission_info.dataset_id, self.rules_path)
- error_report_config = config.error_report
+ if not self.rules_path:
+ raise AttributeError("rules path not provided")
+ _, config, _ = load_config(submission_info.dataset_id, self.rules_path)
+ error_report_config: V1ErrorReportConfig = config.error_report
errors_df, aggregates = self._get_error_dataframes(
submission_info.submission_id,
error_report_config.record_rejection_reporting,
@@ -1007,6 +1005,7 @@ def error_report(
summary_items = er.SummaryItems(
submission_status=submission_status,
summary_dict=summary_dict,
+ table_columns=[entity.upper() for entity in config.contract.datasets],
row_headings=[
e.reporting_name
for e in ErrorReportCategories
From 3a7994ac4b76e60005e9517b33652e306f643ef6 Mon Sep 17 00:00:00 2001
From: George Robertson <50412379+georgeRobertson@users.noreply.github.com>
Date: Thu, 29 Oct 2026 13:56:21 +0000
Subject: [PATCH 44/47] docs: update docs related to v0.10.0 release (#195)
* docs: update docs related to v0.10.0 release
* docs: fix typo in upperbound python version for v0.10.0
---
.../entity_relationships.schema.json | 4 +--
docs/user_guidance/custom_error_report.md | 28 ++++++++++++++++
docs/user_guidance/entity_relationships.md | 33 +++++++++++++++++--
docs/user_guidance/file_transformation.md | 32 ++++++++++++++++++
docs/user_guidance/install.md | 18 +++++-----
includes/jargon_and_acronyms.md | 2 ++
zensical.toml | 1 +
7 files changed, 105 insertions(+), 13 deletions(-)
create mode 100644 docs/user_guidance/custom_error_report.md
diff --git a/docs/advanced_guidance/json_schemas/entity_relationships.schema.json b/docs/advanced_guidance/json_schemas/entity_relationships.schema.json
index 0c2abbb2..6a641173 100644
--- a/docs/advanced_guidance/json_schemas/entity_relationships.schema.json
+++ b/docs/advanced_guidance/json_schemas/entity_relationships.schema.json
@@ -35,11 +35,11 @@
"no_valid_records_error_message": {
"type": "string"
},
- "empty_entity_error_code": {
+ "empty_entity_post_br_error_code": {
"type": "string",
"minLength": 1
},
- "empty_entity_error_message": {
+ "empty_entity_post_br_error_message": {
"type": "string",
"minLength": 1
}
diff --git a/docs/user_guidance/custom_error_report.md b/docs/user_guidance/custom_error_report.md
new file mode 100644
index 00000000..ebbac1fa
--- /dev/null
+++ b/docs/user_guidance/custom_error_report.md
@@ -0,0 +1,28 @@
+---
+title: Customising your Error Report
+tags:
+ - Feedback
+ - Error Reports
+---
+
+!!! note
+
+ This feature is WIP and functionality will be expanded over time.
+
+This feature will allow you to customise elements of the error report generated. To use this functionality simply add the following to your dischema document:
+
+```json
+{
+ "contract": {},
+ "transformations": {},
+ "error_report": {
+ "record_rejection_reporting": false
+ }
+}
+```
+
+## Current features available
+
+| Feature | Purpose | Default |
+| ------- | ------- | ------- |
+| record_rejection_reporting | Whether to show record rejection figures in the Data Summary Tab. This will remove "Total Number of Record Rejections" from the headlines figures and it will remove the "Record Rejection" type from the errors raised per entity. | True |
diff --git a/docs/user_guidance/entity_relationships.md b/docs/user_guidance/entity_relationships.md
index 68fe76d7..9375bcdb 100644
--- a/docs/user_guidance/entity_relationships.md
+++ b/docs/user_guidance/entity_relationships.md
@@ -9,9 +9,13 @@ tags:
- Rejections
---
+!!! note
+
+ This feature is performed after all business rules have been applied. A `missing_parent` check will be performed, followed by a `no_valid_records` and then finally with a `missing_parent` check again (in case this results in new missing parents). Details on `missing_parent` and `no_valid_records` can be found within this page.
+
Sometimes a user may choose to use the file transformation stage to `normalise` a heavily nested dataset into separate entities during the initial reading of data. This would be done by specifying different entities in the dataset section of the contract configuration in the `dischema` file. This allows for easier interaction when customising errors in the data contract or writing transformations in the business rules.
-`Normalising` assets can lead to more complex validations being required. For example in the flights dataset:
+`Normalising` assets can lead to structural validations being required. For example in the flights dataset:
```mermaid
erDiagram
@@ -21,6 +25,19 @@ erDiagram
AIRPORT ||--|{ STAFF_MEMBER : ""
```
+We now have a number of entity relationships generated from the normalised model. In the case above, the ERD states...
+
+- A Country must have at least one valid airport
+- An Airport must have at least one valid country
+- An Airport must have at least one valid staff member
+- An Airport may have zero to many flights
+- A Staff Member must have at least one valid Airport
+- A Flights may have zero to many passengers
+- A Flights must have at least one valid Airport
+- A Passenger must have at least one valid Flight
+
+This creates a scenario where you would want to apply structual integrity checks to ensure that partial and/or invalid data is not passing validation. I.e. you wouldn't want an aiport flowing beyond validation if there is no valid country.
+
### Missing Parent Records
It could be that an airport record is deemed invalid and removed. Due to this, any flight records that linked to the now removed airport record are themselves invalid - a situation we refer to as a `missing_parent` issue, but are now existing in an entirely different entity.
@@ -31,7 +48,19 @@ It could also be the case that staff records are a mandatory field for airport r
### Dischema
-In order to perform these validations, how to link normalised entities needs to be provided. This can be specified in the `entity_relationships` section of the `dischema`.
+In order to perform these validations, you must specify an `entity_relationships` section within the `dischema`. For example:
+
+```json
+{
+ "contract": {...},
+ "transformations": {...},
+ "entity_relationships": {
+ "my_entity": {...}
+ }
+}
+```
+
+You can find complete examples [here](https://github.com/NHSDigital/data-validation-engine/tree/main/tests/testdata/flights/).
## Entity Relationships Content
diff --git a/docs/user_guidance/file_transformation.md b/docs/user_guidance/file_transformation.md
index 411b1e0f..ae34ace7 100644
--- a/docs/user_guidance/file_transformation.md
+++ b/docs/user_guidance/file_transformation.md
@@ -157,6 +157,38 @@ The secondary use of the File Transformation stage is the ability to normalise y
!!! abstract ""
You can read more about the readers and kwargs [here](../advanced_guidance/package_documentation/data_contract/readers.md).
+## Reader additional checks
+
+The file transformation section supports additional reader checks which will apply across all reader configs for a given entity. These are the checks you can currently perform:
+
+### Check Empty
+
+The `check empty` feature allows you check whether an entity is empty on submission. When this error is raised, the DVE will generate a feedback message and report. This won't process any further entities as you are effectively saying this entity cannot be empty on submission.
+
+Here is an example of this being implemented in a dischema document:
+
+```json
+{
+ "contract": {
+ "datasets": {
+ "my_entity": {
+ "fields": {...},
+ "reader_config": {
+ ".csv": {...},
+ ".xml": {...}
+ },
+ "reader_additional_checks": {
+ "check_empty": {
+ "error_code": "Empty entity",
+ "error_message": "'my_entity' is empty. Please resolve"
+ }
+ }
+ }
+ }
+ }
+}
+```
+
## Supported Formats
| Format | DuckDB | Spark | Version Available |
diff --git a/docs/user_guidance/install.md b/docs/user_guidance/install.md
index 2c86b122..982c96a0 100644
--- a/docs/user_guidance/install.md
+++ b/docs/user_guidance/install.md
@@ -78,12 +78,12 @@ Once you have installed the DVE you are almost ready to use it. To be able to ru
## DVE Version Compatability Matrix
-| DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version |
-| ------------ | -------------- | ---------------- | --------------- | ---------------- |
-| >=0.10.0 | >=3.10,<1.13 | __>=1.4,<1.4.5__ | >=3.5.0,<=3.5.5 | 2.13.4 |
-| >=0.9.0 | >=3.10,<3.13 | 1.1.3 | >=3.5.0,<=3.5.5 | __2.13.4__ |
-| >=0.8.0 | >=3.10,<3.13 | __1.1.3__ | __3.5.2__ | 1.10.19 |
-| >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | __1.10.16__ |
-| >=0.6 | >=3.10,<3.12 | __1.1.*__ | __3.4.*__ | 1.10.15 |
-| >=0.2,<0.6 | __>=3.10,<3.12__ | 1.1.0 | 3.4.4 | 1.10.15 |
-| 0.1 | >=3.7.2,<3.8 | 1.1.0 | 3.2.1 | 1.10.15 |
+| DVE Version | Python Version | DuckDB Version | Spark Version | Pydantic Version |
+| ------------ | -------------------- | ---------------- | --------------- | ---------------- |
+| >=0.10.0 | __>=3.11,<3.13__ | __>=1.4,<1.4.5__ | >=3.5.0,<=3.5.5 | 2.13.4 |
+| >=0.9.0 | >=3.10,<3.13 | 1.1.3 | >=3.5.0,<=3.5.5 | __2.13.4__ |
+| >=0.8.0 | >=3.10,<3.13 | __1.1.3__ | __3.5.2__ | 1.10.19 |
+| >=0.7.2 | >=3.10,<3.12 | 1.1.* | 3.4.* | __1.10.16__ |
+| >=0.6 | >=3.10,<3.12 | __1.1.*__ | __3.4.*__ | 1.10.15 |
+| >=0.2,<0.6 | __>=3.10,<3.12__ | 1.1.0 | 3.4.4 | 1.10.15 |
+| 0.1 | >=3.7.2,<3.8 | 1.1.0 | 3.2.1 | 1.10.15 |
diff --git a/includes/jargon_and_acronyms.md b/includes/jargon_and_acronyms.md
index 9b42c4e6..60c5e535 100644
--- a/includes/jargon_and_acronyms.md
+++ b/includes/jargon_and_acronyms.md
@@ -4,3 +4,5 @@
*[kwarg]: Key Word Arguments
*[constr]: Constrained String
*[immutable]: Unchanging over time or unable to be changed
+*[ERD]: Entity Relationship Diagram
+*[WIP]: Work in progress
diff --git a/zensical.toml b/zensical.toml
index fc47343e..4ae3b6c7 100644
--- a/zensical.toml
+++ b/zensical.toml
@@ -38,6 +38,7 @@ nav = [
{"Reporting" = [
{"Feedback Messages" = "user_guidance/feedback_messages.md"},
{"Error Reports" = "user_guidance/error_reports.md"},
+ {"Customising your Error Report" = "user_guidance/custom_error_report.md"},
]}
]},
{"Advanced User Guidance" = [
From 011cfddbd75cf2d6b201650a29fc88ccddd40f9d Mon Sep 17 00:00:00 2001
From: stevenhsd <56357022+stevenhsd@users.noreply.github.com>
Date: Fri, 30 Oct 2026 16:30:18 +0000
Subject: [PATCH 45/47] build: bump pypi publish action to latest release
(#196)
---
.github/workflows/ci_pypi_publish.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/ci_pypi_publish.yml b/.github/workflows/ci_pypi_publish.yml
index 11764ddf..e2eecc3d 100644
--- a/.github/workflows/ci_pypi_publish.yml
+++ b/.github/workflows/ci_pypi_publish.yml
@@ -54,4 +54,4 @@ jobs:
path: dist/
- name: publish
- uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
+ uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3
From 88d3d4e5744017994672d94568d0277c70ae3230 Mon Sep 17 00:00:00 2001
From: stevenhsd <56357022+stevenhsd@users.noreply.github.com>
Date: Fri, 30 Oct 2026 16:32:52 +0000
Subject: [PATCH 46/47] =?UTF-8?q?bump:=20version=200.9.0=20=E2=86=92=200.1?=
=?UTF-8?q?0.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++
pyproject.toml | 2 +-
2 files changed, 46 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bd1928b4..84b480a9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,48 @@
+## v0.10.0 (2026-10-30)
+
+### Feat
+
+- add error_report config to toggle record rejection reporting (#189)
+- add orphan and group rejections to spark backend (#174)
+- add facility to add additional checks at end of file transforma… (#165)
+- add empty_entity check post filters for mandatory entities (#163)
+- group rejections for mandatory primary keys (#152)
+- add orphan record identification and removal (#150)
+
+### Fix
+
+- ensure error summary sheet populates entities in deterministic way (#191)
+- total no records rejected showing when it shouldn't (#194)
+- total no records rejected showing when it shouldn't
+- adjust regex check for timezone offsets, allowing : to be included (#181)
+- refactor the join_header method in duckdb to deal with out of bound date values (#177)
+- group rejection not filtering out invalid records correctly (#175)
+- exit processing for files that fail file transformation (#171)
+- ensure that original entity detials persisted in feedback messages for further processing (#168)
+- ensure network and load_dtd are disabled in xml reading (#167)
+- raise processing error where filter is materialising null values (#162)
+- add entity into data contract details to deal with duplicate column definitions (#159)
+- add message templating to error messages on orph+group rej (#161)
+- ensure missing parent and group rejection records are being removed (#160)
+- revert change to SubStats population in error reports (#156)
+- change record rej ct to be eq to records processed when file rejected (#154)
+- add user feedback message for unsupported file type (#142)
+
+### Refactor
+
+- uppercase all entity names in error report (#190)
+- rename of error code and messages related to empty entities post business rules (#187)
+- removed ods udfs as they were functionally useless
+- remove original entities (#164)
+- tweak hierarchy node design to allow iteration both directions on hierarchies (#155)
+- unify HierarchyNode and ChildHierarchyNode also tweaked background writer queue type and use in orphaned records implementation (#153)
+
+### Perf
+
+- cache entities during orphan and group rejections to avoid recomputing multiple times (#172)
+- remove orphan tracker as it's causing significant performance degradation (#170)
+- only check for orphaned records on entities with at least one record (#166)
+
## v0.9.0 (2026-08-12)
### Fix
diff --git a/pyproject.toml b/pyproject.toml
index 8e129cbd..795ff082 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,7 +25,7 @@ Issues = "https://github.com/NHSDigital/data-validation-engine/issues"
Changelog = "https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md"
[tool.poetry]
-version = "0.9.0"
+version = "0.10.0"
packages = [
{ include = "dve", from = "src" },
]
From 81ff5c38fc6904b558c7739ea28eb5d8b451648f Mon Sep 17 00:00:00 2001
From: stevenhsd <56357022+stevenhsd@users.noreply.github.com>
Date: Fri, 30 Oct 2026 16:36:55 +0000
Subject: [PATCH 47/47] docs: tweaked changelog from autogen cz bump
---
CHANGELOG.md | 1 -
1 file changed, 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 84b480a9..d101b3c7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,7 +13,6 @@
- ensure error summary sheet populates entities in deterministic way (#191)
- total no records rejected showing when it shouldn't (#194)
-- total no records rejected showing when it shouldn't
- adjust regex check for timezone offsets, allowing : to be included (#181)
- refactor the join_header method in duckdb to deal with out of bound date values (#177)
- group rejection not filtering out invalid records correctly (#175)