From 7b5a6a7ce147d666de325ab69ffe7f9d915ebdad Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:49:25 +0100 Subject: [PATCH 1/3] fix: add entity into data contract details to deal with duplicate column definitions --- src/dve/core_engine/message.py | 12 ++-- src/dve/core_engine/validation.py | 22 ++++--- .../test_backends/fixtures.py | 46 +++++++++------ .../test_duckdb/test_data_contract.py | 6 +- tests/test_core_engine/test_message.py | 57 ++++++++++++------- .../flights_data_contract_error_details.json | 20 ++++--- .../movies/movies_contract_error_details.json | 46 ++++++++------- 7 files changed, 127 insertions(+), 82 deletions(-) diff --git a/src/dve/core_engine/message.py b/src/dve/core_engine/message.py index 78024e9..3df5414 100644 --- a/src/dve/core_engine/message.py +++ b/src/dve/core_engine/message.py @@ -239,7 +239,7 @@ def from_pydantic_error( record: Record, error: ValidationError, error_details: Optional[ - dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]] + dict[EntityName, dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]] ] = None, ) -> Messages: """Create messages from a `pydantic` validation error.""" @@ -255,9 +255,13 @@ def from_pydantic_error( error_field = ".".join([idx for idx in error_dict["loc"] if not isinstance(idx, int)]) - error_detail: DataContractErrorDetail = error_details.get( # type: ignore - error_field, DEFAULT_ERROR_DETAIL - ).get(category) + error_detail_entity = error_details.get(entity) # type: ignore + if error_detail_entity is None: + error_detail = DEFAULT_ERROR_DETAIL.get(category) # type: ignore + else: + error_detail: DataContractErrorDetail = error_detail_entity.get( # type: ignore + error_field, DEFAULT_ERROR_DETAIL + ).get(category) messages.append( cls( diff --git a/src/dve/core_engine/validation.py b/src/dve/core_engine/validation.py index 05f2e2b..b03aef4 100644 --- a/src/dve/core_engine/validation.py +++ b/src/dve/core_engine/validation.py @@ -38,7 +38,7 @@ def __init__( self._model: Optional[BaseModel] = None self._error_info = error_info or {} self._error_details: Optional[ - dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]] + dict[EntityName, dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]] ] = None def __reduce__(self): # Don't attempt to pickle Pydantic models. @@ -63,17 +63,21 @@ def model(self) -> BaseModel: return self._model @property - def error_details(self) -> dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]: + def error_details(self) -> dict[ + EntityName, dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]] + ]: """Custom error code and message mapping for contract phase""" if not self._error_details: - _error_details = { - field: { - err_type: DataContractErrorDetail(**detail) - for err_type, detail in err_details.items() + _temp_dict = {} + for entity_name, fields in self._error_info.items(): + _temp_dict[entity_name] = { + field: { + err_type: DataContractErrorDetail(**detail) + for err_type, detail in err_details.items() + } + for field, err_details in fields.items() } - for field, err_details in self._error_info.items() - } - self._error_details = _error_details + self._error_details = _temp_dict return self._error_details def __call__(self, record: Record) -> tuple[Optional[Record], Messages]: diff --git a/tests/test_core_engine/test_backends/fixtures.py b/tests/test_core_engine/test_backends/fixtures.py index 31c23d7..84a132c 100644 --- a/tests/test_core_engine/test_backends/fixtures.py +++ b/tests/test_core_engine/test_backends/fixtures.py @@ -565,25 +565,37 @@ def nested_all_string_parquet_w_errors(temp_dir, def nested_parquet_custom_dc_err_details(temp_dir): file_path = Path(temp_dir).joinpath("nested_parquet_data_contract_codes.json") err_details = { - "id": { - "Blank": {"error_code": "TESTIDBLANK", - "error_message": "id cannot be null", - "reporting_entity": "test_rename"}, - "Bad value": {"error_code": "TESTIDBAD", - "error_message": "id is invalid: id - {{id}}", - "reporting_entity": "test_rename"} + "nested_model": { + "id": { + "Blank": { + "error_code": "TESTIDBLANK", + "error_message": "id cannot be null", + "reporting_entity": "test_rename" }, - "datetimefield": { - "Bad value": {"error_code": "TESTDTFIELDBAD", - "error_message": "datetimefield is invalid: id - {{id}}, datetimefield - {{datetimefield}}"} - }, - "subfield.id": { - "Blank": {"error_code": "SUBFIELDTESTIDBLANK", - "error_message": "subfield id cannot be null"}, - "Bad value": {"error_code": "SUBFIELDTESTIDBAD", - "error_message": "subfield id is invalid: subfield.id - {{__error_value}}"} + "Bad value": { + "error_code": "TESTIDBAD", + "error_message": "id is invalid: id - {{id}}", + "reporting_entity": "test_rename" + } + }, + "datetimefield": { + "Bad value": { + "error_code": "TESTDTFIELDBAD", + "error_message": "datetimefield is invalid: id - {{id}}, datetimefield - {{datetimefield}}" + } + }, + "subfield.id": { + "Blank": { + "error_code": "SUBFIELDTESTIDBLANK", + "error_message": "subfield id cannot be null" }, - } + "Bad value": { + "error_code": "SUBFIELDTESTIDBAD", + "error_message": "subfield id is invalid: subfield.id - {{__error_value}}" + } + }, + } + } with open(file_path, mode="w") as fle: json.dump(err_details, fle) diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py index 2019a66..8548e53 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py @@ -372,7 +372,9 @@ def test_duckdb_data_contract_custom_error_details(nested_all_string_parquet_w_e reporting_fields={"nested_model": ["id"]}, ) - entities, feedback_errors_uri, stage_successful = data_contract.apply_data_contract(get_parent(parquet_uri), entities, {"nested_model": parquet_uri}, dc_meta) + entities, feedback_errors_uri, stage_successful = data_contract.apply_data_contract( + get_parent(parquet_uri), entities, {"nested_model": parquet_uri}, dc_meta + ) assert stage_successful messages: list[UserMessage] = [msg for msg in load_feedback_messages(feedback_errors_uri)] assert len(messages) == 2 @@ -381,4 +383,4 @@ def test_duckdb_data_contract_custom_error_details(nested_all_string_parquet_w_e assert messages[0].ErrorMessage == "subfield id is invalid: subfield.id - WRONG" assert messages[1].ErrorCode == "TESTIDBAD" assert messages[1].ErrorMessage == "id is invalid: id - WRONG" - assert messages[1].Entity == "test_rename" \ No newline at end of file + assert messages[1].Entity == "test_rename" diff --git a/tests/test_core_engine/test_message.py b/tests/test_core_engine/test_message.py index bb0d0d8..346334e 100644 --- a/tests/test_core_engine/test_message.py +++ b/tests/test_core_engine/test_message.py @@ -183,46 +183,56 @@ class TestModel(BaseModel): unimportant_field: Annotated[int, Field(default=None)] custom_error_details: str = """ - {"idx": {"Blank": {"error_code": "IDBLANKERRCODE", + {"test_entity": {"idx": {"Blank": {"error_code": "IDBLANKERRCODE", "error_message": "idx is a mandatory field", "is_informational": true}, "Bad value": {"error_code": "IDDODGYVALCODE", "error_message": "idx value is dodgy: {{idx}}", "error_level": "submission"}}, "date_field": {"Bad value": {"error_code": "DATEDODGYVALCODE", - "error_message": "date_field value is dodgy: idx: {{idx}}, date_field: {{date_field}}"}}} + "error_message": "date_field value is dodgy: idx: {{idx}}, date_field: {{date_field}}"}}} + } """ - error_details: Dict[str, Dict[str, DataContractErrorDetail]] = {field: {err_type: DataContractErrorDetail(**detail) - for err_type, detail in err_details.items()} - for field, err_details in json.loads(custom_error_details).items()} - + error_details: Dict[str, Dict[str, Dict[str, DataContractErrorDetail]]] = { + entity_name: { + field: { + err_type: DataContractErrorDetail(**detail) + for err_type, detail in err_details.items() + } + for field, err_details in fields.items() + } + for entity_name, fields in json.loads(custom_error_details).items() + } + _bad_value_data = {"idx": "ABC", "str_field": "test", "date_field": "terry", "unimportant_field": "dog"} _blank_value_data = {} - + try: TestModel(**_bad_value_data) except ValidationError as e: _error_bad_value = e - + try: TestModel(**_blank_value_data) except ValidationError as e: _error_blank = e - + msgs_bad= FeedbackMessage.from_pydantic_error(entity="test_entity", record = _bad_value_data, error=_error_bad_value, error_details=error_details) msgs_bad = sorted(msgs_bad, key=lambda x: x.error_location) + + entity_error_details = error_details["test_entity"] assert len(msgs_bad) == 3 - assert msgs_bad[0].error_code == error_details.get("date_field").get("Bad value").error_code - assert msgs_bad[0].error_message == error_details.get("date_field").get("Bad value").template_message(_bad_value_data) + assert msgs_bad[0].error_code == entity_error_details.get("date_field").get("Bad value").error_code + assert msgs_bad[0].error_message == entity_error_details.get("date_field").get("Bad value").template_message(_bad_value_data) assert msgs_bad[0].failure_type == "record" assert not msgs_bad[0].is_informational - assert msgs_bad[1].error_code == error_details.get("idx").get("Bad value").error_code - assert msgs_bad[1].error_message == error_details.get("idx").get("Bad value").template_message(_bad_value_data) + assert msgs_bad[1].error_code == entity_error_details.get("idx").get("Bad value").error_code + assert msgs_bad[1].error_message == entity_error_details.get("idx").get("Bad value").template_message(_bad_value_data) assert msgs_bad[1].failure_type == "submission" assert not msgs_bad[1].is_informational assert msgs_bad[2].error_code == bad_val_default.error_code @@ -239,8 +249,8 @@ class TestModel(BaseModel): msgs_blank = sorted(msgs_blank, key=lambda x: x.error_location) assert len(msgs_blank) == 2 - assert msgs_blank[0].error_code == error_details.get("idx").get("Blank").error_code - assert msgs_blank[0].error_message == error_details.get("idx").get("Blank").template_message(_blank_value_data) + assert msgs_blank[0].error_code == entity_error_details.get("idx").get("Blank").error_code + assert msgs_blank[0].error_message == entity_error_details.get("idx").get("Blank").template_message(_blank_value_data) assert msgs_blank[0].is_informational assert msgs_blank[1].error_code == blank_default.error_code assert msgs_blank[1].error_message == blank_default.error_message @@ -271,11 +281,18 @@ class TestModel(BaseModel): } ] } - custom_error_details: str = """{"sub_field.nested_field_2.test_date": {"Bad value": {"error_code": "DATEDODGYVALCODE", - "error_message": "date_field value is dodgy: a_field: {{a_field}}, date_field: {{__error_value}}"}}}""" - error_details: Dict[str, Dict[str, DataContractErrorDetail]] = {field: {err_type: DataContractErrorDetail(**detail) - for err_type, detail in err_details.items()} - for field, err_details in json.loads(custom_error_details).items()} + custom_error_details: str = """{"test_entity": {"sub_field.nested_field_2.test_date": {"Bad value": {"error_code": "DATEDODGYVALCODE", + "error_message": "date_field value is dodgy: a_field: {{a_field}}, date_field: {{__error_value}}"}}}}""" + error_details: Dict[str, Dict[str, Dict[str, DataContractErrorDetail]]] = { + entity_name: { + field: { + err_type: DataContractErrorDetail(**detail) + for err_type, detail in err_details.items() + } + for field, err_details in fields.items() + } + for entity_name, fields in json.loads(custom_error_details).items() + } try: TestModel(**test_record) except ValidationError as err: diff --git a/tests/testdata/flights/flights_data_contract_error_details.json b/tests/testdata/flights/flights_data_contract_error_details.json index 78e694d..3cf4327 100644 --- a/tests/testdata/flights/flights_data_contract_error_details.json +++ b/tests/testdata/flights/flights_data_contract_error_details.json @@ -1,14 +1,18 @@ { - "country_id": { - "Blank": { - "error_code": "CountryIdIsMissing", - "error_message": "Record Rejected - Country is missing an id" + "country": { + "country_id": { + "Blank": { + "error_code": "CountryIdIsMissing", + "error_message": "Record Rejected - Country is missing an id" + } } }, - "airport_id": { - "Blank": { - "error_code": "AirportIdIsMissing", - "error_message": "Record Rejected - Airport is missing an id" + "airport": { + "airport_id": { + "Blank": { + "error_code": "AirportIdIsMissing", + "error_message": "Record Rejected - Airport is missing an id" + } } } } \ No newline at end of file diff --git a/tests/testdata/movies/movies_contract_error_details.json b/tests/testdata/movies/movies_contract_error_details.json index 260ee96..e457555 100644 --- a/tests/testdata/movies/movies_contract_error_details.json +++ b/tests/testdata/movies/movies_contract_error_details.json @@ -1,27 +1,29 @@ { - "title": { - "Blank": { - "error_code": "BLANKTITLE", - "error_message": "title should not be blank", - "error_level": "submission" - } - }, - "year": { - "Blank": { - "error_code": "BLANKYEAR", - "error_message": "year not provided", - "is_informational": true + "movies": { + "title": { + "Blank": { + "error_code": "BLANKTITLE", + "error_message": "title should not be blank", + "error_level": "submission" + } }, - "Bad value": { - "error_code": "DODGYYEAR", - "error_message": "year value ({{year}}) is invalid", - "reporting_entity": "movies_rename_test" - } - }, - "cast.date_joined": { - "Bad value": { - "error_code": "DODGYDATE", - "error_message": "date_joined value is not valid: {{__error_value}}" + "year": { + "Blank": { + "error_code": "BLANKYEAR", + "error_message": "year not provided", + "is_informational": true + }, + "Bad value": { + "error_code": "DODGYYEAR", + "error_message": "year value ({{year}}) is invalid", + "reporting_entity": "movies_rename_test" + } + }, + "cast.date_joined": { + "Bad value": { + "error_code": "DODGYDATE", + "error_message": "date_joined value is not valid: {{__error_value}}" + } } } } \ No newline at end of file From 93f8239a46908e8bf830638bdc3a9a34a1caf3e1 Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Wed, 23 Sep 2026 16:49:39 +0100 Subject: [PATCH 2/3] Revert "fix: add entity into data contract details to deal with duplicate column definitions" This reverts commit 7b5a6a7ce147d666de325ab69ffe7f9d915ebdad. --- src/dve/core_engine/message.py | 12 ++-- src/dve/core_engine/validation.py | 22 +++---- .../test_backends/fixtures.py | 46 ++++++--------- .../test_duckdb/test_data_contract.py | 6 +- tests/test_core_engine/test_message.py | 57 +++++++------------ .../flights_data_contract_error_details.json | 20 +++---- .../movies/movies_contract_error_details.json | 46 +++++++-------- 7 files changed, 82 insertions(+), 127 deletions(-) diff --git a/src/dve/core_engine/message.py b/src/dve/core_engine/message.py index 3df5414..78024e9 100644 --- a/src/dve/core_engine/message.py +++ b/src/dve/core_engine/message.py @@ -239,7 +239,7 @@ def from_pydantic_error( record: Record, error: ValidationError, error_details: Optional[ - dict[EntityName, dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]] + dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]] ] = None, ) -> Messages: """Create messages from a `pydantic` validation error.""" @@ -255,13 +255,9 @@ def from_pydantic_error( error_field = ".".join([idx for idx in error_dict["loc"] if not isinstance(idx, int)]) - error_detail_entity = error_details.get(entity) # type: ignore - if error_detail_entity is None: - error_detail = DEFAULT_ERROR_DETAIL.get(category) # type: ignore - else: - error_detail: DataContractErrorDetail = error_detail_entity.get( # type: ignore - error_field, DEFAULT_ERROR_DETAIL - ).get(category) + error_detail: DataContractErrorDetail = error_details.get( # type: ignore + error_field, DEFAULT_ERROR_DETAIL + ).get(category) messages.append( cls( diff --git a/src/dve/core_engine/validation.py b/src/dve/core_engine/validation.py index b03aef4..05f2e2b 100644 --- a/src/dve/core_engine/validation.py +++ b/src/dve/core_engine/validation.py @@ -38,7 +38,7 @@ def __init__( self._model: Optional[BaseModel] = None self._error_info = error_info or {} self._error_details: Optional[ - dict[EntityName, dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]] + dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]] ] = None def __reduce__(self): # Don't attempt to pickle Pydantic models. @@ -63,21 +63,17 @@ def model(self) -> BaseModel: return self._model @property - def error_details(self) -> dict[ - EntityName, dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]] - ]: + def error_details(self) -> dict[FieldName, dict[ErrorCategory, DataContractErrorDetail]]: """Custom error code and message mapping for contract phase""" if not self._error_details: - _temp_dict = {} - for entity_name, fields in self._error_info.items(): - _temp_dict[entity_name] = { - field: { - err_type: DataContractErrorDetail(**detail) - for err_type, detail in err_details.items() - } - for field, err_details in fields.items() + _error_details = { + field: { + err_type: DataContractErrorDetail(**detail) + for err_type, detail in err_details.items() } - self._error_details = _temp_dict + for field, err_details in self._error_info.items() + } + self._error_details = _error_details return self._error_details def __call__(self, record: Record) -> tuple[Optional[Record], Messages]: diff --git a/tests/test_core_engine/test_backends/fixtures.py b/tests/test_core_engine/test_backends/fixtures.py index 84a132c..31c23d7 100644 --- a/tests/test_core_engine/test_backends/fixtures.py +++ b/tests/test_core_engine/test_backends/fixtures.py @@ -565,37 +565,25 @@ def nested_all_string_parquet_w_errors(temp_dir, def nested_parquet_custom_dc_err_details(temp_dir): file_path = Path(temp_dir).joinpath("nested_parquet_data_contract_codes.json") err_details = { - "nested_model": { - "id": { - "Blank": { - "error_code": "TESTIDBLANK", - "error_message": "id cannot be null", - "reporting_entity": "test_rename" + "id": { + "Blank": {"error_code": "TESTIDBLANK", + "error_message": "id cannot be null", + "reporting_entity": "test_rename"}, + "Bad value": {"error_code": "TESTIDBAD", + "error_message": "id is invalid: id - {{id}}", + "reporting_entity": "test_rename"} }, - "Bad value": { - "error_code": "TESTIDBAD", - "error_message": "id is invalid: id - {{id}}", - "reporting_entity": "test_rename" - } - }, - "datetimefield": { - "Bad value": { - "error_code": "TESTDTFIELDBAD", - "error_message": "datetimefield is invalid: id - {{id}}, datetimefield - {{datetimefield}}" - } - }, - "subfield.id": { - "Blank": { - "error_code": "SUBFIELDTESTIDBLANK", - "error_message": "subfield id cannot be null" + "datetimefield": { + "Bad value": {"error_code": "TESTDTFIELDBAD", + "error_message": "datetimefield is invalid: id - {{id}}, datetimefield - {{datetimefield}}"} + }, + "subfield.id": { + "Blank": {"error_code": "SUBFIELDTESTIDBLANK", + "error_message": "subfield id cannot be null"}, + "Bad value": {"error_code": "SUBFIELDTESTIDBAD", + "error_message": "subfield id is invalid: subfield.id - {{__error_value}}"} }, - "Bad value": { - "error_code": "SUBFIELDTESTIDBAD", - "error_message": "subfield id is invalid: subfield.id - {{__error_value}}" - } - }, - } - } + } with open(file_path, mode="w") as fle: json.dump(err_details, fle) diff --git a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py index 8548e53..2019a66 100644 --- a/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py +++ b/tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_data_contract.py @@ -372,9 +372,7 @@ def test_duckdb_data_contract_custom_error_details(nested_all_string_parquet_w_e reporting_fields={"nested_model": ["id"]}, ) - entities, feedback_errors_uri, stage_successful = data_contract.apply_data_contract( - get_parent(parquet_uri), entities, {"nested_model": parquet_uri}, dc_meta - ) + entities, feedback_errors_uri, stage_successful = data_contract.apply_data_contract(get_parent(parquet_uri), entities, {"nested_model": parquet_uri}, dc_meta) assert stage_successful messages: list[UserMessage] = [msg for msg in load_feedback_messages(feedback_errors_uri)] assert len(messages) == 2 @@ -383,4 +381,4 @@ def test_duckdb_data_contract_custom_error_details(nested_all_string_parquet_w_e assert messages[0].ErrorMessage == "subfield id is invalid: subfield.id - WRONG" assert messages[1].ErrorCode == "TESTIDBAD" assert messages[1].ErrorMessage == "id is invalid: id - WRONG" - assert messages[1].Entity == "test_rename" + assert messages[1].Entity == "test_rename" \ No newline at end of file diff --git a/tests/test_core_engine/test_message.py b/tests/test_core_engine/test_message.py index 346334e..bb0d0d8 100644 --- a/tests/test_core_engine/test_message.py +++ b/tests/test_core_engine/test_message.py @@ -183,56 +183,46 @@ class TestModel(BaseModel): unimportant_field: Annotated[int, Field(default=None)] custom_error_details: str = """ - {"test_entity": {"idx": {"Blank": {"error_code": "IDBLANKERRCODE", + {"idx": {"Blank": {"error_code": "IDBLANKERRCODE", "error_message": "idx is a mandatory field", "is_informational": true}, "Bad value": {"error_code": "IDDODGYVALCODE", "error_message": "idx value is dodgy: {{idx}}", "error_level": "submission"}}, "date_field": {"Bad value": {"error_code": "DATEDODGYVALCODE", - "error_message": "date_field value is dodgy: idx: {{idx}}, date_field: {{date_field}}"}}} - } + "error_message": "date_field value is dodgy: idx: {{idx}}, date_field: {{date_field}}"}}} """ - error_details: Dict[str, Dict[str, Dict[str, DataContractErrorDetail]]] = { - entity_name: { - field: { - err_type: DataContractErrorDetail(**detail) - for err_type, detail in err_details.items() - } - for field, err_details in fields.items() - } - for entity_name, fields in json.loads(custom_error_details).items() - } - + error_details: Dict[str, Dict[str, DataContractErrorDetail]] = {field: {err_type: DataContractErrorDetail(**detail) + for err_type, detail in err_details.items()} + for field, err_details in json.loads(custom_error_details).items()} + _bad_value_data = {"idx": "ABC", "str_field": "test", "date_field": "terry", "unimportant_field": "dog"} _blank_value_data = {} - + try: TestModel(**_bad_value_data) except ValidationError as e: _error_bad_value = e - + try: TestModel(**_blank_value_data) except ValidationError as e: _error_blank = e - + msgs_bad= FeedbackMessage.from_pydantic_error(entity="test_entity", record = _bad_value_data, error=_error_bad_value, error_details=error_details) msgs_bad = sorted(msgs_bad, key=lambda x: x.error_location) - - entity_error_details = error_details["test_entity"] assert len(msgs_bad) == 3 - assert msgs_bad[0].error_code == entity_error_details.get("date_field").get("Bad value").error_code - assert msgs_bad[0].error_message == entity_error_details.get("date_field").get("Bad value").template_message(_bad_value_data) + assert msgs_bad[0].error_code == error_details.get("date_field").get("Bad value").error_code + assert msgs_bad[0].error_message == error_details.get("date_field").get("Bad value").template_message(_bad_value_data) assert msgs_bad[0].failure_type == "record" assert not msgs_bad[0].is_informational - assert msgs_bad[1].error_code == entity_error_details.get("idx").get("Bad value").error_code - assert msgs_bad[1].error_message == entity_error_details.get("idx").get("Bad value").template_message(_bad_value_data) + assert msgs_bad[1].error_code == error_details.get("idx").get("Bad value").error_code + assert msgs_bad[1].error_message == error_details.get("idx").get("Bad value").template_message(_bad_value_data) assert msgs_bad[1].failure_type == "submission" assert not msgs_bad[1].is_informational assert msgs_bad[2].error_code == bad_val_default.error_code @@ -249,8 +239,8 @@ class TestModel(BaseModel): msgs_blank = sorted(msgs_blank, key=lambda x: x.error_location) assert len(msgs_blank) == 2 - assert msgs_blank[0].error_code == entity_error_details.get("idx").get("Blank").error_code - assert msgs_blank[0].error_message == entity_error_details.get("idx").get("Blank").template_message(_blank_value_data) + assert msgs_blank[0].error_code == error_details.get("idx").get("Blank").error_code + assert msgs_blank[0].error_message == error_details.get("idx").get("Blank").template_message(_blank_value_data) assert msgs_blank[0].is_informational assert msgs_blank[1].error_code == blank_default.error_code assert msgs_blank[1].error_message == blank_default.error_message @@ -281,18 +271,11 @@ class TestModel(BaseModel): } ] } - custom_error_details: str = """{"test_entity": {"sub_field.nested_field_2.test_date": {"Bad value": {"error_code": "DATEDODGYVALCODE", - "error_message": "date_field value is dodgy: a_field: {{a_field}}, date_field: {{__error_value}}"}}}}""" - error_details: Dict[str, Dict[str, Dict[str, DataContractErrorDetail]]] = { - entity_name: { - field: { - err_type: DataContractErrorDetail(**detail) - for err_type, detail in err_details.items() - } - for field, err_details in fields.items() - } - for entity_name, fields in json.loads(custom_error_details).items() - } + custom_error_details: str = """{"sub_field.nested_field_2.test_date": {"Bad value": {"error_code": "DATEDODGYVALCODE", + "error_message": "date_field value is dodgy: a_field: {{a_field}}, date_field: {{__error_value}}"}}}""" + error_details: Dict[str, Dict[str, DataContractErrorDetail]] = {field: {err_type: DataContractErrorDetail(**detail) + for err_type, detail in err_details.items()} + for field, err_details in json.loads(custom_error_details).items()} try: TestModel(**test_record) except ValidationError as err: diff --git a/tests/testdata/flights/flights_data_contract_error_details.json b/tests/testdata/flights/flights_data_contract_error_details.json index 3cf4327..78e694d 100644 --- a/tests/testdata/flights/flights_data_contract_error_details.json +++ b/tests/testdata/flights/flights_data_contract_error_details.json @@ -1,18 +1,14 @@ { - "country": { - "country_id": { - "Blank": { - "error_code": "CountryIdIsMissing", - "error_message": "Record Rejected - Country is missing an id" - } + "country_id": { + "Blank": { + "error_code": "CountryIdIsMissing", + "error_message": "Record Rejected - Country is missing an id" } }, - "airport": { - "airport_id": { - "Blank": { - "error_code": "AirportIdIsMissing", - "error_message": "Record Rejected - Airport is missing an id" - } + "airport_id": { + "Blank": { + "error_code": "AirportIdIsMissing", + "error_message": "Record Rejected - Airport is missing an id" } } } \ No newline at end of file diff --git a/tests/testdata/movies/movies_contract_error_details.json b/tests/testdata/movies/movies_contract_error_details.json index e457555..260ee96 100644 --- a/tests/testdata/movies/movies_contract_error_details.json +++ b/tests/testdata/movies/movies_contract_error_details.json @@ -1,29 +1,27 @@ { - "movies": { - "title": { - "Blank": { - "error_code": "BLANKTITLE", - "error_message": "title should not be blank", - "error_level": "submission" - } - }, - "year": { - "Blank": { - "error_code": "BLANKYEAR", - "error_message": "year not provided", - "is_informational": true - }, - "Bad value": { - "error_code": "DODGYYEAR", - "error_message": "year value ({{year}}) is invalid", - "reporting_entity": "movies_rename_test" - } + "title": { + "Blank": { + "error_code": "BLANKTITLE", + "error_message": "title should not be blank", + "error_level": "submission" + } + }, + "year": { + "Blank": { + "error_code": "BLANKYEAR", + "error_message": "year not provided", + "is_informational": true }, - "cast.date_joined": { - "Bad value": { - "error_code": "DODGYDATE", - "error_message": "date_joined value is not valid: {{__error_value}}" - } + "Bad value": { + "error_code": "DODGYYEAR", + "error_message": "year value ({{year}}) is invalid", + "reporting_entity": "movies_rename_test" + } + }, + "cast.date_joined": { + "Bad value": { + "error_code": "DODGYDATE", + "error_message": "date_joined value is not valid: {{__error_value}}" } } } \ No newline at end of file From b05162c15a73f92aab4fdd2475f90c25e339514d Mon Sep 17 00:00:00 2001 From: georgeRobertson <50412379+georgeRobertson@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:46:53 +0100 Subject: [PATCH 3/3] fix: add entity hierarchy for data contract details to fix issue with duplicate field definitions --- .../core_engine/configuration/v1/__init__.py | 2 +- .../flights_data_contract_error_details.json | 20 ++++---- .../movies/movies_contract_error_details.json | 46 ++++++++++--------- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/src/dve/core_engine/configuration/v1/__init__.py b/src/dve/core_engine/configuration/v1/__init__.py index b9b08b6..10a452d 100644 --- a/src/dve/core_engine/configuration/v1/__init__.py +++ b/src/dve/core_engine/configuration/v1/__init__.py @@ -380,7 +380,7 @@ def get_contract_metadata(self) -> DataContractMetadata: } reporting_fields[entity_name] = dataset_config.reporting_fields validators[entity_name] = RowValidator( - contract_dict, entity_name, error_info=error_info + contract_dict, entity_name, error_info=error_info.get(entity_name) ) return DataContractMetadata( diff --git a/tests/testdata/flights/flights_data_contract_error_details.json b/tests/testdata/flights/flights_data_contract_error_details.json index 78e694d..3cf4327 100644 --- a/tests/testdata/flights/flights_data_contract_error_details.json +++ b/tests/testdata/flights/flights_data_contract_error_details.json @@ -1,14 +1,18 @@ { - "country_id": { - "Blank": { - "error_code": "CountryIdIsMissing", - "error_message": "Record Rejected - Country is missing an id" + "country": { + "country_id": { + "Blank": { + "error_code": "CountryIdIsMissing", + "error_message": "Record Rejected - Country is missing an id" + } } }, - "airport_id": { - "Blank": { - "error_code": "AirportIdIsMissing", - "error_message": "Record Rejected - Airport is missing an id" + "airport": { + "airport_id": { + "Blank": { + "error_code": "AirportIdIsMissing", + "error_message": "Record Rejected - Airport is missing an id" + } } } } \ No newline at end of file diff --git a/tests/testdata/movies/movies_contract_error_details.json b/tests/testdata/movies/movies_contract_error_details.json index 260ee96..e457555 100644 --- a/tests/testdata/movies/movies_contract_error_details.json +++ b/tests/testdata/movies/movies_contract_error_details.json @@ -1,27 +1,29 @@ { - "title": { - "Blank": { - "error_code": "BLANKTITLE", - "error_message": "title should not be blank", - "error_level": "submission" - } - }, - "year": { - "Blank": { - "error_code": "BLANKYEAR", - "error_message": "year not provided", - "is_informational": true + "movies": { + "title": { + "Blank": { + "error_code": "BLANKTITLE", + "error_message": "title should not be blank", + "error_level": "submission" + } }, - "Bad value": { - "error_code": "DODGYYEAR", - "error_message": "year value ({{year}}) is invalid", - "reporting_entity": "movies_rename_test" - } - }, - "cast.date_joined": { - "Bad value": { - "error_code": "DODGYDATE", - "error_message": "date_joined value is not valid: {{__error_value}}" + "year": { + "Blank": { + "error_code": "BLANKYEAR", + "error_message": "year not provided", + "is_informational": true + }, + "Bad value": { + "error_code": "DODGYYEAR", + "error_message": "year value ({{year}}) is invalid", + "reporting_entity": "movies_rename_test" + } + }, + "cast.date_joined": { + "Bad value": { + "error_code": "DODGYDATE", + "error_message": "date_joined value is not valid: {{__error_value}}" + } } } } \ No newline at end of file