From c5ec4ef00d124a483b2de0aeb17587783832f430 Mon Sep 17 00:00:00 2001 From: Kay Robbins Date: Tue, 10 Nov 2026 12:25:32 -0600 Subject: [PATCH 1/3] Report missed errors; pin hed-tests babc444 Fix the false negatives in review items 8.1.1-8.1.7: - misnested parentheses are PARENTHESES_MISMATCH - a placeholder that is not the whole value is PLACEHOLDER_INVALID - curly-brace column names may contain any character - a definition beside other tags in a sidecar entry is DEFINITION_INVALID - a HED key nested in a sidecar column is SIDECAR_INVALID - units in a Def value are checked against the definition - sidecar strings that reference {HED} skip combination checks df_to_hed no longer adds a Description tag beside definitions. Pin hed-tests babc444 and hed-examples de5c4b0. --- CHANGELOG.md | 9 +++ hed/errors/error_messages.py | 4 +- hed/models/model_constants.py | 5 ++ hed/models/sidecar.py | 4 +- hed/tools/analysis/annotation_util.py | 6 +- hed/validator/def_validator.py | 13 +++- hed/validator/hed_validator.py | 42 ++++++++++ hed/validator/sidecar_validator.py | 76 +++++++++++++++---- hed/validator/util/string_util.py | 22 +++++- spec_tests/hed-examples | 2 +- spec_tests/hed-tests | 2 +- .../both_types_events_with_defs.json | 2 +- tests/models/test_definition_dict.py | 8 +- tests/validator/test_def_validator.py | 24 ++++++ tests/validator/test_hed_validator.py | 34 +++++++++ tests/validator/test_sidecar_validator.py | 61 ++++++++++++++- 16 files changed, 285 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0da057de..f3a012bb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ - **A row whose onset is `n/a` is validated wherever it is.** Such a row was skipped unless it was the last row of the file, so its assembled-row errors (for example a tag repeated across the `HED` column and the sidecar) went unreported. ([#1481](https://github.com/hed-standard/hed-python/issues/1481)) - **`n/a` onsets no longer cause `ONSETS_UNORDERED`**: only the numeric onsets must be in order. `EventManager` still requires every onset to be numeric. - **The HED string context of a `TEMPORAL_TAG_ERROR` for an unconvertible `Delay`** (`(Delay/3 month, (Red))`) keeps the `Delay` group; it was removed after the issue was made. +- **Errors that were missed are now reported** (hed-tests cases added in hed-tests #99): + - Misnested parentheses with equal counts, such as `(Red)), ((Blue)` and `)(Red)(` (`PARENTHESES_MISMATCH`). + - A `#` that is not the whole value of its tag, such as `Label/x#`, `Label/#x` or `Age/##`, in a sidecar or a definition (`PLACEHOLDER_INVALID`, specification Appendix B reason f). Units after one blank (`Frequency/# Hz`) are not part of the value. A definition containing `Age/##` now gets this code instead of `DEFINITION_INVALID`. + - A curly-brace reference whose name has characters other than letters, digits, hyphen and underscore, such as `{my.col}`: it was ignored, so an unknown column went unreported and a known one was never substituted (`SIDECAR_BRACES_INVALID` when unknown). + - A definition beside other tags in a sidecar entry, categorical or value, such as `(Definition/MyDef, (Red)), Label/#` (`DEFINITION_INVALID`). An entry that holds only definitions is still valid. + - A `HED` key below the second level of an entry that also has a valid one, such as in `Levels` (`SIDECAR_INVALID`). + - Units in a `Def` value checked against the definition's placeholder tag with its units: `Def/Freq/3 kHz` with `(Definition/Freq/#, (Frequency/# Hz))` gives `Frequency/3 kHz Hz` (`UNITS_INVALID`). +- **A sidecar string that references `{HED}` is no longer checked for repeated tags and group rules in the sidecar**, since what the `HED` column contributes is known only per row (specification 3.2.9.3 rule 10). `(Red, {HED}), (Red)` was reported as `TAG_EXPRESSION_REPEATED` although a row with `HED=Blue` is valid; such errors are now found when each row is assembled. +- **`annotation_util.df_to_hed` with `description_tag=True` no longer adds a `Description` tag beside definitions**, which made the sidecar invalid; the description stays in the `Levels` or `Description` text. - In a file without an `onset` column, the `char_index` of an assembled-row issue now counts in the assembled string with `", "` between cells, as for files with onsets (one more per earlier cell than before). ## Specification files diff --git a/hed/errors/error_messages.py b/hed/errors/error_messages.py index e58b419d8..ebd83a003 100644 --- a/hed/errors/error_messages.py +++ b/hed/errors/error_messages.py @@ -177,7 +177,9 @@ def val_error_reserved_tag_splice(tag, group): @hed_error(ValidationErrors.PARENTHESES_MISMATCH) -def val_error_parentheses(opening_parentheses_count, closing_parentheses_count): +def val_error_parentheses(opening_parentheses_count, closing_parentheses_count, unmatched_index=None): + if unmatched_index is not None: + return f"Closing parenthesis at index {unmatched_index} has no opening parenthesis before it." return ( f"Number of opening and closing parentheses are unequal. " f"{opening_parentheses_count} opening parentheses. {closing_parentheses_count} " diff --git a/hed/models/model_constants.py b/hed/models/model_constants.py index 0c6f87934..7b1c714df 100644 --- a/hed/models/model_constants.py +++ b/hed/models/model_constants.py @@ -1,7 +1,12 @@ """Defined constants for definitions, def labels, and expanded labels.""" +import re from enum import IntEnum +# A curly-brace column reference in a sidecar annotation, ``{column_name}`` or ``{HED}``. A column name may hold +# any character but a brace (``{my.col}``); nested and unmatched braces are reported before this is applied. +COLUMN_REF_PATTERN = re.compile(r"\{([^{}]+)\}") + # The whitespace that may not begin or end a column name or a top-level sidecar key (specification 3.2.10): # blank, tab, line feed and carriage return only. A non-breaking space is not padding. NAME_WHITESPACE = " \t\n\r" diff --git a/hed/models/sidecar.py b/hed/models/sidecar.py index 484a1200a..2ff83054a 100644 --- a/hed/models/sidecar.py +++ b/hed/models/sidecar.py @@ -1,7 +1,6 @@ """Contents of a JSON file or merged JSON files.""" import json -import re from hed.errors import ErrorHandler from hed.errors.error_types import ErrorContext @@ -9,6 +8,7 @@ from hed.models.column_metadata import ColumnMetadata, ColumnType from hed.models.definition_dict import DefinitionDict from hed.models.hed_string import HedString +from hed.models.model_constants import COLUMN_REF_PATTERN class Sidecar: @@ -240,7 +240,7 @@ def get_column_refs(self) -> list[str]: if column_data.column_type == ColumnType.Ignore: continue hed_strings = column_data.get_hed_strings() - matches = hed_strings.str.findall(r"\{([a-z_\-0-9]+)\}", re.IGNORECASE) + matches = hed_strings.str.findall(COLUMN_REF_PATTERN) u_vals = [match for sublist in matches for match in sublist] found_vals.update(u_vals) diff --git a/hed/tools/analysis/annotation_util.py b/hed/tools/analysis/annotation_util.py index 1018f0b95..99e603438 100644 --- a/hed/tools/analysis/annotation_util.py +++ b/hed/tools/analysis/annotation_util.py @@ -388,7 +388,11 @@ def _get_value_entry(hed_entry, description_entry, description_tag=True): tags = hed_entry if description_entry and description_entry != "n/a": value_dict["Description"] = description_entry - if description_tag and tags: + if "definition/" in tags.casefold(): + # An entry that holds definitions may hold nothing else (specification 3.2.9.1), so the description + # is kept only in the Description or Levels text, not as a Description tag beside the definitions. + pass + elif description_tag and tags: tags = tags + ", Description/" + description_entry elif description_tag and not tags: tags = "Description/" + description_entry diff --git a/hed/validator/def_validator.py b/hed/validator/def_validator.py index 2edca10df..2f7fe8de4 100644 --- a/hed/validator/def_validator.py +++ b/hed/validator/def_validator.py @@ -144,8 +144,19 @@ def validate_def_value_units(self, def_tag, hed_validator, allow_placeholders=Fa def_contents = def_entry.get_definition(def_tag, placeholder_value=placeholder, return_copy_of_tag=True) if def_contents and def_entry.takes_value and hed_validator: placeholder_tag = def_contents.get_first_group().find_placeholder_tag() + # When the definition writes units after its placeholder (Frequency/# Hz), the substituted tag is + # what must be valid: Def/Freq/3 kHz gives Frequency/3 kHz Hz, which is UNITS_INVALID. The value + # starts the extension in both, so the index offset is the same. + validate_text = placeholder + template_tag = def_entry.contents.find_placeholder_tag() if def_entry.contents else None + if template_tag is not None and template_tag.extension != "#": + validate_text = placeholder_tag.extension def_issues += hed_validator.validate_units( - placeholder_tag, placeholder, report_as=def_tag, error_code=error_code, index_offset=len(tag_label) + 1 + placeholder_tag, + validate_text, + report_as=def_tag, + error_code=error_code, + index_offset=len(tag_label) + 1, ) return def_issues diff --git a/hed/validator/hed_validator.py b/hed/validator/hed_validator.py index 8eeb0ec52..df140e01e 100644 --- a/hed/validator/hed_validator.py +++ b/hed/validator/hed_validator.py @@ -281,6 +281,8 @@ def _validate_individual_tags_in_hed_string(self, hed_string_obj, allow_placehol ) elif hed_tag.short_base_tag == DefTagNames.DEFINITION_KEY: validation_issues += self._validate_definition_value(hed_tag, allow_placeholders) + elif allow_placeholders and self._placeholder_not_whole_value(hed_tag): + validation_issues += self._report_placeholder_not_whole_value(hed_tag) elif allow_placeholders and hed_tag.is_unit_class_tag() and hed_tag.extension.startswith("# "): # If placeholder is followed by units (e.g., "# m-per-s^2"), validate the units validation_issues += self.validate_units(hed_tag, allow_placeholders=allow_placeholders) @@ -289,6 +291,46 @@ def _validate_individual_tags_in_hed_string(self, hed_string_obj, allow_placehol return validation_issues + @staticmethod + def _placeholder_not_whole_value(hed_tag) -> bool: + """Return True if a takes-value tag has a ``#`` that is not the whole of its value. + + Parameters: + hed_tag (HedTag): A tag that is not Definition, Def or Def-expand. + + Returns: + bool: True for ``Label/x#`` or ``Label/#x``; False for ``Label/#``, ``Frequency/# Hz`` and for a + value without ``#``. + + Notes: + - PLACEHOLDER_INVALID reason f of the specification (Appendix B). Units after one blank are not part + of the value. + """ + extension = hed_tag.extension + if "#" not in extension or extension == "#" or not hed_tag.is_takes_value_tag(): + return False + return not (hed_tag.is_unit_class_tag() and extension.startswith("# ")) + + @staticmethod + def _report_placeholder_not_whole_value(hed_tag) -> list[dict]: + """Report PLACEHOLDER_INVALID for a tag whose value has a ``#`` that is not the whole value. + + Parameters: + hed_tag (HedTag): A tag for which ``_placeholder_not_whole_value`` is True. + + Returns: + list[dict]: One PLACEHOLDER_INVALID issue, at the first ``#``. The value-class check is skipped, so + the code is PLACEHOLDER_INVALID rather than CHARACTER_INVALID. + """ + index = len(hed_tag.org_base_tag) + 1 + hed_tag.extension.index("#") + return error_reporter.ErrorHandler.format_error( + ValidationErrors.INVALID_TAG_CHARACTER, + tag=hed_tag, + index_in_tag=index, + index_in_tag_end=index + 1, + actual_error=ValidationErrors.PLACEHOLDER_INVALID, + ) + def _validate_definition_value(self, definition_tag, allow_placeholders) -> list[dict]: """Validate the two-part value of a Definition tag: a name, then optionally ``/#``. diff --git a/hed/validator/sidecar_validator.py b/hed/validator/sidecar_validator.py index 6650dd0d8..ae6bd195b 100644 --- a/hed/validator/sidecar_validator.py +++ b/hed/validator/sidecar_validator.py @@ -4,7 +4,6 @@ import copy import itertools -import re from hed.errors import ColumnErrors, DefinitionErrors, ErrorContext, ErrorHandler, SidecarErrors, ValidationErrors from hed.errors.error_reporter import check_for_any_errors, sort_issues @@ -13,11 +12,9 @@ from hed.models.column_metadata import ColumnMetadata from hed.models.column_source import is_padded from hed.models.hed_string import HedString -from hed.models.model_constants import DefTagNames +from hed.models.model_constants import COLUMN_REF_PATTERN, DefTagNames, TopTagReturnType from hed.validator.reserved_checker import ReservedChecker -COLUMN_REF_PATTERN = re.compile(r"\{([a-z_\-0-9]+)\}", re.IGNORECASE) - # todo: Add/improve validation for definitions being in known columns(right now it just assumes they aren't) class SidecarValidator: @@ -82,6 +79,7 @@ def _validate_sidecar(self, sidecar, extra_def_dicts, error_handler) -> list[dic # todo: Break this function up all_ref_columns = sidecar.get_column_refs() definition_checks = {} + reported_definitions = set() for column_data in sidecar: column_name = column_data.column_name column_data = column_data._get_unvalidated_data() @@ -101,6 +99,10 @@ def _validate_sidecar(self, sidecar, extra_def_dicts, error_handler) -> list[dic def_check_list.append( hed_string_obj.find_tags({DefTagNames.DEFINITION_KEY}, recursive=True, include_groups=0) ) + misplaced = self._misplaced_definitions(hed_string_obj, def_check_list[-1]) + for def_tag in misplaced: + new_issues += ErrorHandler.format_error(DefinitionErrors.BAD_DEFINITION_LOCATION, def_tag) + reported_definitions.update(id(def_tag) for def_tag in misplaced) # Might refine this later - for now just skip checking placeholder counts in definition columns. if not def_check_list[-1]: @@ -110,13 +112,13 @@ def _validate_sidecar(self, sidecar, extra_def_dicts, error_handler) -> list[dic issues += new_issues error_handler.pop_error_context() # Hed String - # Only do full string checks on full columns, not partial ref columns. - if not is_ref_column: - # TODO: Figure out why this pattern is giving lint errors. - refs = COLUMN_REF_PATTERN.findall(hed_string) + # Only do full string checks on full columns, not partial ref columns. Every combination of the + # referenced columns' annotations is checked, whether or not a file uses it (specification + # 3.2.9.3 rule 10). What {HED} contributes is known only per row, so a string that references it + # gets its full-string checks when each row is assembled. + refs = COLUMN_REF_PATTERN.findall(hed_string) + if not is_ref_column and "HED" not in refs: refs_strings = {data.column_name: data.get_hed_strings() for data in sidecar} - if "HED" not in refs_strings: - refs_strings["HED"] = ["n/a"] for combination in itertools.product(*[refs_strings[key] for key in refs]): new_issues = [] ref_dict = dict(zip(refs, combination, strict=False)) @@ -137,7 +139,7 @@ def _validate_sidecar(self, sidecar, extra_def_dicts, error_handler) -> list[dic error_handler.pop_error_context() # Category key error_handler.pop_error_context() # Column Name - issues += self._check_definitions_bad_spot(definition_checks, error_handler) + issues += self._check_definitions_bad_spot(definition_checks, error_handler, reported_definitions) issues = sort_issues(issues) return issues @@ -326,8 +328,14 @@ def _validate_column_structure(self, column_name, dict_for_entry, error_handler) found_hed = self._check_for_key("HED", dict_for_entry) if found_hed: val_issues += error_handler.format_error_with_context(SidecarErrors.SIDECAR_HED_USED) - elif column_type == ColumnType.Categorical: - val_issues += self._validate_categorical_column(column_name, dict_for_entry, error_handler) + else: + # A HED key is allowed only directly under the entry (specification 3.2.9.1): one in "Levels" or + # deeper is SIDECAR_INVALID even when the entry has a valid HED key of its own. + nested_keys = {key: value for key, value in dict_for_entry.items() if key != "HED"} + if self._check_for_key("HED", nested_keys): + val_issues += error_handler.format_error_with_context(SidecarErrors.SIDECAR_HED_USED) + if column_type == ColumnType.Categorical: + val_issues += self._validate_categorical_column(column_name, dict_for_entry, error_handler) return val_issues @@ -381,14 +389,50 @@ def _validate_pound_sign_count(self, hed_string, column_type): return [] - def _check_definitions_bad_spot(self, definition_checks, error_handler): + @staticmethod + def _misplaced_definitions(hed_string_obj, definition_tags) -> list: + """Return the Definition tags of one sidecar string that has tags other than definitions. + + Parameters: + hed_string_obj (HedString): The string of one sidecar entry, with its column references removed. + definition_tags (list[HedTag]): The Definition tags found anywhere in the string. + + Returns: + list[HedTag]: Every Definition tag of the string if anything other than definition groups is at its + top level; else empty. + + Notes: + - Definitions go in dummy entries that hold only definitions, categorical or not (specification + 3.2.9.1); a definition beside other tags, as in a value template, is DEFINITION_INVALID. + """ + if not definition_tags: + return [] + top_level_definition_groups = hed_string_obj.find_top_level_tags( + anchor_tags={DefTagNames.DEFINITION_KEY}, include_groups=TopTagReturnType.GROUPS + ) + if len(top_level_definition_groups) != len(hed_string_obj.children): + return list(definition_tags) + return [] + + def _check_definitions_bad_spot(self, definition_checks, error_handler, reported_definitions=None): + """Report the definitions of a column in which some entries have definitions and some do not. + + Parameters: + definition_checks (dict): Column name -> list, per entry, of the Definition tags it holds. + error_handler (ErrorHandler): Sets the context for the error reporting. + reported_definitions (set or None): ids of Definition tags already reported for their own entry, + which are not reported again. + + Returns: + list[dict]: BAD_DEFINITION_LOCATION issues. + """ issues = [] - # This could be simplified now + reported_definitions = reported_definitions or set() for col_name, has_def in definition_checks.items(): error_handler.push_error_context(ErrorContext.SIDECAR_COLUMN_NAME, col_name) def_check = {bool(d) for d in has_def} if len(def_check) != 1: - flat_def_list = [d for defs in has_def for d in defs] + flat_def_list = [d for defs in has_def for d in defs if id(d) not in reported_definitions] for d in flat_def_list: issues += error_handler.format_error_with_context(DefinitionErrors.BAD_DEFINITION_LOCATION, d) error_handler.pop_error_context() diff --git a/hed/validator/util/string_util.py b/hed/validator/util/string_util.py index daebc89fe..3a5346c27 100644 --- a/hed/validator/util/string_util.py +++ b/hed/validator/util/string_util.py @@ -30,22 +30,38 @@ def run_string_validator(self, hed_string_obj): def check_count_tag_group_parentheses(hed_string) -> list[dict]: """Report unmatched parentheses. + The counts of opening and closing parentheses must be equal, and no closing parenthesis may come + before its opening one (`(Red)), ((Blue)` has equal counts but is misnested). One issue is + reported per string. + Parameters: hed_string (str): A HED string. Returns: list: A list of validation list. Each issue is a dictionary. """ - validation_issues = [] number_open_parentheses = hed_string.count("(") number_closed_parentheses = hed_string.count(")") if number_open_parentheses != number_closed_parentheses: - validation_issues += ErrorHandler.format_error( + return ErrorHandler.format_error( ValidationErrors.PARENTHESES_MISMATCH, opening_parentheses_count=number_open_parentheses, closing_parentheses_count=number_closed_parentheses, ) - return validation_issues + depth = 0 + for index, character in enumerate(hed_string): + if character == "(": + depth += 1 + elif character == ")": + depth -= 1 + if depth < 0: + return ErrorHandler.format_error( + ValidationErrors.PARENTHESES_MISMATCH, + opening_parentheses_count=number_open_parentheses, + closing_parentheses_count=number_closed_parentheses, + unmatched_index=index, + ) + return [] def check_delimiter_issues_in_hed_string(self, hed_string) -> list[dict]: """Report missing commas or commas in value tags. diff --git a/spec_tests/hed-examples b/spec_tests/hed-examples index 799e3c8b3..de5c4b08d 160000 --- a/spec_tests/hed-examples +++ b/spec_tests/hed-examples @@ -1 +1 @@ -Subproject commit 799e3c8b3059738eabf7eebca83738246ca354de +Subproject commit de5c4b08da4e6403f5e8800f9b3861c5ec25933d diff --git a/spec_tests/hed-tests b/spec_tests/hed-tests index b0bd646a0..babc44406 160000 --- a/spec_tests/hed-tests +++ b/spec_tests/hed-tests @@ -1 +1 @@ -Subproject commit b0bd646a042472662178add7b61fcbfb2f05c52e +Subproject commit babc44406dfede6590de63bba5bb48792f97c817 diff --git a/tests/data/sidecar_tests/both_types_events_with_defs.json b/tests/data/sidecar_tests/both_types_events_with_defs.json index 4abd3d3fa..4d62d3ac7 100644 --- a/tests/data/sidecar_tests/both_types_events_with_defs.json +++ b/tests/data/sidecar_tests/both_types_events_with_defs.json @@ -30,7 +30,7 @@ "unit_class_def": { "LongName": "Def with a value class", "Description": "Relative path of the stimulus image file", - "HED": "Age/#, (Definition/ValueClassDef/#, (Acceleration/#))" + "HED": "(Definition/ValueClassDef/#, (Acceleration/#))" }, "Defs": { "HED": "(Definition/JsonFileDef/#, (Acceleration/#,Item/JsonDef1))" diff --git a/tests/models/test_definition_dict.py b/tests/models/test_definition_dict.py index f76a93a9e..b697360ce 100644 --- a/tests/models/test_definition_dict.py +++ b/tests/models/test_definition_dict.py @@ -153,8 +153,14 @@ def test_definitions(self): "doublePoundSignPlaceholder": self.format_error( DefinitionErrors.INVALID_DEFINITION_EXTENSION, tag=0, def_name="InvalidDef" ), + # A '#' that is not the whole value is PLACEHOLDER_INVALID (reason f), found before the count of + # placeholder tags is checked. "doublePoundSignDiffPlaceholder": self.format_error( - DefinitionErrors.WRONG_NUMBER_PLACEHOLDER_TAGS, "InvalidDef", expected_count=1, tag_list=["Age/##"] + ValidationErrors.INVALID_TAG_CHARACTER, + tag=1, + index_in_tag=4, + index_in_tag_end=5, + actual_error=ValidationErrors.PLACEHOLDER_INVALID, ), "placeholdersWrongSpot": [], } diff --git a/tests/validator/test_def_validator.py b/tests/validator/test_def_validator.py index 714313bc7..2ecdc5c3c 100644 --- a/tests/validator/test_def_validator.py +++ b/tests/validator/test_def_validator.py @@ -389,3 +389,27 @@ def test_a_plain_tag_is_unchanged(self): if __name__ == "__main__": unittest.main() + + +class TestDefValueUnits(unittest.TestCase): + """A Def value is checked as substituted into its definition's placeholder tag (HED 8.4.0).""" + + @classmethod + def setUpClass(cls): + cls.hed_schema = schema.load_schema_version("8.4.0") + cls.def_dict = DefinitionDict( + "(Definition/Freq/#, (Frequency/# Hz)), (Definition/Acc/#, (Acceleration/#))", cls.hed_schema + ) + + def codes(self, hed_string): + validator = HedValidator(self.hed_schema, def_dicts=self.def_dict) + issues = validator.validate(HedString(hed_string, self.hed_schema), allow_placeholders=False) + return [issue["code"] for issue in issues] + + def test_units_after_definition_placeholder(self): + self.assertIn("UNITS_INVALID", self.codes("Def/Freq/3 kHz")) + self.assertEqual(self.codes("Def/Freq/3"), []) + + def test_units_in_def_value(self): + self.assertIn("UNITS_INVALID", self.codes("Def/Acc/3 m")) + self.assertEqual(self.codes("Def/Acc/3 m-per-s^2"), []) diff --git a/tests/validator/test_hed_validator.py b/tests/validator/test_hed_validator.py index 13b40071b..2d7097876 100644 --- a/tests/validator/test_hed_validator.py +++ b/tests/validator/test_hed_validator.py @@ -317,3 +317,37 @@ def test_severity_enum_not_string_github_hedit_161(self): if __name__ == "__main__": unittest.main() + + +class TestStringAndPlaceholderChecks(unittest.TestCase): + """Misnested parentheses and a placeholder that is not the whole value (HED 8.4.0).""" + + @classmethod + def setUpClass(cls): + cls.hed_schema = schema.load_schema_version("8.4.0") + + def codes(self, hed_string, allow_placeholders=False): + validator = HedValidator(self.hed_schema) + issues = validator.validate(HedString(hed_string, self.hed_schema), allow_placeholders=allow_placeholders) + return [issue["code"] for issue in issues] + + def test_misnested_parentheses_equal_counts(self): + for hed_string in ["(Red)), ((Blue)", ")(Red)("]: + with self.subTest(hed_string=hed_string): + self.assertIn("PARENTHESES_MISMATCH", self.codes(hed_string)) + + def test_nested_parentheses_valid(self): + self.assertEqual(self.codes("(Red, (Blue)), (Green)"), []) + + def test_unequal_counts_one_issue(self): + self.assertEqual(self.codes("(Red))").count("PARENTHESES_MISMATCH"), 1) + + def test_placeholder_not_whole_value(self): + for hed_string in ["Label/x#", "Label/#x", "Frequency/3# Hz", "Age/##"]: + with self.subTest(hed_string=hed_string): + self.assertEqual(self.codes(hed_string, allow_placeholders=True), ["PLACEHOLDER_INVALID"]) + + def test_placeholder_whole_value_valid(self): + for hed_string in ["Label/#", "Frequency/# Hz", "Age/#"]: + with self.subTest(hed_string=hed_string): + self.assertEqual(self.codes(hed_string, allow_placeholders=True), []) diff --git a/tests/validator/test_sidecar_validator.py b/tests/validator/test_sidecar_validator.py index d7c3dec12..d490c9243 100644 --- a/tests/validator/test_sidecar_validator.py +++ b/tests/validator/test_sidecar_validator.py @@ -48,7 +48,9 @@ def test_malformed_refs(self): sidecar = Sidecar(self._malformed_refs_json_filename) issues = sidecar.validate(self.hed_schema) - self.assertEqual(len(issues), 4) + # The {Property/...} reference is an unknown column (any character but a brace may name one), which + # also makes column1 a reference column that column2 and column4 reference: 3 more issues. + self.assertEqual(len(issues), 7) def test_padded_keys(self): # A padded top-level key is SIDECAR_INVALID with or without a "HED" entry; an inner blank is fine. @@ -259,3 +261,60 @@ def test_validate_pops_context_on_structure_error(self): self.assertIn(SidecarErrors.SIDECAR_BRACES_INVALID, {issue["code"] for issue in issues}) self.assertEqual(error_handler.error_context, []) + + +class TestSidecarFalseNegatives(unittest.TestCase): + """Sidecar errors that hedtools 1.3.0 did not report (HED 8.4.0).""" + + @classmethod + def setUpClass(cls): + cls.hed_schema = schema.load_schema_version("8.4.0") + + def codes(self, sidecar_dict): + sidecar = Sidecar(io.StringIO(json.dumps(sidecar_dict))) + return [issue["code"] for issue in sidecar.validate(self.hed_schema)] + + def test_brace_name_with_period_unknown(self): + self.assertIn("SIDECAR_BRACES_INVALID", self.codes({"cond": {"HED": {"a": "Red, {my.col}"}}})) + + def test_brace_name_with_period_known(self): + sidecar = {"cond": {"HED": {"a": "Red, {my.col}"}}, "my.col": {"HED": {"b": "Blue"}}} + self.assertEqual(self.codes(sidecar), []) + self.assertEqual(Sidecar(io.StringIO(json.dumps(sidecar))).get_column_refs(), ["my.col"]) + + def test_every_categorical_substitution_checked(self): + sidecar = {"event_code": {"HED": {"face": "(Red, {color})"}}, "color": {"HED": {"warm": "Red", "cool": "Blue"}}} + self.assertEqual(self.codes(sidecar), ["TAG_EXPRESSION_REPEATED"]) + + def test_hed_reference_not_checked_in_sidecar(self): + # What {HED} contributes is known only per row (specification 3.2.9.3 rule 10). + self.assertEqual(self.codes({"c": {"HED": {"a": "(Red, {HED}), (Red)"}}}), []) + + def test_definition_beside_tags(self): + cases = [ + {"cond": {"HED": {"a": "(Definition/MyDef, (Red)), Blue"}}}, + {"val": {"HED": "(Definition/MyDef, (Red)), Label/#"}}, + {"val": {"HED": "(Definition/MyDef, (Red)), Label/#, Age/#"}}, + ] + for sidecar in cases: + with self.subTest(sidecar=sidecar): + self.assertEqual(self.codes(sidecar), ["DEFINITION_INVALID"]) + + def test_definitions_alone_valid(self): + cases = [ + {"defs": {"HED": {"a": "(Definition/MyDef, (Red))", "b": "(Definition/Other, (Blue))"}}}, + {"defs": {"HED": "(Definition/MyDef, (Red)), (Definition/Other/#, (Label/#))"}}, + ] + for sidecar in cases: + with self.subTest(sidecar=sidecar): + self.assertEqual(self.codes(sidecar), []) + + def test_nested_hed_key(self): + cases = [ + {"cond": {"HED": {"a": "Red"}, "Levels": {"a": "x", "HED": "Blue"}}}, + {"cond": {"HED": {"a": "Red"}, "extra": {"inner": {"HED": "Blue"}}}}, + {"val": {"HED": "Label/#", "extra": {"HED": "Blue"}}}, + ] + for sidecar in cases: + with self.subTest(sidecar=sidecar): + self.assertEqual(self.codes(sidecar), ["SIDECAR_INVALID"]) From a964920d382ab704be5be0029ed9a82df3963bc8 Mon Sep 17 00:00:00 2001 From: Kay Robbins Date: Tue, 10 Nov 2026 13:33:02 -0600 Subject: [PATCH 2/3] Address review of PR 1501 - A brace name with a comma or parenthesis is not a column reference, since a reference must parse as one tag. - Escape the reference in replace_ref's removal regex, so {my.col} and {col+1} remove only themselves. - df_to_hed detects Definition tags, not any "definition/" substring, before leaving out the Description tag. --- CHANGELOG.md | 2 +- hed/models/df_util.py | 3 ++- hed/models/model_constants.py | 6 ++++-- hed/tools/analysis/annotation_util.py | 5 ++++- tests/models/test_df_util.py | 12 ++++++++++++ tests/tools/analysis/test_annotation_util.py | 10 ++++++++++ tests/validator/test_sidecar_validator.py | 9 +++++++++ 7 files changed, 42 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3a012bb9..ce81cd69b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ - **Errors that were missed are now reported** (hed-tests cases added in hed-tests #99): - Misnested parentheses with equal counts, such as `(Red)), ((Blue)` and `)(Red)(` (`PARENTHESES_MISMATCH`). - A `#` that is not the whole value of its tag, such as `Label/x#`, `Label/#x` or `Age/##`, in a sidecar or a definition (`PLACEHOLDER_INVALID`, specification Appendix B reason f). Units after one blank (`Frequency/# Hz`) are not part of the value. A definition containing `Age/##` now gets this code instead of `DEFINITION_INVALID`. - - A curly-brace reference whose name has characters other than letters, digits, hyphen and underscore, such as `{my.col}`: it was ignored, so an unknown column went unreported and a known one was never substituted (`SIDECAR_BRACES_INVALID` when unknown). + - A curly-brace reference whose name has characters other than letters, digits, hyphen and underscore, such as `{my.col}`: it was ignored, so an unknown column went unreported and a known one was never substituted (`SIDECAR_BRACES_INVALID` when unknown). A name with a comma or parenthesis is still not a reference, since a reference must parse as one tag. - A definition beside other tags in a sidecar entry, categorical or value, such as `(Definition/MyDef, (Red)), Label/#` (`DEFINITION_INVALID`). An entry that holds only definitions is still valid. - A `HED` key below the second level of an entry that also has a valid one, such as in `Levels` (`SIDECAR_INVALID`). - Units in a `Def` value checked against the definition's placeholder tag with its units: `Def/Freq/3 kHz` with `(Definition/Freq/#, (Frequency/# Hz))` gives `Frequency/3 kHz Hz` (`UNITS_INVALID`). diff --git a/hed/models/df_util.py b/hed/models/df_util.py index 81188136b..46144d9d9 100644 --- a/hed/models/df_util.py +++ b/hed/models/df_util.py @@ -173,7 +173,8 @@ def _remover(match): # c1/c2 contain the comma(and possibly spaces) separating this ref from other tags # p1/p2 contain the parentheses directly surrounding the tag # All four groups can have spaces. - pattern = r"(?P[\s,]*)(?P[(\s]*)" + old_value + r"(?P[\s)]*)(?P[\s,]*)" + # The reference is escaped: a column name may hold regex characters ({my.col}, {col+1}). + pattern = r"(?P[\s,]*)(?P[(\s]*)" + re.escape(old_value) + r"(?P[\s)]*)(?P[\s,]*)" return re.sub(pattern, _remover, text) diff --git a/hed/models/model_constants.py b/hed/models/model_constants.py index 7b1c714df..b4c2a6752 100644 --- a/hed/models/model_constants.py +++ b/hed/models/model_constants.py @@ -4,8 +4,10 @@ from enum import IntEnum # A curly-brace column reference in a sidecar annotation, ``{column_name}`` or ``{HED}``. A column name may hold -# any character but a brace (``{my.col}``); nested and unmatched braces are reported before this is applied. -COLUMN_REF_PATTERN = re.compile(r"\{([^{}]+)\}") +# any character but a brace, comma or parenthesis (``{my.col}``): a reference must parse as a single tag +# (specification 3.2.9.3, rule 8), so ``{a,b}`` is braces around HED, not a reference. Nested and unmatched braces +# are reported before this is applied. +COLUMN_REF_PATTERN = re.compile(r"\{([^{},()]+)\}") # The whitespace that may not begin or end a column name or a top-level sidecar key (specification 3.2.10): # blank, tab, line feed and carriage return only. A non-breaking space is not padding. diff --git a/hed/tools/analysis/annotation_util.py b/hed/tools/analysis/annotation_util.py index 99e603438..e828440f6 100644 --- a/hed/tools/analysis/annotation_util.py +++ b/hed/tools/analysis/annotation_util.py @@ -12,6 +12,9 @@ from hed.models.sidecar import Sidecar from hed.models.tabular_input import TabularInput +# A Definition tag, short or long form, at the start of a tag. Not "Def/MyDefinition/3" or "Description/a definition/b". +_DEFINITION_TAG_PATTERN = re.compile(r"(?:^|[,(]|organizational-property/)\s*definition/", re.IGNORECASE) + def check_df_columns(df, required_cols=("column_name", "column_value", "description", "HED")) -> list[str]: """Return a list of the specified columns that are missing from a dataframe. @@ -388,7 +391,7 @@ def _get_value_entry(hed_entry, description_entry, description_tag=True): tags = hed_entry if description_entry and description_entry != "n/a": value_dict["Description"] = description_entry - if "definition/" in tags.casefold(): + if _DEFINITION_TAG_PATTERN.search(tags): # An entry that holds definitions may hold nothing else (specification 3.2.9.1), so the description # is kept only in the Description or Levels text, not as a Description tag beside the definitions. pass diff --git a/tests/models/test_df_util.py b/tests/models/test_df_util.py index 02c3bd236..06c1408e6 100644 --- a/tests/models/test_df_util.py +++ b/tests/models/test_df_util.py @@ -10,6 +10,7 @@ convert_to_form, expand_defs, process_def_expands, + replace_ref, shrink_defs, split_delay_tags, ) @@ -17,6 +18,17 @@ hed_schema_global = load_schema_version("8.4.0") +class TestReplaceRef(unittest.TestCase): + def test_remove_ref_with_regex_characters(self): + # A column name may hold regex characters; only the named reference is removed. + self.assertEqual(replace_ref("{my.col}, {myXcol}", "{my.col}"), "{myXcol}") + self.assertEqual(replace_ref("Red, {col+1}", "{col+1}"), "Red") + self.assertEqual(replace_ref("(Red, {col+1}), Blue", "{col+1}"), "(Red), Blue") + + def test_replace_ref_with_value(self): + self.assertEqual(replace_ref("Red, {col+1}", "{col+1}", "Blue"), "Red, Blue") + + class TestShrinkDefs(unittest.TestCase): def setUp(self): self.schema = hed_schema_global diff --git a/tests/tools/analysis/test_annotation_util.py b/tests/tools/analysis/test_annotation_util.py index 3f9c4ab1d..748029d97 100644 --- a/tests/tools/analysis/test_annotation_util.py +++ b/tests/tools/analysis/test_annotation_util.py @@ -470,6 +470,16 @@ def test_get_value_entry(self): self.assertNotIn("HED", dict8, "_get_value_entry should not have a HED entry when Description not used") self.assertIn("Description", dict8, "_get_value_entry should have Description entry when Description not used") + def test_get_value_entry_definitions(self): + # An entry holding definitions may hold nothing else, so no Description tag is added beside them. + for tags in ["(Definition/MyDef, (Red))", "(Property/Organizational-property/Definition/MyDef, (Red))"]: + with self.subTest(tags=tags): + self.assertEqual(annotation_util._get_value_entry(tags, "A test")["HED"], tags) + # "definition/" inside a label or value is not a Definition tag. + for tags in ["Def/MyDefinition/3", "Description/See definition/example"]: + with self.subTest(tags=tags): + self.assertEqual(annotation_util._get_value_entry(tags, "A test")["HED"], tags + ", Description/A test") + def test_tag_list_to_str(self): ext1 = ["apple", "banana"] str1 = annotation_util._tag_list_to_str(ext1) diff --git a/tests/validator/test_sidecar_validator.py b/tests/validator/test_sidecar_validator.py index d490c9243..0078adcf4 100644 --- a/tests/validator/test_sidecar_validator.py +++ b/tests/validator/test_sidecar_validator.py @@ -282,6 +282,15 @@ def test_brace_name_with_period_known(self): self.assertEqual(self.codes(sidecar), []) self.assertEqual(Sidecar(io.StringIO(json.dumps(sidecar))).get_column_refs(), ["my.col"]) + def test_brace_name_with_delimiter_not_a_reference(self): + # A reference must parse as one tag (specification 3.2.9.3 rule 8), so a name holding a comma or a + # parenthesis is never a reference, even when such a column is annotated. + for name in ["a,b", "a(b)"]: + with self.subTest(name=name): + sidecar = {"cond": {"HED": {"a": "Red, {" + name + "}"}}, name: {"HED": {"b": "Blue"}}} + self.assertTrue(self.codes(sidecar)) + self.assertEqual(Sidecar(io.StringIO(json.dumps(sidecar))).get_column_refs(), []) + def test_every_categorical_substitution_checked(self): sidecar = {"event_code": {"HED": {"face": "(Red, {color})"}}, "color": {"HED": {"warm": "Red", "cool": "Blue"}}} self.assertEqual(self.codes(sidecar), ["TAG_EXPRESSION_REPEATED"]) From c8df3d414f1940d9f26f51e96a51942f62ec803b Mon Sep 17 00:00:00 2001 From: Kay Robbins Date: Tue, 10 Nov 2026 15:53:08 -0600 Subject: [PATCH 3/3] Address second review of PR 1501 - Detect Definition tags with a library prefix and only at a tag boundary, so df_to_hed keeps Description tags on values that merely mention a definition. - Leave "#" inside curly-brace references alone when filling a value placeholder, so {col#1} is still substituted. --- CHANGELOG.md | 2 +- hed/models/column_mapper.py | 11 ++++++++++- hed/tools/analysis/annotation_util.py | 7 +++++-- tests/tools/analysis/test_annotation_util.py | 13 +++++++++++-- tests/validator/test_sidecar_validator.py | 12 +++++++++++- 5 files changed, 38 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ce81cd69b..5ce655b38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ - **Errors that were missed are now reported** (hed-tests cases added in hed-tests #99): - Misnested parentheses with equal counts, such as `(Red)), ((Blue)` and `)(Red)(` (`PARENTHESES_MISMATCH`). - A `#` that is not the whole value of its tag, such as `Label/x#`, `Label/#x` or `Age/##`, in a sidecar or a definition (`PLACEHOLDER_INVALID`, specification Appendix B reason f). Units after one blank (`Frequency/# Hz`) are not part of the value. A definition containing `Age/##` now gets this code instead of `DEFINITION_INVALID`. - - A curly-brace reference whose name has characters other than letters, digits, hyphen and underscore, such as `{my.col}`: it was ignored, so an unknown column went unreported and a known one was never substituted (`SIDECAR_BRACES_INVALID` when unknown). A name with a comma or parenthesis is still not a reference, since a reference must parse as one tag. + - A curly-brace reference whose name has characters other than letters, digits, hyphen and underscore, such as `{my.col}`: it was ignored, so an unknown column went unreported and a known one was never substituted (`SIDECAR_BRACES_INVALID` when unknown). A name with a comma or parenthesis is still not a reference, since a reference must parse as one tag. A `#` in a referenced name (`{col#1}`) is no longer filled with the row value when the HED is assembled. - A definition beside other tags in a sidecar entry, categorical or value, such as `(Definition/MyDef, (Red)), Label/#` (`DEFINITION_INVALID`). An entry that holds only definitions is still valid. - A `HED` key below the second level of an entry that also has a valid one, such as in `Levels` (`SIDECAR_INVALID`). - Units in a `Def` value checked against the definition's placeholder tag with its units: `Def/Freq/3 kHz` with `(Definition/Freq/#, (Frequency/# Hz))` gives `Frequency/3 kHz Hz` (`UNITS_INVALID`). diff --git a/hed/models/column_mapper.py b/hed/models/column_mapper.py index 8589b32f3..2254aa86c 100644 --- a/hed/models/column_mapper.py +++ b/hed/models/column_mapper.py @@ -3,6 +3,7 @@ """ import copy +import re import warnings from collections import Counter from functools import partial @@ -12,6 +13,10 @@ from hed.models.column_metadata import ColumnMetadata, ColumnType from hed.models.column_source import is_blank_name from hed.models.definition_dict import DefinitionDict +from hed.models.model_constants import COLUMN_REF_PATTERN + +# A curly-brace reference, matched so it is kept whole, or a placeholder outside one. +_PLACEHOLDER_OUTSIDE_REFS = re.compile(COLUMN_REF_PATTERN.pattern + "|#") NO_WARN_COLUMNS = ["onset", "duration"] @@ -459,4 +464,8 @@ def _value_handler(value_str, x): if x == "n/a": return "n/a" - return value_str.replace("#", str(x)) + # A column name may hold "#" ({col#1}), so placeholders inside curly-brace references are left alone. + value = str(x) + return _PLACEHOLDER_OUTSIDE_REFS.sub( + lambda match: value if match.group(0) == "#" else match.group(0), value_str + ) diff --git a/hed/tools/analysis/annotation_util.py b/hed/tools/analysis/annotation_util.py index e828440f6..8dd1b3f73 100644 --- a/hed/tools/analysis/annotation_util.py +++ b/hed/tools/analysis/annotation_util.py @@ -12,8 +12,11 @@ from hed.models.sidecar import Sidecar from hed.models.tabular_input import TabularInput -# A Definition tag, short or long form, at the start of a tag. Not "Def/MyDefinition/3" or "Description/a definition/b". -_DEFINITION_TAG_PATTERN = re.compile(r"(?:^|[,(]|organizational-property/)\s*definition/", re.IGNORECASE) +# A Definition tag at the start of a tag: short, long or intermediate form, with an optional library prefix +# ("tl:Definition/MyDef"). Not "Def/MyDefinition/3" or "Description/See organizational-property/definition/x". +_DEFINITION_TAG_PATTERN = re.compile( + r"(?:^|[,(])\s*(?:[a-z]+:)?(?:(?:property/)?organizational-property/)?definition/", re.IGNORECASE +) def check_df_columns(df, required_cols=("column_name", "column_value", "description", "HED")) -> list[str]: diff --git a/tests/tools/analysis/test_annotation_util.py b/tests/tools/analysis/test_annotation_util.py index 748029d97..2ce3a887e 100644 --- a/tests/tools/analysis/test_annotation_util.py +++ b/tests/tools/analysis/test_annotation_util.py @@ -472,11 +472,20 @@ def test_get_value_entry(self): def test_get_value_entry_definitions(self): # An entry holding definitions may hold nothing else, so no Description tag is added beside them. - for tags in ["(Definition/MyDef, (Red))", "(Property/Organizational-property/Definition/MyDef, (Red))"]: + for tags in [ + "(Definition/MyDef, (Red))", + "(tl:Definition/MyDef, (tl:Red))", + "(Organizational-property/Definition/MyDef, (Red))", + "(Property/Organizational-property/Definition/MyDef, (Red))", + ]: with self.subTest(tags=tags): self.assertEqual(annotation_util._get_value_entry(tags, "A test")["HED"], tags) # "definition/" inside a label or value is not a Definition tag. - for tags in ["Def/MyDefinition/3", "Description/See definition/example"]: + for tags in [ + "Def/MyDefinition/3", + "Description/See definition/example", + "Description/See organizational-property/definition/example", + ]: with self.subTest(tags=tags): self.assertEqual(annotation_util._get_value_entry(tags, "A test")["HED"], tags + ", Description/A test") diff --git a/tests/validator/test_sidecar_validator.py b/tests/validator/test_sidecar_validator.py index 0078adcf4..fd383fdaf 100644 --- a/tests/validator/test_sidecar_validator.py +++ b/tests/validator/test_sidecar_validator.py @@ -6,7 +6,7 @@ from hed import schema from hed.errors.error_reporter import ErrorHandler from hed.errors.error_types import ErrorContext, SidecarErrors -from hed.models import Sidecar +from hed.models import Sidecar, TabularInput from hed.validator.sidecar_validator import SidecarValidator @@ -282,6 +282,16 @@ def test_brace_name_with_period_known(self): self.assertEqual(self.codes(sidecar), []) self.assertEqual(Sidecar(io.StringIO(json.dumps(sidecar))).get_column_refs(), ["my.col"]) + def test_brace_name_with_hash_assembled(self): + # The value fills the template's placeholder only, not a "#" in a referenced column name. + sidecar = Sidecar( + io.StringIO(json.dumps({"val": {"HED": "Label/#, {col#1}"}, "col#1": {"HED": {"x": "Blue"}}})) + ) + tsv = io.StringIO("onset\tval\tcol#1\n1.0\t3\tx\n") + tabular = TabularInput(tsv, sidecar=sidecar) + self.assertEqual(tabular.validate(self.hed_schema), []) + self.assertEqual([str(hed) for hed in tabular.series_a], ["Label/3, Blue"]) + def test_brace_name_with_delimiter_not_a_reference(self): # A reference must parse as one tag (specification 3.2.9.3 rule 8), so a name holding a comma or a # parenthesis is never a reference, even when such a column is annotated.