diff --git a/CHANGELOG.md b/CHANGELOG.md index e1c0ea66e..ebf0fb1c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Unreleased + +## Validation + +- **A tag extension may contain only `name` characters** (letters, digits, hyphen, underscore, and from 8.3.0 non-ASCII), as specification 3.2.5 rule 2 requires. Period, plus, caret, blank, and colon were accepted before (`Item/my.ext`, `Item/a b`) and are now errors. Any character outside `name` in an extension, including ones that were already errors such as `$`, is now reported as `TAG_EXTENSION_INVALID` (Appendix B, reason b) instead of `CHARACTER_INVALID`, naming the character and its position. `#` stays `PLACEHOLDER_INVALID`. ([#1471](https://github.com/hed-standard/hed-python/issues/1471)) + # Release 1.3.0 October 23, 2026 This release implements HED specification 4.0.0 (reserved tags, column names and padded values, tag-extension rules, value-text and character sets, BIDS date-times, compound units) and bundles HED 8.5.0. Many annotations that passed under 1.2.0 now give errors, and tabular validation reports issues differently: read the upgrade notes first. Runtime dependencies and supported Python versions (3.10 and later) are unchanged. diff --git a/hed/validator/util/char_util.py b/hed/validator/util/char_util.py index afc43a47a..4551110c2 100644 --- a/hed/validator/util/char_util.py +++ b/hed/validator/util/char_util.py @@ -9,21 +9,10 @@ from hed.schema.schema_io.schema_util import schema_version_greater_equal from hed.validator.util.character_sets import CharacterSets, code_point -# The character sets a tag extension may use (specification 3.2.5 says ``name`` characters; hedtools has always -# also admitted the characters below, which appear in extensions of existing datasets). -EXTENSION_SETS = ( - "alphanumeric", - "hyphen", - "underscore", - "forward-slash", - "period", - "plus", - "caret", - "blank", - "number-sign", - "colon", - "nonascii", -) +# The character sets a tag extension may use: ``name`` characters (specification 3.2.5 rule 2) and the forward +# slash that separates extension terms. Any other character is TAG_EXTENSION_INVALID (Appendix B, reason b), +# except ``#``, which TagValidator.check_for_placeholder reports as PLACEHOLDER_INVALID. +EXTENSION_SETS = ("name", "forward-slash") # The character sets of the schema path of a tag (the part matched to schema nodes). TAG_PATH_SETS = ("alphanumeric", "hyphen", "underscore", "forward-slash", "colon", "nonascii") # The value class used for a value whose placeholder declares no valueClass (decision 2026-09-30, D3). @@ -131,7 +120,7 @@ def check_for_invalid_extension_chars( Parameters: original_tag (HedTag): The original tag that is used to report the error. validate_text (str): the text we want to validate, if not the full extension. - error_code (str): The code to override the error as. Again mostly for def/def-expand tags. + error_code (str): The code to report instead of TAG_EXTENSION_INVALID (Appendix B, reason b). index_offset (int): Offset into the extension validate_text starts at. Returns: @@ -142,12 +131,13 @@ def check_for_invalid_extension_chars( list(EXTENSION_SETS), original_tag, starting_index=len(original_tag.org_base_tag) + 1 + index_offset, - error_code=error_code, + error_code=error_code or ValidationErrors.TAG_EXTENSION_INVALID, subject="a tag extension", + skip_chars="#", ) def _check_invalid_chars( - self, check_string, allowed_names, source_tag, starting_index=0, error_code=None, subject="a tag" + self, check_string, allowed_names, source_tag, starting_index=0, error_code=None, subject="a tag", skip_chars="" ): """Helper for checking for invalid characters. @@ -158,6 +148,7 @@ def _check_invalid_chars( starting_index (int): Starting index of check_string within the tag. error_code (str): The code to override the error as. Again mostly for def/def-expand tags. subject (str): What is being checked, for the message ("a tag extension"). + skip_chars (str): Characters outside allowed_names that another check reports, so not reported here. Returns: list: List of dictionaries with validation issues. @@ -165,6 +156,8 @@ def _check_invalid_chars( validation_issues = [] allows = f"{subject} allows {self._sets.describe(allowed_names)}" for i, character in self._sets.problem_characters(check_string, allowed_names): + if character in skip_chars: + continue validation_issues += ErrorHandler.format_error( ValidationErrors.INVALID_TAG_CHARACTER, tag=source_tag, diff --git a/spec_tests/hed-tests b/spec_tests/hed-tests index 1fa5c1a9d..568ac2b92 160000 --- a/spec_tests/hed-tests +++ b/spec_tests/hed-tests @@ -1 +1 @@ -Subproject commit 1fa5c1a9dd172e7352bfc0c02ee898339b0ce84f +Subproject commit 568ac2b92b8b2e38dc9528218b9ec922b0c67fd1 diff --git a/tests/data/sidecar_tests/both_types_events_with_defs.json b/tests/data/sidecar_tests/both_types_events_with_defs.json index 63e8eb758..4abd3d3fa 100644 --- a/tests/data/sidecar_tests/both_types_events_with_defs.json +++ b/tests/data/sidecar_tests/both_types_events_with_defs.json @@ -15,7 +15,7 @@ "LongName": "Response time after stimulus", "Description": "Time from stimulus presentation until subject presses button", "Units": "ms", - "HED": "Def/JsonFileDef/#, Action/Button press" + "HED": "Def/JsonFileDef/#, Action/Button-press" }, "stim_file": { "LongName": "Stimulus file name", diff --git a/tests/data/sidecar_tests/both_types_events_without_definitions.json b/tests/data/sidecar_tests/both_types_events_without_definitions.json index 6129f7405..99e75ad73 100644 --- a/tests/data/sidecar_tests/both_types_events_without_definitions.json +++ b/tests/data/sidecar_tests/both_types_events_without_definitions.json @@ -15,7 +15,7 @@ "LongName": "Response time after stimulus", "Description": "Time from stimulus presentation until subject presses button", "Units": "ms", - "HED": "Def/JsonFileDef/#, Action/Button press" + "HED": "Def/JsonFileDef/#, Action/Button-press" }, "stim_file": { "LongName": "Stimulus file name", diff --git a/tests/validator/test_staged_validation.py b/tests/validator/test_staged_validation.py index fd9d31c56..7390ce492 100644 --- a/tests/validator/test_staged_validation.py +++ b/tests/validator/test_staged_validation.py @@ -577,7 +577,8 @@ def test_prefix_template_with_only_a_warning_still_checks_the_values(self): template_issues = [issue["code"] for issue in issues if ErrorContext.ROW not in issue] self.assertEqual(template_issues, [ValidationErrors.TAG_EXTENDED]) self.assertIn( - (ValidationErrors.CHARACTER_INVALID, 2), [(issue["code"], issue.get(ErrorContext.ROW)) for issue in issues] + (ValidationErrors.TAG_EXTENSION_INVALID, 2), + [(issue["code"], issue.get(ErrorContext.ROW)) for issue in issues], ) diff --git a/tests/validator/test_tag_validator.py b/tests/validator/test_tag_validator.py index 1646f530c..c7410d385 100644 --- a/tests/validator/test_tag_validator.py +++ b/tests/validator/test_tag_validator.py @@ -121,7 +121,8 @@ def test_required_units(self): "notRequiredNumber": "Age/0.5", "notRequiredScientific": "Age/5.2e-1", # Clock-face takes no value before 8.3.0 (this score library is partnered with 8.2.0), so 08:30 is an - # extension term, and one with no letter (specification 3.2.5 rule 6). + # extension term with no letter (specification 3.2.5 rule 6) and a colon, which is not a name character + # (rule 2). "timeValue": "Clock-face/08:30", "invalidTimeValue": "Clock-face/8:30", } @@ -143,9 +144,31 @@ def test_required_units(self): "notRequiredScientific": [], "timeValue": self.format_error( ValidationErrors.EXTENSION_TERM_NO_LETTER, tag=0, index_in_tag=11, index_in_tag_end=16 + ) + + self.format_error( + ValidationErrors.INVALID_TAG_CHARACTER, + tag=0, + index_in_tag=13, + index_in_tag_end=14, + actual_error=ValidationErrors.TAG_EXTENSION_INVALID, + char_index=13, + char_set="name, forward-slash", + code_point="U+003A", + allows="a tag extension allows name, forward-slash", ), "invalidTimeValue": self.format_error( ValidationErrors.EXTENSION_TERM_NO_LETTER, tag=0, index_in_tag=11, index_in_tag_end=15 + ) + + self.format_error( + ValidationErrors.INVALID_TAG_CHARACTER, + tag=0, + index_in_tag=12, + index_in_tag_end=13, + actual_error=ValidationErrors.TAG_EXTENSION_INVALID, + char_index=12, + char_set="name, forward-slash", + code_point="U+003A", + allows="a tag extension allows name, forward-slash", ), } self.validator_semantic(test_strings, expected_results, expected_issues, True) @@ -360,6 +383,50 @@ def test_extension_term_without_letter(self): } self.validator_semantic(test_strings, expected_results, expected_issues, False) + def test_extension_characters_outside_name(self): + # Specification 3.2.5 rule 2: an extension term contains only name characters; any other character is + # TAG_EXTENSION_INVALID (Appendix B, reason b), reported at the character (hed-python issue #1471). + test_strings = { + "period": "Item/my.ext", + "plus": "Item/a+b", + "caret": "Item/a^b", + "blank": "Item/a b", + "colon": "Item/a:b", + "dollar": "Item/a$b", + "secondTerm": "Item/Blech/a.b", + "twoBad": "Item/a.b+c", + "nameCharacters": "Item/My-thing_2/Blech", + } + bad = { + "period": [(7, ".", "U+002E")], + "plus": [(6, "+", "U+002B")], + "caret": [(6, "^", "U+005E")], + "blank": [(6, " ", "U+0020")], + "colon": [(6, ":", "U+003A")], + "dollar": [(6, "$", "U+0024")], + "secondTerm": [(12, ".", "U+002E")], + "twoBad": [(6, ".", "U+002E"), (8, "+", "U+002B")], + "nameCharacters": [], + } + expected_results = {key: not positions for key, positions in bad.items()} + expected_issues = {} + for key, positions in bad.items(): + expected_issues[key] = [] + for index, character, point in positions: + self.assertEqual(test_strings[key][index], character) + expected_issues[key] += self.format_error( + ValidationErrors.INVALID_TAG_CHARACTER, + tag=0, + index_in_tag=index, + index_in_tag_end=index + 1, + actual_error=ValidationErrors.TAG_EXTENSION_INVALID, + char_index=index, + char_set="name, forward-slash", + code_point=point, + allows="a tag extension allows name, forward-slash", + ) + self.validator_semantic(test_strings, expected_results, expected_issues, False) + def test_extension_warning(self): test_strings = {"noWarning": "Condition-variable/ValidExt", "warning": "Task-property/WarningExt"} expected_results = { diff --git a/tests/validator/test_tag_validator_library.py b/tests/validator/test_tag_validator_library.py index 9a0714372..bbddcea38 100644 --- a/tests/validator/test_tag_validator_library.py +++ b/tests/validator/test_tag_validator_library.py @@ -147,7 +147,7 @@ def test_required_units(self): "notRequiredNumber": "Age/0.5", "notRequiredScientific": "Age/5.2e-1", # Clock-face takes no value before 8.3.0 (these schemas are 8.0.0-based), so 08:30 is an extension - # term, and one with no letter (specification 3.2.5 rule 6). + # term with no letter (specification 3.2.5 rule 6) and a colon, which is not a name character (rule 2). "timeValue": "Clock-face/08:30", "invalidTimeValue": "Clock-face/8:30", } @@ -169,9 +169,31 @@ def test_required_units(self): "notRequiredScientific": [], "timeValue": self.format_error( ValidationErrors.EXTENSION_TERM_NO_LETTER, tag=0, index_in_tag=11, index_in_tag_end=16 + ) + + self.format_error( + ValidationErrors.INVALID_TAG_CHARACTER, + tag=0, + index_in_tag=13, + index_in_tag_end=14, + actual_error=ValidationErrors.TAG_EXTENSION_INVALID, + char_index=13, + char_set="name, forward-slash", + code_point="U+003A", + allows="a tag extension allows name, forward-slash", ), "invalidTimeValue": self.format_error( ValidationErrors.EXTENSION_TERM_NO_LETTER, tag=0, index_in_tag=11, index_in_tag_end=15 + ) + + self.format_error( + ValidationErrors.INVALID_TAG_CHARACTER, + tag=0, + index_in_tag=12, + index_in_tag_end=13, + actual_error=ValidationErrors.TAG_EXTENSION_INVALID, + char_index=12, + char_set="name, forward-slash", + code_point="U+003A", + allows="a tag extension allows name, forward-slash", ), } self.validator_semantic(test_strings, expected_results, expected_issues, True)