Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
# Unreleased

## Validation

- **A tag extension may contain only `name` characters** (letters, digits, hyphen, underscore, and from 8.3.0 non-ASCII), as specification 3.2.5 rule 2 requires. Period, plus, caret, blank, and colon were accepted before (`Item/my.ext`, `Item/a b`) and are now errors. Any character outside `name` in an extension, including ones that were already errors such as `$`, is now reported as `TAG_EXTENSION_INVALID` (Appendix B, reason b) instead of `CHARACTER_INVALID`, naming the character and its position. `#` stays `PLACEHOLDER_INVALID`. ([#1471](https://github.com/hed-standard/hed-python/issues/1471))

# Release 1.3.0 October 23, 2026

This release implements HED specification 4.0.0 (reserved tags, column names and padded values, tag-extension rules, value-text and character sets, BIDS date-times, compound units) and bundles HED 8.5.0. Many annotations that passed under 1.2.0 now give errors, and tabular validation reports issues differently: read the upgrade notes first. Runtime dependencies and supported Python versions (3.10 and later) are unchanged.
Expand Down
29 changes: 11 additions & 18 deletions hed/validator/util/char_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,21 +9,10 @@
from hed.schema.schema_io.schema_util import schema_version_greater_equal
from hed.validator.util.character_sets import CharacterSets, code_point

# The character sets a tag extension may use (specification 3.2.5 says ``name`` characters; hedtools has always
# also admitted the characters below, which appear in extensions of existing datasets).
EXTENSION_SETS = (
"alphanumeric",
"hyphen",
"underscore",
"forward-slash",
"period",
"plus",
"caret",
"blank",
"number-sign",
"colon",
"nonascii",
)
# The character sets a tag extension may use: ``name`` characters (specification 3.2.5 rule 2) and the forward
# slash that separates extension terms. Any other character is TAG_EXTENSION_INVALID (Appendix B, reason b),
# except ``#``, which TagValidator.check_for_placeholder reports as PLACEHOLDER_INVALID.
EXTENSION_SETS = ("name", "forward-slash")
# The character sets of the schema path of a tag (the part matched to schema nodes).
TAG_PATH_SETS = ("alphanumeric", "hyphen", "underscore", "forward-slash", "colon", "nonascii")
# The value class used for a value whose placeholder declares no valueClass (decision 2026-09-30, D3).
Expand Down Expand Up @@ -131,7 +120,7 @@ def check_for_invalid_extension_chars(
Parameters:
original_tag (HedTag): The original tag that is used to report the error.
validate_text (str): the text we want to validate, if not the full extension.
error_code (str): The code to override the error as. Again mostly for def/def-expand tags.
error_code (str): The code to report instead of TAG_EXTENSION_INVALID (Appendix B, reason b).
index_offset (int): Offset into the extension validate_text starts at.

Returns:
Expand All @@ -142,12 +131,13 @@ def check_for_invalid_extension_chars(
list(EXTENSION_SETS),
original_tag,
starting_index=len(original_tag.org_base_tag) + 1 + index_offset,
error_code=error_code,
error_code=error_code or ValidationErrors.TAG_EXTENSION_INVALID,
subject="a tag extension",
skip_chars="#",
)

def _check_invalid_chars(
self, check_string, allowed_names, source_tag, starting_index=0, error_code=None, subject="a tag"
self, check_string, allowed_names, source_tag, starting_index=0, error_code=None, subject="a tag", skip_chars=""
):
"""Helper for checking for invalid characters.

Expand All @@ -158,13 +148,16 @@ def _check_invalid_chars(
starting_index (int): Starting index of check_string within the tag.
error_code (str): The code to override the error as. Again mostly for def/def-expand tags.
subject (str): What is being checked, for the message ("a tag extension").
skip_chars (str): Characters outside allowed_names that another check reports, so not reported here.

Returns:
list: List of dictionaries with validation issues.
"""
validation_issues = []
allows = f"{subject} allows {self._sets.describe(allowed_names)}"
for i, character in self._sets.problem_characters(check_string, allowed_names):
if character in skip_chars:
continue
validation_issues += ErrorHandler.format_error(
ValidationErrors.INVALID_TAG_CHARACTER,
tag=source_tag,
Expand Down
2 changes: 1 addition & 1 deletion tests/data/sidecar_tests/both_types_events_with_defs.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"LongName": "Response time after stimulus",
"Description": "Time from stimulus presentation until subject presses button",
"Units": "ms",
"HED": "Def/JsonFileDef/#, Action/Button press"
"HED": "Def/JsonFileDef/#, Action/Button-press"
},
"stim_file": {
"LongName": "Stimulus file name",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"LongName": "Response time after stimulus",
"Description": "Time from stimulus presentation until subject presses button",
"Units": "ms",
"HED": "Def/JsonFileDef/#, Action/Button press"
"HED": "Def/JsonFileDef/#, Action/Button-press"
},
"stim_file": {
"LongName": "Stimulus file name",
Expand Down
3 changes: 2 additions & 1 deletion tests/validator/test_staged_validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,8 @@ def test_prefix_template_with_only_a_warning_still_checks_the_values(self):
template_issues = [issue["code"] for issue in issues if ErrorContext.ROW not in issue]
self.assertEqual(template_issues, [ValidationErrors.TAG_EXTENDED])
self.assertIn(
(ValidationErrors.CHARACTER_INVALID, 2), [(issue["code"], issue.get(ErrorContext.ROW)) for issue in issues]
(ValidationErrors.TAG_EXTENSION_INVALID, 2),
[(issue["code"], issue.get(ErrorContext.ROW)) for issue in issues],
)


Expand Down
69 changes: 68 additions & 1 deletion tests/validator/test_tag_validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,8 @@ def test_required_units(self):
"notRequiredNumber": "Age/0.5",
"notRequiredScientific": "Age/5.2e-1",
# Clock-face takes no value before 8.3.0 (this score library is partnered with 8.2.0), so 08:30 is an
# extension term, and one with no letter (specification 3.2.5 rule 6).
# extension term with no letter (specification 3.2.5 rule 6) and a colon, which is not a name character
# (rule 2).
"timeValue": "Clock-face/08:30",
"invalidTimeValue": "Clock-face/8:30",
}
Expand All @@ -143,9 +144,31 @@ def test_required_units(self):
"notRequiredScientific": [],
"timeValue": self.format_error(
ValidationErrors.EXTENSION_TERM_NO_LETTER, tag=0, index_in_tag=11, index_in_tag_end=16
)
+ self.format_error(
ValidationErrors.INVALID_TAG_CHARACTER,
tag=0,
index_in_tag=13,
index_in_tag_end=14,
actual_error=ValidationErrors.TAG_EXTENSION_INVALID,
char_index=13,
char_set="name, forward-slash",
code_point="U+003A",
allows="a tag extension allows name, forward-slash",
),
"invalidTimeValue": self.format_error(
ValidationErrors.EXTENSION_TERM_NO_LETTER, tag=0, index_in_tag=11, index_in_tag_end=15
)
+ self.format_error(
ValidationErrors.INVALID_TAG_CHARACTER,
tag=0,
index_in_tag=12,
index_in_tag_end=13,
actual_error=ValidationErrors.TAG_EXTENSION_INVALID,
char_index=12,
char_set="name, forward-slash",
code_point="U+003A",
allows="a tag extension allows name, forward-slash",
),
}
self.validator_semantic(test_strings, expected_results, expected_issues, True)
Expand Down Expand Up @@ -360,6 +383,50 @@ def test_extension_term_without_letter(self):
}
self.validator_semantic(test_strings, expected_results, expected_issues, False)

def test_extension_characters_outside_name(self):
# Specification 3.2.5 rule 2: an extension term contains only name characters; any other character is
# TAG_EXTENSION_INVALID (Appendix B, reason b), reported at the character (hed-python issue #1471).
test_strings = {
"period": "Item/my.ext",
"plus": "Item/a+b",
"caret": "Item/a^b",
"blank": "Item/a b",
"colon": "Item/a:b",
"dollar": "Item/a$b",
"secondTerm": "Item/Blech/a.b",
"twoBad": "Item/a.b+c",
"nameCharacters": "Item/My-thing_2/Blech",
}
bad = {
"period": [(7, ".", "U+002E")],
"plus": [(6, "+", "U+002B")],
"caret": [(6, "^", "U+005E")],
"blank": [(6, " ", "U+0020")],
"colon": [(6, ":", "U+003A")],
"dollar": [(6, "$", "U+0024")],
"secondTerm": [(12, ".", "U+002E")],
"twoBad": [(6, ".", "U+002E"), (8, "+", "U+002B")],
"nameCharacters": [],
}
expected_results = {key: not positions for key, positions in bad.items()}
expected_issues = {}
for key, positions in bad.items():
expected_issues[key] = []
for index, character, point in positions:
self.assertEqual(test_strings[key][index], character)
expected_issues[key] += self.format_error(
ValidationErrors.INVALID_TAG_CHARACTER,
tag=0,
index_in_tag=index,
index_in_tag_end=index + 1,
actual_error=ValidationErrors.TAG_EXTENSION_INVALID,
char_index=index,
char_set="name, forward-slash",
code_point=point,
allows="a tag extension allows name, forward-slash",
)
self.validator_semantic(test_strings, expected_results, expected_issues, False)

def test_extension_warning(self):
test_strings = {"noWarning": "Condition-variable/ValidExt", "warning": "Task-property/WarningExt"}
expected_results = {
Expand Down
24 changes: 23 additions & 1 deletion tests/validator/test_tag_validator_library.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def test_required_units(self):
"notRequiredNumber": "Age/0.5",
"notRequiredScientific": "Age/5.2e-1",
# Clock-face takes no value before 8.3.0 (these schemas are 8.0.0-based), so 08:30 is an extension
# term, and one with no letter (specification 3.2.5 rule 6).
# term with no letter (specification 3.2.5 rule 6) and a colon, which is not a name character (rule 2).
"timeValue": "Clock-face/08:30",
"invalidTimeValue": "Clock-face/8:30",
}
Expand All @@ -169,9 +169,31 @@ def test_required_units(self):
"notRequiredScientific": [],
"timeValue": self.format_error(
ValidationErrors.EXTENSION_TERM_NO_LETTER, tag=0, index_in_tag=11, index_in_tag_end=16
)
+ self.format_error(
ValidationErrors.INVALID_TAG_CHARACTER,
tag=0,
index_in_tag=13,
index_in_tag_end=14,
actual_error=ValidationErrors.TAG_EXTENSION_INVALID,
char_index=13,
char_set="name, forward-slash",
code_point="U+003A",
allows="a tag extension allows name, forward-slash",
),
"invalidTimeValue": self.format_error(
ValidationErrors.EXTENSION_TERM_NO_LETTER, tag=0, index_in_tag=11, index_in_tag_end=15
)
+ self.format_error(
ValidationErrors.INVALID_TAG_CHARACTER,
tag=0,
index_in_tag=12,
index_in_tag_end=13,
actual_error=ValidationErrors.TAG_EXTENSION_INVALID,
char_index=12,
char_set="name, forward-slash",
code_point="U+003A",
allows="a tag extension allows name, forward-slash",
),
}
self.validator_semantic(test_strings, expected_results, expected_issues, True)
Expand Down
Loading