diff --git a/CHANGELOG.md b/CHANGELOG.md index ebf0fb1c..3f8ce22f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ - **A tag extension may contain only `name` characters** (letters, digits, hyphen, underscore, and from 8.3.0 non-ASCII), as specification 3.2.5 rule 2 requires. Period, plus, caret, blank, and colon were accepted before (`Item/my.ext`, `Item/a b`) and are now errors. Any character outside `name` in an extension, including ones that were already errors such as `$`, is now reported as `TAG_EXTENSION_INVALID` (Appendix B, reason b) instead of `CHARACTER_INVALID`, naming the character and its position. `#` stays `PLACEHOLDER_INVALID`. ([#1471](https://github.com/hed-standard/hed-python/issues/1471)) +## Specification files + +- **The packaged `reserved_tags.json` carries `specificationVersion` and `fileVersion`** (hed-specification #731), refreshed verbatim from hed-specification. Both specification files, `reserved_tags.json` and `character_sets.json`, are now refused with `HedFileError` `INVALID_FILE_FORMAT` when their file version is missing or its major number is not 1; any minor and patch number is accepted (specification Appendix A.6.1). The check lives in the new internal module `hed/validator/util/spec_files.py`. ([#1489](https://github.com/hed-standard/hed-python/issues/1489)) + # Release 1.3.0 October 23, 2026 This release implements HED specification 4.0.0 (reserved tags, column names and padded values, tag-extension rules, value-text and character sets, BIDS date-times, compound units) and bundles HED 8.5.0. Many annotations that passed under 1.2.0 now give errors, and tabular validation reports issues differently: read the upgrade notes first. Runtime dependencies and supported Python versions (3.10 and later) are unchanged. diff --git a/hed/validator/data/reserved_tags.json b/hed/validator/data/reserved_tags.json index ef959c62..5b30c0e8 100644 --- a/hed/validator/data/reserved_tags.json +++ b/hed/validator/data/reserved_tags.json @@ -1,6 +1,8 @@ { "title": "HED reserved tags", "description": "Machine-readable form of the validation rules of the structural reserved tags, HED specification section 3.2.8 (Reserved tags). The tables in docs/source/03_HED_formats.md are generated from this file by scripts/generate_reserved_table.py, and Appendix A.6 of the specification defines every field. Validators load this file unchanged. Each row of tags gives the group rules of one tag; a Def tag or the Def-expand group that is its expanded form is the anchor these rows refer to and is never counted as a companion tag or an inner group. Every row applies in every standard schema version >= 8.0.0; requireValue, tagGroup and topLevelTagGroup supply what the 8.0.0 to 8.2.0 schemas lack on Delay and Duration. A row with secondaryWith is secondary in a group that also holds one of the listed tags: that tag's row gives the group rules (requiresDef, otherAllowedNonDefTags, minNonDefSubgroups, maxNonDefSubgroups, allowSpliceInGroup), the secondary row's group rules apply only when its tag is the only one of the table in its group, and its other fields apply wherever its tag appears. valueForms lists the tags whose value has two parts, name or name/value.", + "specificationVersion": "4.0.0", + "fileVersion": "1.0.0", "tags": { "Definition": { "name": "Definition", diff --git a/hed/validator/reserved_checker.py b/hed/validator/reserved_checker.py index 2dc6fded..27975910 100644 --- a/hed/validator/reserved_checker.py +++ b/hed/validator/reserved_checker.py @@ -5,6 +5,7 @@ drifts from hed-specification main. Every row of the file applies in every standard schema version >= 8.0.0 (hed-specification decision 2026-10-15): its ``requireValue``, ``tagGroup`` and ``topLevelTagGroup`` fields supply what the 8.0.0 to 8.2.0 schemas lack on ``Delay`` and ``Duration``, so no check here takes a version. +The loader refuses a file whose file version major number it does not support (specification Appendix A.6.1). """ import json @@ -14,6 +15,25 @@ from hed.errors.error_reporter import ErrorHandler from hed.errors.error_types import TemporalErrors, ValidationErrors +from hed.validator.util.spec_files import check_file_version + + +def load_reserved_file(path) -> dict: + """Read a reserved_tags.json file, refusing a file version this hedtools does not support. + + Parameters: + path (str): The path of a ``reserved_tags.json`` file. + + Returns: + dict: The parsed content of the file. + + Raises: + HedFileError: INVALID_FILE_FORMAT if the file's ``fileVersion`` is missing or has an unsupported major number. + """ + with open(path, encoding="utf-8") as file: + data = json.load(file) + check_file_version(data, "reserved_tags.json") + return data class ReservedChecker: @@ -52,8 +72,7 @@ def __new__(cls): def _initialize(self): # Load the JSON file during the first instantiation if not hasattr(self, "reserved_map"): - with open(self.reserved_reqs_path, encoding="utf-8") as file: - data = json.load(file) + data = load_reserved_file(self.reserved_reqs_path) self.reserved_map = data["tags"] self.value_forms = data["valueForms"] self._initialize_special_tags() diff --git a/hed/validator/util/character_sets.py b/hed/validator/util/character_sets.py index 900dff85..c265a14a 100644 --- a/hed/validator/util/character_sets.py +++ b/hed/validator/util/character_sets.py @@ -4,6 +4,7 @@ hed-specification (``docs/source/_static/character_sets.json``). Both validators load the same file, so the named sets, the per-class defaults, the whole-value rules and the string-level exclusions cannot drift between them or from the specification text, which is generated from the file. Update the copy; never edit it. +The loader refuses a file whose file version major number it does not support (specification Appendix A.6.1). """ import json @@ -12,6 +13,8 @@ from semantic_version import Version +from hed.validator.util.spec_files import check_file_version + CHARACTER_SETS_FILENAME = "../data/character_sets.json" @@ -33,7 +36,12 @@ def __init__(self, data): Parameters: data (dict): The parsed content of a ``character_sets.json`` file. + + Raises: + HedFileError: INVALID_FILE_FORMAT if the file's ``file_version`` is missing or has an unsupported major + number. """ + check_file_version(data, "character_sets.json") self._data = data self.sets = data["sets"] self.aliases = data.get("aliases", {}) diff --git a/hed/validator/util/spec_files.py b/hed/validator/util/spec_files.py new file mode 100644 index 00000000..48591883 --- /dev/null +++ b/hed/validator/util/spec_files.py @@ -0,0 +1,48 @@ +"""File-version gate for the JSON files that hedtools copies verbatim from the HED specification. + +Specification Appendix A.6.1: each file carries a specification version and a file version. A validator MUST NOT +apply a file whose file version major number it does not support, and it accepts any minor and patch number under +a major number it supports. The two files spell the key differently, each in the style of its other keys. +""" + +import re + +from hed.errors.exceptions import HedExceptions, HedFileError + +# For each specification file: the key that holds its file version and the major number this hedtools supports. +SUPPORTED_FILE_MAJOR = { + "reserved_tags.json": ("fileVersion", 1), + "character_sets.json": ("file_version", 1), +} + +# ASCII digits only, matched against the whole string (fullmatch, so a trailing newline does not pass). +_VERSION_PATTERN = re.compile(r"([0-9]+)\.([0-9]+)\.([0-9]+)") + + +def check_file_version(data, filename) -> None: + """Raise unless the file version of a specification file has a major number this hedtools supports. + + Parameters: + data (dict): The parsed content of the file. + filename (str): The file's name, a key of SUPPORTED_FILE_MAJOR (``"reserved_tags.json"``). + + Raises: + HedFileError: INVALID_FILE_FORMAT if the version key is missing, is not a ``major.minor.patch`` string, + or has a major number other than the supported one. + """ + key, supported_major = SUPPORTED_FILE_MAJOR[filename] + version = data.get(key) if isinstance(data, dict) else None + match = _VERSION_PATTERN.fullmatch(version) if isinstance(version, str) else None + if match is None: + raise HedFileError( + HedExceptions.INVALID_FILE_FORMAT, + f"{filename} has no valid '{key}' (expected major.minor.patch, found {version!r}).", + filename, + ) + if int(match.group(1)) != supported_major: + raise HedFileError( + HedExceptions.INVALID_FILE_FORMAT, + f"{filename} has {key} {version}; this version of hedtools supports only file version " + f"{supported_major}.x.x of this file (specification Appendix A.6.1).", + filename, + ) diff --git a/tests/validator/test_character_sets.py b/tests/validator/test_character_sets.py index 10c3a1ee..d9623acd 100644 --- a/tests/validator/test_character_sets.py +++ b/tests/validator/test_character_sets.py @@ -5,6 +5,7 @@ import re import unittest +from hed.errors.exceptions import HedExceptions, HedFileError from hed.validator.util.character_sets import CHARACTER_SETS_FILENAME, CharacterSets, code_point @@ -22,6 +23,8 @@ def setUpClass(cls): def test_the_file_has_every_section_the_validators_read(self): for key in ( + "specification_version", + "file_version", "sets", "aliases", "literal_names", @@ -34,6 +37,17 @@ def test_the_file_has_every_section_the_validators_read(self): self.assertEqual(self.raw["declaration_wins_from"]["standard_version"], "8.5.0") self.assertEqual(self.sets.declaration_wins_from, "8.5.0") + def test_a_copy_with_an_unsupported_file_version_is_refused(self): + for version, accepted in (("1.2.3", True), ("1.0.0", True), ("2.0.0", False), ("0.9.9", False)): + with self.subTest(version=version): + data = {**self.raw, "file_version": version} + if accepted: + self.assertEqual(CharacterSets(data).sets, self.sets.sets) + else: + with self.assertRaises(HedFileError) as context: + CharacterSets(data) + self.assertEqual(context.exception.code, HedExceptions.INVALID_FILE_FORMAT) + def test_every_set_regex_compiles_and_single_character_sets_match_their_code(self): for name, entry in self.sets.sets.items(): with self.subTest(name=name): diff --git a/tests/validator/test_reserved_checker.py b/tests/validator/test_reserved_checker.py index 89c76606..29c50d05 100644 --- a/tests/validator/test_reserved_checker.py +++ b/tests/validator/test_reserved_checker.py @@ -3,13 +3,16 @@ import io import json import os +import re +import tempfile import unittest from hed import HedString, Sidecar, TabularInput, load_schema_version, schema from hed.errors import ErrorSeverity, ValidationErrors +from hed.errors.exceptions import HedExceptions, HedFileError from hed.models import DefinitionDict from hed.validator import HedValidator -from hed.validator.reserved_checker import ReservedChecker +from hed.validator.reserved_checker import ReservedChecker, load_reserved_file from hed.validator.spreadsheet_validator import SpreadsheetValidator DEFINITIONS = ["(Definition/MyDef, (Red))", "(Definition/Other, (Blue))"] @@ -54,7 +57,7 @@ def timeline_codes(self, *hed_cells): class TestReservedCheckerData(unittest.TestCase): - """The loader reads the published file's two top-level keys.""" + """The loader reads the published file's tags and valueForms, after checking its fileVersion.""" def test_file_is_the_published_reserved_tags_json(self): checker = ReservedChecker.get_instance() @@ -64,6 +67,27 @@ def test_file_is_the_published_reserved_tags_json(self): self.assertEqual(checker.reserved_map, data["tags"]) self.assertEqual(checker.value_forms, data["valueForms"]) + def test_the_file_carries_its_version_fields(self): + data = load_reserved_file(ReservedChecker.reserved_reqs_path) + for key in ("specificationVersion", "fileVersion"): + self.assertRegex(data[key], re.compile(r"^\d+\.\d+\.\d+$"), key) + + def test_load_reserved_file_checks_the_file_version(self): + with open(ReservedChecker.reserved_reqs_path, encoding="utf-8") as file: + packaged = json.load(file) + with tempfile.TemporaryDirectory() as folder: + path = os.path.join(folder, "reserved_tags.json") + for version, accepted in (("1.1.0", True), ("1.0.3", True), ("2.0.0", False), ("0.1.0", False)): + with self.subTest(version=version): + with open(path, "w", encoding="utf-8") as file: + json.dump({**packaged, "fileVersion": version}, file) + if accepted: + self.assertEqual(load_reserved_file(path)["tags"], packaged["tags"]) + else: + with self.assertRaises(HedFileError) as context: + load_reserved_file(path) + self.assertEqual(context.exception.code, HedExceptions.INVALID_FILE_FORMAT) + def test_derived_sets(self): checker = ReservedChecker.get_instance() rows = {"Definition", "Delay", "Duration", "Event-context", "Inset", "Offset", "Onset"} diff --git a/tests/validator/test_spec_files.py b/tests/validator/test_spec_files.py new file mode 100644 index 00000000..f3b204f6 --- /dev/null +++ b/tests/validator/test_spec_files.py @@ -0,0 +1,72 @@ +"""The file-version gate for the specification's JSON files (specification Appendix A.6.1).""" + +import copy +import json +import os +import unittest + +from hed.errors.exceptions import HedExceptions, HedFileError +from hed.validator.util.spec_files import SUPPORTED_FILE_MAJOR, check_file_version + +DATA_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), "../../hed/validator/data")) + + +def load_packaged(filename): + """Return the parsed packaged copy of a specification file.""" + with open(os.path.join(DATA_DIR, filename), encoding="utf-8") as file: + return json.load(file) + + +class TestCheckFileVersion(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.packaged = {filename: load_packaged(filename) for filename in SUPPORTED_FILE_MAJOR} + + def with_version(self, filename, value, key=None): + """Return a copy of the packaged file with its version key set to value (or removed, for None).""" + data = copy.deepcopy(self.packaged[filename]) + key = key or SUPPORTED_FILE_MAJOR[filename][0] + data.pop(SUPPORTED_FILE_MAJOR[filename][0], None) + if value is not None: + data[key] = value + return data + + def assert_refused(self, data, filename): + with self.assertRaises(HedFileError) as context: + check_file_version(data, filename) + self.assertEqual(context.exception.code, HedExceptions.INVALID_FILE_FORMAT) + self.assertEqual(context.exception.filename, filename) + + def test_the_packaged_files_pass(self): + for filename, data in self.packaged.items(): + with self.subTest(filename=filename): + check_file_version(data, filename) + + def test_any_minor_and_patch_number_passes(self): + for filename in SUPPORTED_FILE_MAJOR: + for version in ("1.0.0", "1.4.0", "1.0.7", "1.12.30"): + with self.subTest(filename=filename, version=version): + check_file_version(self.with_version(filename, version), filename) + + def test_another_major_number_is_refused(self): + for filename in SUPPORTED_FILE_MAJOR: + for version in ("2.0.0", "0.9.0", "10.0.0"): + with self.subTest(filename=filename, version=version): + self.assert_refused(self.with_version(filename, version), filename) + + def test_a_missing_or_malformed_version_is_refused(self): + for filename in SUPPORTED_FILE_MAJOR: + for version in (None, "1.0", "v1.0.0", "1.0.0-beta", " 1.0.0", "1.0.0\n", "١.0.0", "", 1, 1.0): + with self.subTest(filename=filename, version=version): + self.assert_refused(self.with_version(filename, version), filename) + + def test_the_other_files_spelling_of_the_key_is_refused(self): + self.assert_refused(self.with_version("reserved_tags.json", "1.0.0", key="file_version"), "reserved_tags.json") + self.assert_refused(self.with_version("character_sets.json", "1.0.0", key="fileVersion"), "character_sets.json") + + def test_a_file_that_is_not_an_object_is_refused(self): + self.assert_refused([], "reserved_tags.json") + + +if __name__ == "__main__": + unittest.main()