Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@

- **A tag extension may contain only `name` characters** (letters, digits, hyphen, underscore, and from 8.3.0 non-ASCII), as specification 3.2.5 rule 2 requires. Period, plus, caret, blank, and colon were accepted before (`Item/my.ext`, `Item/a b`) and are now errors. Any character outside `name` in an extension, including ones that were already errors such as `$`, is now reported as `TAG_EXTENSION_INVALID` (Appendix B, reason b) instead of `CHARACTER_INVALID`, naming the character and its position. `#` stays `PLACEHOLDER_INVALID`. ([#1471](https://github.com/hed-standard/hed-python/issues/1471))

## Specification files

- **The packaged `reserved_tags.json` carries `specificationVersion` and `fileVersion`** (hed-specification #731), refreshed verbatim from hed-specification. Both specification files, `reserved_tags.json` and `character_sets.json`, are now refused with `HedFileError` `INVALID_FILE_FORMAT` when their file version is missing or its major number is not 1; any minor and patch number is accepted (specification Appendix A.6.1). The check lives in the new internal module `hed/validator/util/spec_files.py`. ([#1489](https://github.com/hed-standard/hed-python/issues/1489))

# Release 1.3.0 October 23, 2026

This release implements HED specification 4.0.0 (reserved tags, column names and padded values, tag-extension rules, value-text and character sets, BIDS date-times, compound units) and bundles HED 8.5.0. Many annotations that passed under 1.2.0 now give errors, and tabular validation reports issues differently: read the upgrade notes first. Runtime dependencies and supported Python versions (3.10 and later) are unchanged.
Expand Down
2 changes: 2 additions & 0 deletions hed/validator/data/reserved_tags.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
{
"title": "HED reserved tags",
"description": "Machine-readable form of the validation rules of the structural reserved tags, HED specification section 3.2.8 (Reserved tags). The tables in docs/source/03_HED_formats.md are generated from this file by scripts/generate_reserved_table.py, and Appendix A.6 of the specification defines every field. Validators load this file unchanged. Each row of tags gives the group rules of one tag; a Def tag or the Def-expand group that is its expanded form is the anchor these rows refer to and is never counted as a companion tag or an inner group. Every row applies in every standard schema version >= 8.0.0; requireValue, tagGroup and topLevelTagGroup supply what the 8.0.0 to 8.2.0 schemas lack on Delay and Duration. A row with secondaryWith is secondary in a group that also holds one of the listed tags: that tag's row gives the group rules (requiresDef, otherAllowedNonDefTags, minNonDefSubgroups, maxNonDefSubgroups, allowSpliceInGroup), the secondary row's group rules apply only when its tag is the only one of the table in its group, and its other fields apply wherever its tag appears. valueForms lists the tags whose value has two parts, name or name/value.",
"specificationVersion": "4.0.0",
"fileVersion": "1.0.0",
"tags": {
"Definition": {
"name": "Definition",
Expand Down
23 changes: 21 additions & 2 deletions hed/validator/reserved_checker.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
drifts from hed-specification main. Every row of the file applies in every standard schema version >= 8.0.0
(hed-specification decision 2026-10-15): its ``requireValue``, ``tagGroup`` and ``topLevelTagGroup`` fields
supply what the 8.0.0 to 8.2.0 schemas lack on ``Delay`` and ``Duration``, so no check here takes a version.
The loader refuses a file whose file version major number it does not support (specification Appendix A.6.1).
"""

import json
Expand All @@ -14,6 +15,25 @@

from hed.errors.error_reporter import ErrorHandler
from hed.errors.error_types import TemporalErrors, ValidationErrors
from hed.validator.util.spec_files import check_file_version


def load_reserved_file(path) -> dict:
"""Read a reserved_tags.json file, refusing a file version this hedtools does not support.

Parameters:
path (str): The path of a ``reserved_tags.json`` file.

Returns:
dict: The parsed content of the file.

Raises:
HedFileError: INVALID_FILE_FORMAT if the file's ``fileVersion`` is missing or has an unsupported major number.
"""
with open(path, encoding="utf-8") as file:
data = json.load(file)
check_file_version(data, "reserved_tags.json")
return data


class ReservedChecker:
Expand Down Expand Up @@ -52,8 +72,7 @@ def __new__(cls):
def _initialize(self):
# Load the JSON file during the first instantiation
if not hasattr(self, "reserved_map"):
with open(self.reserved_reqs_path, encoding="utf-8") as file:
data = json.load(file)
data = load_reserved_file(self.reserved_reqs_path)
self.reserved_map = data["tags"]
self.value_forms = data["valueForms"]
self._initialize_special_tags()
Expand Down
8 changes: 8 additions & 0 deletions hed/validator/util/character_sets.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
hed-specification (``docs/source/_static/character_sets.json``). Both validators load the same file, so the
named sets, the per-class defaults, the whole-value rules and the string-level exclusions cannot drift between
them or from the specification text, which is generated from the file. Update the copy; never edit it.
The loader refuses a file whose file version major number it does not support (specification Appendix A.6.1).
"""

import json
Expand All @@ -12,6 +13,8 @@

from semantic_version import Version

from hed.validator.util.spec_files import check_file_version

CHARACTER_SETS_FILENAME = "../data/character_sets.json"


Expand All @@ -33,7 +36,12 @@ def __init__(self, data):

Parameters:
data (dict): The parsed content of a ``character_sets.json`` file.

Raises:
HedFileError: INVALID_FILE_FORMAT if the file's ``file_version`` is missing or has an unsupported major
number.
"""
check_file_version(data, "character_sets.json")
self._data = data
self.sets = data["sets"]
self.aliases = data.get("aliases", {})
Expand Down
48 changes: 48 additions & 0 deletions hed/validator/util/spec_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
"""File-version gate for the JSON files that hedtools copies verbatim from the HED specification.

Specification Appendix A.6.1: each file carries a specification version and a file version. A validator MUST NOT
apply a file whose file version major number it does not support, and it accepts any minor and patch number under
a major number it supports. The two files spell the key differently, each in the style of its other keys.
"""

import re

from hed.errors.exceptions import HedExceptions, HedFileError

# For each specification file: the key that holds its file version and the major number this hedtools supports.
SUPPORTED_FILE_MAJOR = {
"reserved_tags.json": ("fileVersion", 1),
"character_sets.json": ("file_version", 1),
}

# ASCII digits only, matched against the whole string (fullmatch, so a trailing newline does not pass).
_VERSION_PATTERN = re.compile(r"([0-9]+)\.([0-9]+)\.([0-9]+)")


def check_file_version(data, filename) -> None:
"""Raise unless the file version of a specification file has a major number this hedtools supports.

Parameters:
data (dict): The parsed content of the file.
filename (str): The file's name, a key of SUPPORTED_FILE_MAJOR (``"reserved_tags.json"``).

Raises:
HedFileError: INVALID_FILE_FORMAT if the version key is missing, is not a ``major.minor.patch`` string,
or has a major number other than the supported one.
"""
key, supported_major = SUPPORTED_FILE_MAJOR[filename]
version = data.get(key) if isinstance(data, dict) else None
match = _VERSION_PATTERN.fullmatch(version) if isinstance(version, str) else None
if match is None:
raise HedFileError(
HedExceptions.INVALID_FILE_FORMAT,
f"{filename} has no valid '{key}' (expected major.minor.patch, found {version!r}).",
filename,
)
if int(match.group(1)) != supported_major:
raise HedFileError(
HedExceptions.INVALID_FILE_FORMAT,
f"{filename} has {key} {version}; this version of hedtools supports only file version "
f"{supported_major}.x.x of this file (specification Appendix A.6.1).",
filename,
)
14 changes: 14 additions & 0 deletions tests/validator/test_character_sets.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import re
import unittest

from hed.errors.exceptions import HedExceptions, HedFileError
from hed.validator.util.character_sets import CHARACTER_SETS_FILENAME, CharacterSets, code_point


Expand All @@ -22,6 +23,8 @@ def setUpClass(cls):

def test_the_file_has_every_section_the_validators_read(self):
for key in (
"specification_version",
"file_version",
"sets",
"aliases",
"literal_names",
Expand All @@ -34,6 +37,17 @@ def test_the_file_has_every_section_the_validators_read(self):
self.assertEqual(self.raw["declaration_wins_from"]["standard_version"], "8.5.0")
self.assertEqual(self.sets.declaration_wins_from, "8.5.0")

def test_a_copy_with_an_unsupported_file_version_is_refused(self):
for version, accepted in (("1.2.3", True), ("1.0.0", True), ("2.0.0", False), ("0.9.9", False)):
with self.subTest(version=version):
data = {**self.raw, "file_version": version}
if accepted:
self.assertEqual(CharacterSets(data).sets, self.sets.sets)
else:
with self.assertRaises(HedFileError) as context:
CharacterSets(data)
self.assertEqual(context.exception.code, HedExceptions.INVALID_FILE_FORMAT)

def test_every_set_regex_compiles_and_single_character_sets_match_their_code(self):
for name, entry in self.sets.sets.items():
with self.subTest(name=name):
Expand Down
28 changes: 26 additions & 2 deletions tests/validator/test_reserved_checker.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,16 @@
import io
import json
import os
import re
import tempfile
import unittest

from hed import HedString, Sidecar, TabularInput, load_schema_version, schema
from hed.errors import ErrorSeverity, ValidationErrors
from hed.errors.exceptions import HedExceptions, HedFileError
from hed.models import DefinitionDict
from hed.validator import HedValidator
from hed.validator.reserved_checker import ReservedChecker
from hed.validator.reserved_checker import ReservedChecker, load_reserved_file
from hed.validator.spreadsheet_validator import SpreadsheetValidator

DEFINITIONS = ["(Definition/MyDef, (Red))", "(Definition/Other, (Blue))"]
Expand Down Expand Up @@ -54,7 +57,7 @@ def timeline_codes(self, *hed_cells):


class TestReservedCheckerData(unittest.TestCase):
"""The loader reads the published file's two top-level keys."""
"""The loader reads the published file's tags and valueForms, after checking its fileVersion."""

def test_file_is_the_published_reserved_tags_json(self):
checker = ReservedChecker.get_instance()
Expand All @@ -64,6 +67,27 @@ def test_file_is_the_published_reserved_tags_json(self):
self.assertEqual(checker.reserved_map, data["tags"])
self.assertEqual(checker.value_forms, data["valueForms"])

def test_the_file_carries_its_version_fields(self):
data = load_reserved_file(ReservedChecker.reserved_reqs_path)
for key in ("specificationVersion", "fileVersion"):
self.assertRegex(data[key], re.compile(r"^\d+\.\d+\.\d+$"), key)

def test_load_reserved_file_checks_the_file_version(self):
with open(ReservedChecker.reserved_reqs_path, encoding="utf-8") as file:
packaged = json.load(file)
with tempfile.TemporaryDirectory() as folder:
path = os.path.join(folder, "reserved_tags.json")
for version, accepted in (("1.1.0", True), ("1.0.3", True), ("2.0.0", False), ("0.1.0", False)):
with self.subTest(version=version):
with open(path, "w", encoding="utf-8") as file:
json.dump({**packaged, "fileVersion": version}, file)
if accepted:
self.assertEqual(load_reserved_file(path)["tags"], packaged["tags"])
else:
with self.assertRaises(HedFileError) as context:
load_reserved_file(path)
self.assertEqual(context.exception.code, HedExceptions.INVALID_FILE_FORMAT)

def test_derived_sets(self):
checker = ReservedChecker.get_instance()
rows = {"Definition", "Delay", "Duration", "Event-context", "Inset", "Offset", "Onset"}
Expand Down
72 changes: 72 additions & 0 deletions tests/validator/test_spec_files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""The file-version gate for the specification's JSON files (specification Appendix A.6.1)."""

import copy
import json
import os
import unittest

from hed.errors.exceptions import HedExceptions, HedFileError
from hed.validator.util.spec_files import SUPPORTED_FILE_MAJOR, check_file_version

DATA_DIR = os.path.realpath(os.path.join(os.path.dirname(__file__), "../../hed/validator/data"))


def load_packaged(filename):
"""Return the parsed packaged copy of a specification file."""
with open(os.path.join(DATA_DIR, filename), encoding="utf-8") as file:
return json.load(file)


class TestCheckFileVersion(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.packaged = {filename: load_packaged(filename) for filename in SUPPORTED_FILE_MAJOR}

def with_version(self, filename, value, key=None):
"""Return a copy of the packaged file with its version key set to value (or removed, for None)."""
data = copy.deepcopy(self.packaged[filename])
key = key or SUPPORTED_FILE_MAJOR[filename][0]
data.pop(SUPPORTED_FILE_MAJOR[filename][0], None)
if value is not None:
data[key] = value
return data

def assert_refused(self, data, filename):
with self.assertRaises(HedFileError) as context:
check_file_version(data, filename)
self.assertEqual(context.exception.code, HedExceptions.INVALID_FILE_FORMAT)
self.assertEqual(context.exception.filename, filename)

def test_the_packaged_files_pass(self):
for filename, data in self.packaged.items():
with self.subTest(filename=filename):
check_file_version(data, filename)

def test_any_minor_and_patch_number_passes(self):
for filename in SUPPORTED_FILE_MAJOR:
for version in ("1.0.0", "1.4.0", "1.0.7", "1.12.30"):
with self.subTest(filename=filename, version=version):
check_file_version(self.with_version(filename, version), filename)

def test_another_major_number_is_refused(self):
for filename in SUPPORTED_FILE_MAJOR:
for version in ("2.0.0", "0.9.0", "10.0.0"):
with self.subTest(filename=filename, version=version):
self.assert_refused(self.with_version(filename, version), filename)

def test_a_missing_or_malformed_version_is_refused(self):
for filename in SUPPORTED_FILE_MAJOR:
for version in (None, "1.0", "v1.0.0", "1.0.0-beta", " 1.0.0", "1.0.0\n", "١.0.0", "", 1, 1.0):
with self.subTest(filename=filename, version=version):
self.assert_refused(self.with_version(filename, version), filename)

def test_the_other_files_spelling_of_the_key_is_refused(self):
self.assert_refused(self.with_version("reserved_tags.json", "1.0.0", key="file_version"), "reserved_tags.json")
self.assert_refused(self.with_version("character_sets.json", "1.0.0", key="fileVersion"), "character_sets.json")

def test_a_file_that_is_not_an_object_is_refused(self):
self.assert_refused([], "reserved_tags.json")


if __name__ == "__main__":
unittest.main()
Loading