From 4c59f7b823acedeb303c778f383747b68e81a5ed Mon Sep 17 00:00:00 2001 From: Kay Robbins Date: Sun, 8 Nov 2026 18:18:02 -0600 Subject: [PATCH 1/2] Speed up assembled validation; fix n/a onsets Assembled validation of a timeline file parsed every row, discarded the rows with an onset, and rechecked merged rows. Fixes #1481. - validate_assembled assembles once and skips a row with a numeric onset by row label before parsing. A row whose onset is n/a was tested by position in the sorted split frame, so it was skipped unless it was last; it is now checked wherever it is. - Delete _recheck_duplicates, _is_within_tolerance and ONSET_TOLERANCE: a repeat in merged rows was reported twice, and the 1e-9 merge tolerance in df_util is now the only one. - A bad Delay group is removed from a fresh parse, so the issue's HED string context keeps it. - needs_sorting ignores n/a onsets (no ONSETS_UNORDERED for them); EventManager keeps requiring every onset to be numeric. - combine_dataframe and _filter_by_index_list work on lists. - 20,000 rows: 4.29 s -> 1.01 s with onset, 2.30 s -> 1.05 s without. - Five regression tests; four tests no longer expect ONSETS_UNORDERED for n/a; row char_index now counts in the ", "-joined string. --- CHANGELOG.md | 6 + hed/models/base_input.py | 11 +- hed/models/df_util.py | 14 ++- hed/tools/analysis/event_manager.py | 8 +- hed/validator/spreadsheet_validator.py | 114 ++++++------------ tests/models/test_tabular_input.py | 6 +- tests/validator/test_hed_validator.py | 5 +- tests/validator/test_spreadsheet_validator.py | 88 ++++++++++++-- 8 files changed, 153 insertions(+), 99 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f8ce22f..c0da057d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ ## Validation - **A tag extension may contain only `name` characters** (letters, digits, hyphen, underscore, and from 8.3.0 non-ASCII), as specification 3.2.5 rule 2 requires. Period, plus, caret, blank, and colon were accepted before (`Item/my.ext`, `Item/a b`) and are now errors. Any character outside `name` in an extension, including ones that were already errors such as `$`, is now reported as `TAG_EXTENSION_INVALID` (Appendix B, reason b) instead of `CHARACTER_INVALID`, naming the character and its position. `#` stays `PLACEHOLDER_INVALID`. ([#1471](https://github.com/hed-standard/hed-python/issues/1471)) +- **Assembled validation is about 2 to 4 times faster** on large tables: about 4.3 s -> 1.0 s for 20,000 rows with an `onset` column, and 2.3 s -> 1.0 s without one. Rows are assembled once, a row with a numeric onset is no longer parsed and discarded before the onset checks, and merged rows are no longer checked a second time. ([#1481](https://github.com/hed-standard/hed-python/issues/1481)) +- **A repeated tag in rows merged at one onset is reported once**, not twice. ([#1481](https://github.com/hed-standard/hed-python/issues/1481)) +- **A row whose onset is `n/a` is validated wherever it is.** Such a row was skipped unless it was the last row of the file, so its assembled-row errors (for example a tag repeated across the `HED` column and the sidecar) went unreported. ([#1481](https://github.com/hed-standard/hed-python/issues/1481)) +- **`n/a` onsets no longer cause `ONSETS_UNORDERED`**: only the numeric onsets must be in order. `EventManager` still requires every onset to be numeric. +- **The HED string context of a `TEMPORAL_TAG_ERROR` for an unconvertible `Delay`** (`(Delay/3 month, (Red))`) keeps the `Delay` group; it was removed after the issue was made. +- In a file without an `onset` column, the `char_index` of an assembled-row issue now counts in the assembled string with `", "` between cells, as for files with onsets (one more per earlier cell than before). ## Specification files diff --git a/hed/models/base_input.py b/hed/models/base_input.py index d41e7a22..2c509a85 100644 --- a/hed/models/base_input.py +++ b/hed/models/base_input.py @@ -163,8 +163,9 @@ def needs_sorting(self) -> bool: """Return True if this both has an onset column, and it needs sorting.""" onsets = self.onsets if onsets is not None: + # A row without a numeric onset (n/a) is not part of the order (it is never merged or sorted). onsets = pd.to_numeric(self.dataframe["onset"], errors="coerce") - return not onsets.is_monotonic_increasing + return not onsets.dropna().is_monotonic_increasing else: return False @@ -676,8 +677,12 @@ def combine_dataframe(dataframe) -> pd.Series: Returns: pd.Series: The assembled series. """ - dataframe = dataframe.apply(lambda x: ", ".join(filter(lambda e: bool(e) and e != "n/a", map(str, x))), axis=1) - return dataframe + if len(dataframe.columns) == 0: + return pd.Series([""] * len(dataframe), index=dataframe.index) + # Column lists joined row by row: the same text as str() of each cell, without a per-row pandas Series. + columns = [[str(value) for value in dataframe[column].tolist()] for column in dataframe.columns] + joined = [", ".join(e for e in row if e and e != "n/a") for row in zip(*columns, strict=True)] + return pd.Series(joined, index=dataframe.index) def get_def_dict(self, hed_schema, extra_def_dicts=None) -> "DefinitionDict": """Return the definition dict for this file. diff --git a/hed/models/df_util.py b/hed/models/df_util.py index 658613b0..81188136 100644 --- a/hed/models/df_util.py +++ b/hed/models/df_util.py @@ -324,11 +324,15 @@ def _filter_by_index_list(original_data, indexed_dict): else: raise TypeError("Input must be a pandas Series or DataFrame") - new_series = pd.Series([""] * len(data_series), dtype=data_series.dtype) - for _onset, indices in indexed_dict.items(): - if indices: - first_index = indices[0] - new_series.iloc[first_index] = ",".join([str(data_series.iloc[i]) for i in indices]) + # Work on a list: only groups of two or more rows need a join, and positional pandas writes are slow. + values = [str(value) for value in data_series.tolist()] + joined = [""] * len(values) + for indices in indexed_dict.values(): + if len(indices) == 1: + joined[indices[0]] = values[indices[0]] + elif indices: + joined[indices[0]] = ",".join(values[i] for i in indices) + new_series = pd.Series(joined, dtype=data_series.dtype) if isinstance(original_data, pd.Series): return new_series diff --git a/hed/tools/analysis/event_manager.py b/hed/tools/analysis/event_manager.py index 7b227714..c3a86588 100644 --- a/hed/tools/analysis/event_manager.py +++ b/hed/tools/analysis/event_manager.py @@ -31,8 +31,12 @@ def __init__(self, input_data, hed_schema, extra_defs=None): are separated from the rest of the annotations, which are contained in self.hed_strings. """ - if input_data.onsets is not None and input_data.needs_sorting: - raise HedFileError("OnsetsNotOrdered", "Events must have numeric non-decreasing onset values", "") + if input_data.onsets is not None: + # Every onset must be numeric here, n/a included: event times are computed from them. (needs_sorting + # ignores n/a rows, which validation allows, so it is not the test for this.) + onsets = pd.to_numeric(input_data.onsets, errors="coerce") + if onsets.isna().any() or not onsets.is_monotonic_increasing: + raise HedFileError("OnsetsNotOrdered", "Events must have numeric non-decreasing onset values", "") self.hed_schema = hed_schema self.input_data = input_data self.def_dict = input_data.get_def_dict(hed_schema, extra_def_dicts=extra_defs) diff --git a/hed/validator/spreadsheet_validator.py b/hed/validator/spreadsheet_validator.py index abf5fdc5..e11eebf9 100644 --- a/hed/validator/spreadsheet_validator.py +++ b/hed/validator/spreadsheet_validator.py @@ -3,7 +3,6 @@ from __future__ import annotations import copy -import math import re import warnings @@ -66,7 +65,6 @@ class SpreadsheetValidator: SIDECAR_KEY_MISSING warning per column naming every unknown value, with no row, as before. """ - ONSET_TOLERANCE = 1e-7 # The rows of reserved_tags.json with requiresTimeline (specification 3.2.8 rule 4): they need a row time. TIMELINE_TAGS = frozenset(ReservedChecker.get_instance().timeline_tags) TEMPORAL_ANCHORS = re.compile(r"|".join(map(re.escape, sorted(tag.casefold() for tag in TIMELINE_TAGS)))) @@ -579,49 +577,54 @@ def validate_assembled(self, data, error_handler, row_offset=None) -> list[dict] issues += error_handler.format_error_with_context(ValidationErrors.ONSETS_UNORDERED) data = data_new - # If there are n/a errors in the onset column, further validation cannot proceed + # One assembly. A row with a numeric onset is checked after the onset merge (_run_onset_checks); every + # other row, which is every row of a file without an onset column, is checked on its own here. + assembled = data.series_a onsets = data.onsets + split = None + has_onset = None if onsets is not None: - onsets = onsets.astype(str).str.strip() - onsets = pd.to_numeric(onsets, errors="coerce") - assembled = data.series_a + onsets = pd.to_numeric(onsets.astype(str).str.strip(), errors="coerce") + # If there are n/a errors in the onset column, further validation cannot proceed na_issues = self._check_onset_nans(onsets, assembled, self._schema, error_handler, row_offset) issues += na_issues if len(na_issues) > 0: return issues assembled, time_issues = self._check_time_conversions(assembled, error_handler, row_offset) issues += time_issues - onsets = df_util.split_delay_tags(assembled, self._schema, onsets) - else: - onsets = None - - df = data.dataframe_a - - if onsets is not None: + split = df_util.split_delay_tags(assembled, self._schema, onsets) + # Keyed by row label: the split frame is sorted and renumbered, so it cannot say which row is which. + has_onset = onsets.notna().to_dict() self._onset_validator = OnsetValidator() - onset_mask = ~pd.isna(pd.to_numeric(onsets["onset"], errors="coerce")) else: self._onset_validator = None - onset_mask = None - issues += self._run_row_checks(df, error_handler=error_handler, row_offset=row_offset, onset_mask=onset_mask) - if self._onset_validator: - issues += self._run_onset_checks(onsets, error_handler=error_handler, row_offset=row_offset) - issues += self._recheck_duplicates(onsets, error_handler=error_handler, row_offset=row_offset) + issues += self._run_row_checks(assembled, error_handler, row_offset, has_onset) + if split is not None: + issues += self._run_onset_checks(split, error_handler=error_handler, row_offset=row_offset) return issues - def _run_row_checks(self, hed_df, error_handler, row_offset, onset_mask=None): - """Full-string checks on each assembled row of a non-timeline file (timeline rows are checked after - the onset merge in _run_onset_checks).""" + def _run_row_checks(self, assembled, error_handler, row_offset, has_onset=None): + """Full-string checks on each assembled row that has no numeric onset. + + Parameters: + assembled (pd.Series): The assembled HED strings, indexed by row label. + error_handler (ErrorHandler): Holds context. + row_offset (int): Added to the row label for reporting. + has_onset (dict or None): For a timeline file, row label -> True if the row has a numeric onset; such + rows are skipped here before they are parsed. None for a file without an onset column. + + Returns: + list[dict]: The issues found. + """ issues = [] - for row_number, text_file_row in hed_df.iterrows(): - row_strings = [HedString(cell, self._schema) for cell in text_file_row if cell and cell != "n/a"] - if not row_strings or (onset_mask is not None and onset_mask.iloc[row_number]): + for label, text in zip(assembled.index, assembled.tolist(), strict=True): + if not text or (has_onset is not None and has_onset[label]): continue - row_string = HedString.from_hed_strings(row_strings) + row_string = HedString(text, self._schema) if not row_string: continue - error_handler.push_error_context(ErrorContext.ROW, row_number + row_offset) + error_handler.push_error_context(ErrorContext.ROW, label + row_offset) error_handler.push_error_context(ErrorContext.HED_STRING, row_string) row_issues = self._hed_validator.run_full_string_checks(row_string) row_issues += OnsetValidator.check_for_banned_tags(row_string) @@ -647,54 +650,6 @@ def _run_onset_checks(self, onset_filtered, error_handler, row_offset): error_handler.pop_error_context() # Row return issues - def _recheck_duplicates(self, onset_filtered, error_handler, row_offset): - issues = [] - for i in range(len(onset_filtered) - 1): - current_row = onset_filtered.iloc[i] - next_row = onset_filtered.iloc[i + 1] - - # Skip if the HED column is empty or the onsets differ - if not current_row["HED"] or not self._is_within_tolerance(next_row["onset"], current_row["onset"]): - continue - - # At least two rows have been merged with their onsets recognized as the same. - error_handler.push_error_context(ErrorContext.ROW, current_row.original_index + row_offset) - row_string = HedString(current_row.HED, self._schema, self._hed_validator._def_validator) - error_handler.push_error_context(ErrorContext.HED_STRING, row_string) - new_column_issues = self._hed_validator.run_full_string_checks(row_string) - error_handler.add_context_and_filter(new_column_issues) - error_handler.pop_error_context() # HedString - issues += new_column_issues - error_handler.pop_error_context() # Row - - return issues - - def _is_within_tolerance(self, onset1, onset2): - """ - Checks if two onset strings are within the specified tolerance. - - Parameters: - onset1 (str): The first onset value as a string. - onset2 (str): The second onset value as a string. - - Returns: - bool: True if the values are within tolerance and valid, False otherwise. - """ - try: - # Convert to floats - onset1 = float(onset1) - onset2 = float(onset2) - - # Check if both values are finite - if not (math.isfinite(onset1) and math.isfinite(onset2)): - return False - - # Check if the difference is within tolerance - return abs(onset1 - onset2) <= self.ONSET_TOLERANCE - except ValueError: - # Return False if either value is not convertible to a float - return False - def _check_time_conversions(self, assembled, error_handler, row_offset): """Report Delay and Duration tags whose value cannot be converted to default units. @@ -745,8 +700,13 @@ def _check_time_conversions(self, assembled, error_handler, row_offset): if not copied: assembled = assembled.copy() copied = True - hed_obj.remove(bad_groups) - assembled.at[index] = str(hed_obj) + # The issues above hold hed_obj as their HED string context, so it must keep the bad groups. + # Remove them from a fresh parse instead, matching the groups by position. + positions = [i for i, group in enumerate(hed_obj.groups()) if any(group is b for b in bad_groups)] + trimmed = HedString(value, self._schema) + trimmed_groups = trimmed.groups() + trimmed.remove([trimmed_groups[i] for i in positions]) + assembled.at[index] = str(trimmed) return assembled, issues def _check_onset_nans(self, onsets, assembled, hed_schema, error_handler, row_offset): diff --git a/tests/models/test_tabular_input.py b/tests/models/test_tabular_input.py index ec478149..4cdb73ed 100644 --- a/tests/models/test_tabular_input.py +++ b/tests/models/test_tabular_input.py @@ -4,7 +4,7 @@ import unittest from hed import schema -from hed.errors import ErrorContext, ErrorHandler, HedFileError +from hed.errors import ErrorContext, ErrorHandler, HedFileError, ValidationErrors from hed.models import Sidecar, TabularInput @@ -116,10 +116,12 @@ def test_nan_issues(self): validation_issues = tab_input3.validate(hed_schema=self.hed_schema, error_handler=error_handler1) self.assertEqual(len(validation_issues), 2) + # With warnings on: no ONSETS_UNORDERED, since the n/a onsets are not part of the order. error_handler2 = ErrorHandler(check_for_warnings=True) tab_input3 = self.get_tabular(events3, sidecar) validation_issues = tab_input3.validate(hed_schema=self.hed_schema, error_handler=error_handler2) - self.assertEqual(len(validation_issues), 3) + self.assertEqual(len(validation_issues), 2) + self.assertNotIn(ValidationErrors.ONSETS_UNORDERED, [issue["code"] for issue in validation_issues]) def test_only_empty_and_na_cells_are_missing_in_a_text_file(self): """A TSV cell is missing only when it is empty or n/a; nan, NaN, None and NULL are values, as for a diff --git a/tests/validator/test_hed_validator.py b/tests/validator/test_hed_validator.py index 30b0f931..13b40071 100644 --- a/tests/validator/test_hed_validator.py +++ b/tests/validator/test_hed_validator.py @@ -198,8 +198,9 @@ def test_error_spans_from_file_and_missing_required_column(self): input_file = SpreadsheetInput(events_path, tag_columns=[0, 1, "error"]) validation_issues = input_file.validate(hed_schema=self.hed_schema) - self.assertEqual(validation_issues[1]["char_index"], 6) - self.assertEqual(validation_issues[2]["char_index"], 6) + # Indexes into the assembled row string, whose cells are joined with ", " ("Event, Event", "Event, ()"). + self.assertEqual(validation_issues[1]["char_index"], 7) + self.assertEqual(validation_issues[2]["char_index"], 7) self.assertEqual(len(validation_issues), 3) # todo: move this test somewhere more appropriate diff --git a/tests/validator/test_spreadsheet_validator.py b/tests/validator/test_spreadsheet_validator.py index e4f86fe6..192ec219 100644 --- a/tests/validator/test_spreadsheet_validator.py +++ b/tests/validator/test_spreadsheet_validator.py @@ -189,8 +189,9 @@ def test_onset_na(self): } df_with_nans = pd.DataFrame(tsv) issues = self.validator.validate(TabularInput(df_with_nans), extra_def_dicts=def_dict) - self.assertEqual(len(issues), 3) - self.assertEqual(issues[2]["code"], ValidationErrors.TEMPORAL_TAG_ERROR) + # The n/a onsets are not out of order (ONSETS_UNORDERED ignores rows without a numeric onset). + self.assertEqual(len(issues), 2) + self.assertEqual(issues[1]["code"], ValidationErrors.TEMPORAL_TAG_ERROR) # Test with sidecar sidecar_dict = { @@ -206,15 +207,14 @@ def test_onset_na(self): sidecar1 = Sidecar(io.StringIO(json.dumps(sidecar_dict))) issues1 = self.validator.validate(TabularInput(df_with_nans, sidecar=sidecar1), extra_def_dicts=def_dict) - self.assertEqual(len(issues1), 2) - self.assertEqual(issues1[1]["code"], ValidationErrors.TEMPORAL_TAG_ERROR) + self.assertEqual(len(issues1), 1) + self.assertEqual(issues1[0]["code"], ValidationErrors.TEMPORAL_TAG_ERROR) # The whatelse does not use the bad HED columns and HED column appears in {} so only when assembled is used. sidecar_dict["event_code"]["HED"]["whatever"] = "Black, {HED}" sidecar2 = Sidecar(io.StringIO(json.dumps(sidecar_dict))) issues2 = self.validator.validate(TabularInput(df_with_nans, sidecar=sidecar2), extra_def_dicts=def_dict) - self.assertEqual(len(issues2), 1) - self.assertEqual(issues1[0]["code"], ValidationErrors.ONSETS_UNORDERED) + self.assertEqual(issues2, []) def test_delay_without_conversion_factor(self): # 'Delay/3 month' is valid HED but month has no conversionFactor, so the delayed onset cannot be @@ -345,8 +345,8 @@ def test_validate_pops_context_on_onset_na_early_return(self): ) codes = {issue["code"] for issue in issues} - # The n/a onset also makes the onsets non-monotonic, which is reported alongside. - self.assertEqual(codes, {ValidationErrors.TEMPORAL_TAG_ERROR, ValidationErrors.ONSETS_UNORDERED}) + # The n/a onset is not out of order: ONSETS_UNORDERED ignores rows without a numeric onset. + self.assertEqual(codes, {ValidationErrors.TEMPORAL_TAG_ERROR}) self.assertTrue(all(issue.get("ec_filename") == "events.tsv" for issue in issues)) self.assertEqual(error_handler.error_context, []) @@ -547,3 +547,75 @@ def test_column_source(self): issues = SpreadsheetValidator(self.schema).validate(source, sidecar=sidecar, error_handler=ErrorHandler()) self.assertEqual([issue["code"] for issue in issues], [ValidationErrors.TABULAR_INVALID]) self.assertEqual(issues[0][ErrorContext.ROW], 1) + + +class TestAssembledRows(unittest.TestCase): + """Assembled-row checks of a timeline file (issue #1481): merged onsets, n/a onsets, Delay context.""" + + SIDECAR = {"event_code": {"HED": {"show": "Sensory-event, Visual-presentation", "press": "Agent-action"}}} + + @classmethod + def setUpClass(cls): + cls.schema = load_schema_version("8.4.0") + + def validate(self, table, warnings=False, sidecar=True): + sidecar_obj = Sidecar(io.StringIO(json.dumps(self.SIDECAR))) if sidecar else None + return SpreadsheetValidator(self.schema).validate( + TabularInput(pd.DataFrame(table), sidecar=sidecar_obj), + error_handler=ErrorHandler(check_for_warnings=warnings), + extra_def_dicts="(Definition/MyDef, (Label/x))", + ) + + def test_repeat_across_merged_rows_is_reported_once(self): + table = { + "onset": [1.0, 1.0, 2.0], + "event_code": ["show", "press", "press"], + "HED": ["Red", "Sensory-event", "Blue"], + } + issues = self.validate(table) + self.assertEqual( + [(issue["code"], issue[ErrorContext.ROW]) for issue in issues], + [(ValidationErrors.TAG_EXPRESSION_REPEATED, 2)], + ) + + def test_na_onset_row_is_checked_wherever_it_sits(self): + rows = [[1.5, "show", "Red"], [2.5, "press", "Blue"], [3.5, "show", "Green"]] + for position in (0, 1, 3): + with self.subTest(position=position): + table_rows = [list(row) for row in rows] + table_rows.insert(position, ["n/a", "show", "Sensory-event, Yellow"]) + table = { + "onset": [row[0] for row in table_rows], + "event_code": [row[1] for row in table_rows], + "HED": [row[2] for row in table_rows], + } + issues = self.validate(table) + # Row label plus 2 (the header line and 1-based lines). + self.assertEqual( + [(issue["code"], issue[ErrorContext.ROW]) for issue in issues], + [(ValidationErrors.TAG_EXPRESSION_REPEATED, position + 2)], + ) + + def test_onsets_closer_than_the_merge_tolerance_only_are_merged(self): + # 5e-8 apart is above the 1e-9 merge tolerance of df_util, so the rows are separate events. + table = {"onset": [1.0, 1.00000005], "HED": ["Sensory-event", "Sensory-event"]} + self.assertEqual(self.validate(table, sidecar=False), []) + table = {"onset": [1.0, 1.0000000001], "HED": ["Sensory-event", "Sensory-event"]} + self.assertEqual( + [issue["code"] for issue in self.validate(table, sidecar=False)], + [ValidationErrors.TAG_EXPRESSION_REPEATED], + ) + + def test_na_onsets_do_not_make_the_file_unordered(self): + table = {"onset": [1.0, "n/a", 2.0, "n/a", 3.0], "HED": ["Red", "Blue", "Green", "Yellow", "Black"]} + codes = [issue["code"] for issue in self.validate(table, warnings=True, sidecar=False)] + self.assertNotIn(ValidationErrors.ONSETS_UNORDERED, codes) + table = {"onset": [2.0, "n/a", 1.0], "HED": ["Red", "Blue", "Green"]} + codes = [issue["code"] for issue in self.validate(table, warnings=True, sidecar=False)] + self.assertIn(ValidationErrors.ONSETS_UNORDERED, codes) + + def test_delay_error_keeps_the_delay_in_its_context(self): + table = {"onset": [1.0, 2.0], "HED": ["(Delay/3 month, (Red))", "Blue"]} + issues = self.validate(table, sidecar=False) + self.assertEqual([issue["code"] for issue in issues], [ValidationErrors.TEMPORAL_TAG_ERROR]) + self.assertIn("Delay/3 month", str(issues[0][ErrorContext.HED_STRING])) From 48d4d5bbb6cb8856ac7e13578f7fbaf9256b2282 Mon Sep 17 00:00:00 2001 From: Kay Robbins Date: Mon, 9 Nov 2026 08:23:29 -0600 Subject: [PATCH 2/2] Pin hed-tests b0bd646 Moves spec_tests/hed-tests from 568ac2b to b0bd646, the merge of hed-tests PR #98, which adds the TAG_EXPRESSION_REPEATED case tag-expression-repeated-na-onset-row for #1481. --- spec_tests/hed-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec_tests/hed-tests b/spec_tests/hed-tests index 568ac2b9..b0bd646a 160000 --- a/spec_tests/hed-tests +++ b/spec_tests/hed-tests @@ -1 +1 @@ -Subproject commit 568ac2b92b8b2e38dc9528218b9ec922b0c67fd1 +Subproject commit b0bd646a042472662178add7b61fcbfb2f05c52e