Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@
## Validation

- **A tag extension may contain only `name` characters** (letters, digits, hyphen, underscore, and from 8.3.0 non-ASCII), as specification 3.2.5 rule 2 requires. Period, plus, caret, blank, and colon were accepted before (`Item/my.ext`, `Item/a b`) and are now errors. Any character outside `name` in an extension, including ones that were already errors such as `$`, is now reported as `TAG_EXTENSION_INVALID` (Appendix B, reason b) instead of `CHARACTER_INVALID`, naming the character and its position. `#` stays `PLACEHOLDER_INVALID`. ([#1471](https://github.com/hed-standard/hed-python/issues/1471))
- **Assembled validation is about 2 to 4 times faster** on large tables: about 4.3 s -> 1.0 s for 20,000 rows with an `onset` column, and 2.3 s -> 1.0 s without one. Rows are assembled once, a row with a numeric onset is no longer parsed and discarded before the onset checks, and merged rows are no longer checked a second time. ([#1481](https://github.com/hed-standard/hed-python/issues/1481))
- **A repeated tag in rows merged at one onset is reported once**, not twice. ([#1481](https://github.com/hed-standard/hed-python/issues/1481))
- **A row whose onset is `n/a` is validated wherever it is.** Such a row was skipped unless it was the last row of the file, so its assembled-row errors (for example a tag repeated across the `HED` column and the sidecar) went unreported. ([#1481](https://github.com/hed-standard/hed-python/issues/1481))
- **`n/a` onsets no longer cause `ONSETS_UNORDERED`**: only the numeric onsets must be in order. `EventManager` still requires every onset to be numeric.
- **The HED string context of a `TEMPORAL_TAG_ERROR` for an unconvertible `Delay`** (`(Delay/3 month, (Red))`) keeps the `Delay` group; it was removed after the issue was made.
- In a file without an `onset` column, the `char_index` of an assembled-row issue now counts in the assembled string with `", "` between cells, as for files with onsets (one more per earlier cell than before).

## Specification files

Expand Down
11 changes: 8 additions & 3 deletions hed/models/base_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,9 @@ def needs_sorting(self) -> bool:
"""Return True if this both has an onset column, and it needs sorting."""
onsets = self.onsets
if onsets is not None:
# A row without a numeric onset (n/a) is not part of the order (it is never merged or sorted).
onsets = pd.to_numeric(self.dataframe["onset"], errors="coerce")
return not onsets.is_monotonic_increasing
return not onsets.dropna().is_monotonic_increasing
else:
return False

Expand Down Expand Up @@ -676,8 +677,12 @@ def combine_dataframe(dataframe) -> pd.Series:
Returns:
pd.Series: The assembled series.
"""
dataframe = dataframe.apply(lambda x: ", ".join(filter(lambda e: bool(e) and e != "n/a", map(str, x))), axis=1)
return dataframe
if len(dataframe.columns) == 0:
return pd.Series([""] * len(dataframe), index=dataframe.index)
# Column lists joined row by row: the same text as str() of each cell, without a per-row pandas Series.
columns = [[str(value) for value in dataframe[column].tolist()] for column in dataframe.columns]
joined = [", ".join(e for e in row if e and e != "n/a") for row in zip(*columns, strict=True)]
return pd.Series(joined, index=dataframe.index)

def get_def_dict(self, hed_schema, extra_def_dicts=None) -> "DefinitionDict":
"""Return the definition dict for this file.
Expand Down
14 changes: 9 additions & 5 deletions hed/models/df_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,11 +324,15 @@ def _filter_by_index_list(original_data, indexed_dict):
else:
raise TypeError("Input must be a pandas Series or DataFrame")

new_series = pd.Series([""] * len(data_series), dtype=data_series.dtype)
for _onset, indices in indexed_dict.items():
if indices:
first_index = indices[0]
new_series.iloc[first_index] = ",".join([str(data_series.iloc[i]) for i in indices])
# Work on a list: only groups of two or more rows need a join, and positional pandas writes are slow.
values = [str(value) for value in data_series.tolist()]
joined = [""] * len(values)
for indices in indexed_dict.values():
if len(indices) == 1:
joined[indices[0]] = values[indices[0]]
elif indices:
joined[indices[0]] = ",".join(values[i] for i in indices)
new_series = pd.Series(joined, dtype=data_series.dtype)

if isinstance(original_data, pd.Series):
return new_series
Expand Down
8 changes: 6 additions & 2 deletions hed/tools/analysis/event_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,12 @@ def __init__(self, input_data, hed_schema, extra_defs=None):
are separated from the rest of the annotations, which are contained in self.hed_strings.

"""
if input_data.onsets is not None and input_data.needs_sorting:
raise HedFileError("OnsetsNotOrdered", "Events must have numeric non-decreasing onset values", "")
if input_data.onsets is not None:
# Every onset must be numeric here, n/a included: event times are computed from them. (needs_sorting
# ignores n/a rows, which validation allows, so it is not the test for this.)
onsets = pd.to_numeric(input_data.onsets, errors="coerce")
if onsets.isna().any() or not onsets.is_monotonic_increasing:
raise HedFileError("OnsetsNotOrdered", "Events must have numeric non-decreasing onset values", "")
self.hed_schema = hed_schema
self.input_data = input_data
self.def_dict = input_data.get_def_dict(hed_schema, extra_def_dicts=extra_defs)
Expand Down
114 changes: 37 additions & 77 deletions hed/validator/spreadsheet_validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
from __future__ import annotations

import copy
import math
import re
import warnings

Expand Down Expand Up @@ -66,7 +65,6 @@ class SpreadsheetValidator:
SIDECAR_KEY_MISSING warning per column naming every unknown value, with no row, as before.
"""

ONSET_TOLERANCE = 1e-7
# The rows of reserved_tags.json with requiresTimeline (specification 3.2.8 rule 4): they need a row time.
TIMELINE_TAGS = frozenset(ReservedChecker.get_instance().timeline_tags)
TEMPORAL_ANCHORS = re.compile(r"|".join(map(re.escape, sorted(tag.casefold() for tag in TIMELINE_TAGS))))
Expand Down Expand Up @@ -579,49 +577,54 @@ def validate_assembled(self, data, error_handler, row_offset=None) -> list[dict]
issues += error_handler.format_error_with_context(ValidationErrors.ONSETS_UNORDERED)
data = data_new

# If there are n/a errors in the onset column, further validation cannot proceed
# One assembly. A row with a numeric onset is checked after the onset merge (_run_onset_checks); every
# other row, which is every row of a file without an onset column, is checked on its own here.
assembled = data.series_a
onsets = data.onsets
split = None
has_onset = None
if onsets is not None:
onsets = onsets.astype(str).str.strip()
onsets = pd.to_numeric(onsets, errors="coerce")
assembled = data.series_a
onsets = pd.to_numeric(onsets.astype(str).str.strip(), errors="coerce")
# If there are n/a errors in the onset column, further validation cannot proceed
na_issues = self._check_onset_nans(onsets, assembled, self._schema, error_handler, row_offset)
issues += na_issues
if len(na_issues) > 0:
return issues
assembled, time_issues = self._check_time_conversions(assembled, error_handler, row_offset)
issues += time_issues
onsets = df_util.split_delay_tags(assembled, self._schema, onsets)
else:
onsets = None

df = data.dataframe_a

if onsets is not None:
split = df_util.split_delay_tags(assembled, self._schema, onsets)
# Keyed by row label: the split frame is sorted and renumbered, so it cannot say which row is which.
has_onset = onsets.notna().to_dict()
self._onset_validator = OnsetValidator()
onset_mask = ~pd.isna(pd.to_numeric(onsets["onset"], errors="coerce"))
else:
self._onset_validator = None
onset_mask = None

issues += self._run_row_checks(df, error_handler=error_handler, row_offset=row_offset, onset_mask=onset_mask)
if self._onset_validator:
issues += self._run_onset_checks(onsets, error_handler=error_handler, row_offset=row_offset)
issues += self._recheck_duplicates(onsets, error_handler=error_handler, row_offset=row_offset)
issues += self._run_row_checks(assembled, error_handler, row_offset, has_onset)
if split is not None:
issues += self._run_onset_checks(split, error_handler=error_handler, row_offset=row_offset)
return issues

def _run_row_checks(self, hed_df, error_handler, row_offset, onset_mask=None):
"""Full-string checks on each assembled row of a non-timeline file (timeline rows are checked after
the onset merge in _run_onset_checks)."""
def _run_row_checks(self, assembled, error_handler, row_offset, has_onset=None):
"""Full-string checks on each assembled row that has no numeric onset.

Parameters:
assembled (pd.Series): The assembled HED strings, indexed by row label.
error_handler (ErrorHandler): Holds context.
row_offset (int): Added to the row label for reporting.
has_onset (dict or None): For a timeline file, row label -> True if the row has a numeric onset; such
rows are skipped here before they are parsed. None for a file without an onset column.

Returns:
list[dict]: The issues found.
"""
issues = []
for row_number, text_file_row in hed_df.iterrows():
row_strings = [HedString(cell, self._schema) for cell in text_file_row if cell and cell != "n/a"]
if not row_strings or (onset_mask is not None and onset_mask.iloc[row_number]):
for label, text in zip(assembled.index, assembled.tolist(), strict=True):
if not text or (has_onset is not None and has_onset[label]):
continue
row_string = HedString.from_hed_strings(row_strings)
row_string = HedString(text, self._schema)
if not row_string:
continue
error_handler.push_error_context(ErrorContext.ROW, row_number + row_offset)
error_handler.push_error_context(ErrorContext.ROW, label + row_offset)
error_handler.push_error_context(ErrorContext.HED_STRING, row_string)
row_issues = self._hed_validator.run_full_string_checks(row_string)
row_issues += OnsetValidator.check_for_banned_tags(row_string)
Expand All @@ -647,54 +650,6 @@ def _run_onset_checks(self, onset_filtered, error_handler, row_offset):
error_handler.pop_error_context() # Row
return issues

def _recheck_duplicates(self, onset_filtered, error_handler, row_offset):
issues = []
for i in range(len(onset_filtered) - 1):
current_row = onset_filtered.iloc[i]
next_row = onset_filtered.iloc[i + 1]

# Skip if the HED column is empty or the onsets differ
if not current_row["HED"] or not self._is_within_tolerance(next_row["onset"], current_row["onset"]):
continue

# At least two rows have been merged with their onsets recognized as the same.
error_handler.push_error_context(ErrorContext.ROW, current_row.original_index + row_offset)
row_string = HedString(current_row.HED, self._schema, self._hed_validator._def_validator)
error_handler.push_error_context(ErrorContext.HED_STRING, row_string)
new_column_issues = self._hed_validator.run_full_string_checks(row_string)
error_handler.add_context_and_filter(new_column_issues)
error_handler.pop_error_context() # HedString
issues += new_column_issues
error_handler.pop_error_context() # Row

return issues

def _is_within_tolerance(self, onset1, onset2):
"""
Checks if two onset strings are within the specified tolerance.

Parameters:
onset1 (str): The first onset value as a string.
onset2 (str): The second onset value as a string.

Returns:
bool: True if the values are within tolerance and valid, False otherwise.
"""
try:
# Convert to floats
onset1 = float(onset1)
onset2 = float(onset2)

# Check if both values are finite
if not (math.isfinite(onset1) and math.isfinite(onset2)):
return False

# Check if the difference is within tolerance
return abs(onset1 - onset2) <= self.ONSET_TOLERANCE
except ValueError:
# Return False if either value is not convertible to a float
return False

def _check_time_conversions(self, assembled, error_handler, row_offset):
"""Report Delay and Duration tags whose value cannot be converted to default units.

Expand Down Expand Up @@ -745,8 +700,13 @@ def _check_time_conversions(self, assembled, error_handler, row_offset):
if not copied:
assembled = assembled.copy()
copied = True
hed_obj.remove(bad_groups)
assembled.at[index] = str(hed_obj)
# The issues above hold hed_obj as their HED string context, so it must keep the bad groups.
# Remove them from a fresh parse instead, matching the groups by position.
positions = [i for i, group in enumerate(hed_obj.groups()) if any(group is b for b in bad_groups)]
trimmed = HedString(value, self._schema)
trimmed_groups = trimmed.groups()
trimmed.remove([trimmed_groups[i] for i in positions])
assembled.at[index] = str(trimmed)
return assembled, issues

def _check_onset_nans(self, onsets, assembled, hed_schema, error_handler, row_offset):
Expand Down
6 changes: 4 additions & 2 deletions tests/models/test_tabular_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import unittest

from hed import schema
from hed.errors import ErrorContext, ErrorHandler, HedFileError
from hed.errors import ErrorContext, ErrorHandler, HedFileError, ValidationErrors
from hed.models import Sidecar, TabularInput


Expand Down Expand Up @@ -116,10 +116,12 @@ def test_nan_issues(self):
validation_issues = tab_input3.validate(hed_schema=self.hed_schema, error_handler=error_handler1)
self.assertEqual(len(validation_issues), 2)

# With warnings on: no ONSETS_UNORDERED, since the n/a onsets are not part of the order.
error_handler2 = ErrorHandler(check_for_warnings=True)
tab_input3 = self.get_tabular(events3, sidecar)
validation_issues = tab_input3.validate(hed_schema=self.hed_schema, error_handler=error_handler2)
self.assertEqual(len(validation_issues), 3)
self.assertEqual(len(validation_issues), 2)
self.assertNotIn(ValidationErrors.ONSETS_UNORDERED, [issue["code"] for issue in validation_issues])

def test_only_empty_and_na_cells_are_missing_in_a_text_file(self):
"""A TSV cell is missing only when it is empty or n/a; nan, NaN, None and NULL are values, as for a
Expand Down
5 changes: 3 additions & 2 deletions tests/validator/test_hed_validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,8 +198,9 @@ def test_error_spans_from_file_and_missing_required_column(self):

input_file = SpreadsheetInput(events_path, tag_columns=[0, 1, "error"])
validation_issues = input_file.validate(hed_schema=self.hed_schema)
self.assertEqual(validation_issues[1]["char_index"], 6)
self.assertEqual(validation_issues[2]["char_index"], 6)
# Indexes into the assembled row string, whose cells are joined with ", " ("Event, Event", "Event, ()").
self.assertEqual(validation_issues[1]["char_index"], 7)
self.assertEqual(validation_issues[2]["char_index"], 7)
self.assertEqual(len(validation_issues), 3)

# todo: move this test somewhere more appropriate
Expand Down
Loading
Loading